20260924 - Boot guard, stack-health inventory, watchdog fix

.github/workflows/test.yml

@@ -1,8 +1,8 @@
 name: Tests
 
-# The forked docker-compose Update Module runs every retina-node install on
-# every node. These tests run the real module under /bin/sh (dash on Ubuntu,
-# as on the nodes) against a stub docker and jq, with real tar payloads.
+# The forked docker-compose Update Module, retina-env-guard and the stack-health
+# inventory script run as root on every node. These tests run the real scripts
+# under /bin/sh (dash on Ubuntu, as on the nodes) against a stub docker.
 
 on:
   push:
@@ -18,7 +18,10 @@ jobs:
         with:
           python-version: "3.12"
       - run: pip install 'pytest>=8.0'
-      - name: Module is executable
-        run: test -x plugins/playbooks/board_support/roles/mender/files/update-modules/docker-compose
+      - name: Scripts are executable
+        run: |
+          test -x plugins/playbooks/board_support/roles/mender/files/update-modules/docker-compose
+          test -x plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard
+          test -x configuration/mender/inventory/mender-inventory-retina-stack
       - name: Tests
-        run: pytest tests/update-modules/
+        run: pytest tests/

configuration/mender/inventory/mender-inventory-retina-stack

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+# Reports the retina-node stack's health from outside the stack.
+#
+# retina-telemetry runs inside the same compose project, so a node whose stack
+# is down goes silent instead of reporting unhealthy. ret9573ecda sat with no
+# radar for four weeks that way. mender-updated runs this every inventory poll
+# (600 s) whether or not a single container is up.
+#
+#   retina_stack             up | degraded | down | absent
+#   retina_stack_running     project containers running
+#   retina_stack_restarting  project containers in a restart loop
+#   retina_blah2             blah2's state (running, restarting, exited, missing)
+#   retina_env_ok            manifests/.env parses (false: NUL bytes)
+#   retina_compose_ok        compose can load the project
+#   retina_mode              radar | spectrum | sdrconnect (retina-gui's mode.txt)
+#
+# In spectrum and sdrconnect modes blah2 is stopped on purpose, so it does not
+# make the stack degraded there.
+
+MANIFESTS="${RETINA_STACK_MANIFESTS:-/data/mender-docker-compose/current/manifests}"
+MODE_FILE="${RETINA_STACK_MODE_FILE:-/data/retina-gui/mode.txt}"
+
+if [ ! -f "$MANIFESTS/docker-compose.yaml" ]; then
+    echo retina_stack=absent
+    exit 0
+fi
+
+mode=radar
+[ -s "$MODE_FILE" ] && mode=$(cat "$MODE_FILE")
+
+env_ok=true
+if [ -f "$MANIFESTS/.env" ] \
+    && [ "$(tr -d '\000' < "$MANIFESTS/.env" | wc -c)" -ne "$(wc -c < "$MANIFESTS/.env")" ]; then
+    env_ok=false
+fi
+
+compose_ok=true
+(cd "$MANIFESTS" && timeout 20 docker compose -p retina-node config -q > /dev/null 2>&1) || compose_ok=false
+
+count() {
+    timeout 10 docker ps -q --filter label=com.docker.compose.project=retina-node --filter "status=$1" 2>/dev/null | wc -l
+}
+running=$(count running)
+restarting=$(count restarting)
+blah2=$(timeout 10 docker inspect -f '{{.State.Status}}' blah2 2>/dev/null) || blah2=missing
+
+state=up
+if [ "$running" -eq 0 ]; then
+    state=down
+elif [ "$restarting" -gt 0 ] || [ "$env_ok" = false ] || [ "$compose_ok" = false ] \
+    || { [ "$mode" = radar ] && [ "$blah2" != running ]; }; then
+    state=degraded
+fi
+
+echo "retina_stack=$state"
+echo "retina_stack_running=$running"
+echo "retina_stack_restarting=$restarting"
+echo "retina_blah2=$blah2"
+echo "retina_env_ok=$env_ok"
+echo "retina_compose_ok=$compose_ok"
+echo "retina_mode=$mode"
+exit 0

plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard

@@ -0,0 +1,40 @@
+#!/bin/sh
+# Set aside a compose .env that compose cannot parse, before retina-node.service
+# starts the stack.
+#
+# The config-merger writes manifests/.env seconds after every boot, and a power
+# cut during that write has left it entirely NUL (ret9573ecda, 2026-08-27).
+# Compose then refuses to load the project at all, including the
+# `run config-merger` that would regenerate the file, so the node stays down
+# through every reboot until someone removes it by hand. Moving it aside lets
+# compose fall back to defaults for one start, and the config-merger writes a
+# good one from user.yml. The retina-node artifact's preflight does the same
+# during an install; this covers the boots in between.
+#
+# Best effort: nothing is deleted, and it always exits 0.
+
+MANIFESTS="${RETINA_ENV_GUARD_MANIFESTS:-/data/mender-docker-compose/current/manifests}"
+env_file="$MANIFESTS/.env"
+
+[ -f "$env_file" ] || exit 0
+
+reason=""
+if [ "$(tr -d '\000' < "$env_file" | wc -c)" -ne "$(wc -c < "$env_file")" ]; then
+    reason="contains NUL bytes"
+elif compose_err=$(cd "$MANIFESTS" && docker compose -p retina-node config -q 2>&1); then
+    :
+else
+    case "$compose_err" in
+        *"$env_file"*|*"/.env:"*) reason="compose cannot parse it" ;;
+    esac
+fi
+[ -n "$reason" ] || exit 0
+
+aside="$env_file.corrupt-$(date -u +%Y%m%dT%H%M%SZ)"
+if mv "$env_file" "$aside"; then
+    sync
+    echo "retina-env-guard: set aside $env_file ($reason) as $aside; the config-merger regenerates it"
+else
+    echo "retina-env-guard: could not set aside $env_file ($reason)"
+fi
+exit 0

plugins/playbooks/os_setup/roles/radar_data_bootstrap/tasks/main.yml

@@ -93,6 +93,17 @@
       ADSBLOL_ENABLED=false
       ADSBLOL_RADIUS=40
 
+# 3a. Boot guard: set aside a compose .env that compose cannot parse, so a
+# power cut during the config-merger's write cannot keep the stack down
+# through every reboot. See the script's header.
+- name: Install retina-env-guard
+  copy:
+    src: retina-env-guard
+    dest: /usr/local/sbin/retina-env-guard
+    mode: '0755'
+    owner: root
+    group: root
+
 # 3. Create systemd service for retina-node auto-start after OS updates
 - name: Create retina-node auto-start service
   copy:
@@ -113,6 +124,8 @@
       RemainAfterExit=yes
       WorkingDirectory={{ retina_node_manifests_path }}
       Environment="RETINA_NODE_PATH={{ retina_node_manifests_path }}"
+      # "-": the guard is best effort and must never stop the stack starting.
+      ExecStartPre=-/usr/local/sbin/retina-env-guard
       ExecStart=/bin/bash -c '/usr/bin/docker compose -p retina-node run --rm config-merger && /usr/bin/docker compose -p retina-node up -d --remove-orphans'
       StartLimitBurst=3
       StartLimitIntervalSec=300

plugins/playbooks/os_setup/roles/radar_packages/tasks/main.yml

@@ -592,6 +592,9 @@
       # one instance blocked while cron keeps starting more every 5 minutes.
       FIRST_CHAR=$(curl -s --max-time 10 127.0.0.1:3000/api/map | head -c1)
       TIMESTAMP=$(curl -s --max-time 10 127.0.0.1:3000/api/map | head -c23 | tail -c10)
+      # No answer, or not a number, reads as stale. Left empty, the subtraction
+      # below failed with "operand expected" whenever blah2-api was down.
+      case "$TIMESTAMP" in ''|*[!0-9]*) TIMESTAMP=0 ;; esac
       CURR_TIMESTAMP=$(date +%s)
       DIFF_TIMESTAMP=$(($CURR_TIMESTAMP-$TIMESTAMP))
 

tests/stack-health/test_stack_health.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+Tests for retina-env-guard (retina-node.service ExecStartPre) and the
+mender-inventory-retina-stack inventory script.
+
+Both run as root on every node: the guard at every boot, the inventory script
+every 600 s. Each test runs the real script under /bin/sh with a stub docker.
+"""
+
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+GUARD = os.path.join(REPO, 'plugins', 'playbooks', 'os_setup', 'roles', 'radar_data_bootstrap',
+                     'files', 'retina-env-guard')
+INVENTORY = os.path.join(REPO, 'configuration', 'mender', 'inventory', 'mender-inventory-retina-stack')
+
+STUB_DOCKER = r'''#!/bin/sh
+echo "$*" >> "$STUB/calls"
+case "$1" in
+    compose)
+        if [ -f "$STUB/compose_error" ]; then cat "$STUB/compose_error" >&2; exit 1; fi
+        exit 0 ;;
+    ps)
+        for a; do case "$a" in status=*) s=${a#status=} ;; esac; done
+        n=$(cat "$STUB/$s" 2>/dev/null || echo 0)
+        i=0; while [ "$i" -lt "$n" ]; do echo "c$i"; i=$((i+1)); done
+        exit 0 ;;
+    inspect)
+        [ -f "$STUB/blah2" ] || exit 1
+        cat "$STUB/blah2"; exit 0 ;;
+esac
+exit 0
+'''
+
+
+class Harness(unittest.TestCase):
+
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        self.stub = os.path.join(self.dir, 'stub')
+        self.bin = os.path.join(self.dir, 'bin')
+        self.manifests = os.path.join(self.dir, 'current', 'manifests')
+        for d in (self.stub, self.bin, self.manifests):
+            os.makedirs(d)
+        docker = os.path.join(self.bin, 'docker')
+        self.write(docker, STUB_DOCKER)
+        os.chmod(docker, 0o755)
+        self.write(os.path.join(self.manifests, 'docker-compose.yaml'), 'services: {}\n')
+        self.mode_file = os.path.join(self.dir, 'mode.txt')
+
+    def tearDown(self):
+        shutil.rmtree(self.dir)
+
+    def write(self, path, text, mode='w'):
+        with open(path, mode) as f:
+            f.write(text)
+
+    def env(self):
+        return dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", STUB=self.stub,
+                    RETINA_ENV_GUARD_MANIFESTS=self.manifests,
+                    RETINA_STACK_MANIFESTS=self.manifests,
+                    RETINA_STACK_MODE_FILE=self.mode_file)
+
+    def run_script(self, script):
+        result = subprocess.run(['/bin/sh', script], env=self.env(), capture_output=True, text=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        return result.stdout
+
+    def set_env(self, content):
+        self.write(os.path.join(self.manifests, '.env'), content,
+                   mode='wb' if isinstance(content, bytes) else 'w')
+
+    def set_stub(self, name, value):
+        self.write(os.path.join(self.stub, name), str(value))
+
+
+class TestEnvGuard(Harness):
+
+    def test_a_nul_filled_env_is_set_aside(self):
+        # ret9573ecda: 294 NUL bytes since 2026-08-27, stack down on every boot.
+        self.set_env(b'\x00' * 294)
+
+        out = self.run_script(GUARD)
+
+        names = os.listdir(self.manifests)
+        self.assertNotIn('.env', names)
+        self.assertEqual(len([n for n in names if n.startswith('.env.corrupt-')]), 1)
+        self.assertIn('NUL bytes', out)
+
+    def test_an_env_compose_cannot_parse_is_set_aside(self):
+        env = os.path.join(self.manifests, '.env')
+        self.set_env('BAD LINE\n')
+        self.write(os.path.join(self.stub, 'compose_error'), f'failed to read {env}: line 1: unexpected character\n')
+
+        self.assertIn('compose cannot parse it', self.run_script(GUARD))
+        self.assertFalse(os.path.exists(env))
+
+    def test_a_good_env_is_left_alone(self):
+        self.set_env('RECEIVER_LAT=51.5\n')
+        self.assertEqual(self.run_script(GUARD), '')
+        self.assertTrue(os.path.exists(os.path.join(self.manifests, '.env')))
+
+    def test_an_unrelated_compose_error_leaves_env_alone(self):
+        self.set_env('RECEIVER_LAT=51.5\n')
+        self.write(os.path.join(self.stub, 'compose_error'), 'service "x" has neither an image nor a build\n')
+        self.run_script(GUARD)
+        self.assertTrue(os.path.exists(os.path.join(self.manifests, '.env')))
+
+    def test_no_env_and_no_composition_are_fine(self):
+        self.run_script(GUARD)
+        shutil.rmtree(os.path.join(self.dir, 'current'))
+        self.run_script(GUARD)
+
+
+class TestInventory(Harness):
+
+    def attrs(self):
+        out = self.run_script(INVENTORY)
+        return dict(line.split('=', 1) for line in out.split())
+
+    def healthy(self):
+        self.set_env('A=1\n')
+        self.set_stub('running', 8)
+        self.set_stub('restarting', 0)
+        self.set_stub('blah2', 'running')
+
+    def test_a_healthy_stack_is_up(self):
+        self.healthy()
+        self.assertEqual(self.attrs(), {
+            'retina_stack': 'up', 'retina_stack_running': '8', 'retina_stack_restarting': '0',
+            'retina_blah2': 'running', 'retina_env_ok': 'true', 'retina_compose_ok': 'true',
+            'retina_mode': 'radar'})
+
+    def test_nothing_running_is_down(self):
+        self.healthy()
+        self.set_stub('running', 0)
+        self.assertEqual(self.attrs()['retina_stack'], 'down')
+
+    def test_the_ret9573ecda_state_is_reported_degraded(self):
+        # Old containers kept alive by Docker's restart policy, blah2 aborting
+        # on a zeroed config.yml, and a NUL .env that compose cannot load.
+        self.set_env(b'\x00' * 294)
+        self.write(os.path.join(self.stub, 'compose_error'), 'failed to read .env\n')
+        self.set_stub('running', 4)
+        self.set_stub('restarting', 3)
+        self.set_stub('blah2', 'restarting')
+
+        attrs = self.attrs()
+
+        self.assertEqual(attrs['retina_stack'], 'degraded')
+        self.assertEqual(attrs['retina_env_ok'], 'false')
+        self.assertEqual(attrs['retina_compose_ok'], 'false')
+        self.assertEqual(attrs['retina_blah2'], 'restarting')
+
+    def test_blah2_stopped_in_spectrum_mode_is_not_degraded(self):
+        self.healthy()
+        self.set_stub('blah2', 'exited')
+        self.write(self.mode_file, 'spectrum')
+        attrs = self.attrs()
+        self.assertEqual(attrs['retina_stack'], 'up')
+        self.assertEqual(attrs['retina_mode'], 'spectrum')
+
+    def test_blah2_stopped_in_radar_mode_is_degraded(self):
+        self.healthy()
+        self.set_stub('blah2', 'exited')
+        self.assertEqual(self.attrs()['retina_stack'], 'degraded')
+
+    def test_a_missing_blah2_is_reported(self):
+        self.healthy()
+        os.remove(os.path.join(self.stub, 'blah2'))
+        self.assertEqual(self.attrs()['retina_blah2'], 'missing')
+
+    def test_no_composition_is_absent(self):
+        shutil.rmtree(os.path.join(self.dir, 'current'))
+        self.assertEqual(self.attrs(), {'retina_stack': 'absent'})
+
+    def test_every_line_is_key_equals_value(self):
+        # mender-updated rejects the whole script's output on a malformed line.
+        self.healthy()
+        for line in self.run_script(INVENTORY).splitlines():
+            key, _, value = line.partition('=')
+            self.assertRegex(key, r'^[a-z0-9_]+$')
+            self.assertTrue(value)
+
+
+if __name__ == '__main__':
+    unittest.main()