fix(device): recover wedged Shell restarts safely

build/dependencies.lock

@@ -209,7 +209,7 @@ manifest = "build/legacy-manifest.json"
 schema = 1
 component_count = 332
 file_count = 2131
-byte_count = 314688405
+byte_count = 314689731
 source_build_components = 228
 unresolved_components = 104
 inherited_baseline = "07505ea58c7bba698d6b9220ff43946a43cac76b"

build/legacy-manifest.json

@@ -488,8 +488,8 @@
       "kind": "runtime",
       "path": "static/build/.tmp_update/bin/bloom-dev-deploy",
       "file_count": 1,
-      "byte_count": 31308,
-      "tree_sha256": "c0bd8724f49740c8877d69fe6db8945d968997deb21afdc4fc1ea1ca1025532d",
+      "byte_count": 32634,
+      "tree_sha256": "14aa71a938a0545d2e747efc3e7eecea17973404034098b488c9fd613c86ac18",
       "source": "https://github.com/boburning/BloomOS",
       "source_revision": "release-commit",
       "license": "GPL-3.0-only",

static/build/.tmp_update/bin/bloom-dev-deploy

@@ -16,15 +16,24 @@ UPDATE_STATE=${BLOOM_UPDATE_STATE_BIN:-$SYSTEM_ROOT/bin/bloom-update-state}
 SESSION=${BLOOM_SESSION_BIN:-$SYSTEM_ROOT/bin/bloom-session}
 JQ=${BLOOM_JQ_BIN:-$SYSTEM_ROOT/bin/jq}
 PIDOF=${BLOOM_PIDOF_BIN:-pidof}
+KILL=${BLOOM_KILL_BIN:-kill}
+SLEEP=${BLOOM_SLEEP_BIN:-sleep}
 SHELL_GUARD=${BLOOM_SHELL_GUARD_BIN:-$SYSTEM_ROOT/bin/bloom-shell-guard}
 NETWORK_TRANSACTION=${BLOOM_NETWORK_TRANSACTION_BIN:-$SYSTEM_ROOT/bin/bloom-network-transaction}
 RESTART_REQUEST="$DEV_ROOT/restart-shell.json"
 MAX_FILES=64
 MAX_BYTES=33554432
+RESTART_TERM_GRACE_ATTEMPTS=${BLOOM_DEV_RESTART_TERM_GRACE_ATTEMPTS:-20}
 LOCKED=0
 
 fail() { printf 'bloom-dev-deploy: %s\n' "$*" >&2; exit 1; }
 
+case "$RESTART_TERM_GRACE_ATTEMPTS" in
+    ''|*[!0-9]*) fail "restart grace is invalid" ;;
+esac
+[ "$RESTART_TERM_GRACE_ATTEMPTS" -gt 0 ] && [ "$RESTART_TERM_GRACE_ATTEMPTS" -lt 80 ] ||
+    fail "restart grace is invalid"
+
 unlock() {
     if [ "$LOCKED" -eq 1 ]; then
         rmdir "$LOCK" 2>/dev/null || true
@@ -498,9 +507,9 @@ command_rollback() {
             [ -x "$SHELL_GUARD" ] || fail "bytes restored but Bloom Shell guard is unavailable"
             request_shell_restart "$rollback_id" "$rollback_component"
             if [ -n "$old_keymon_pid" ]; then
-                kill -TERM "$old_keymon_pid" 2>/dev/null || fail "bytes restored but keymon restart failed"
+                "$KILL" -TERM "$old_keymon_pid" 2>/dev/null || fail "bytes restored but keymon restart failed"
             fi
-            kill -TERM "$old_pid" 2>/dev/null || fail "bytes restored but Bloom Shell restart failed"
+            "$KILL" -TERM "$old_pid" 2>/dev/null || fail "bytes restored but Bloom Shell restart failed"
             attempt=0
             while [ "$attempt" -lt 80 ]; do
                 current_pid="$(single_shell_pid 2>/dev/null || true)"
@@ -516,7 +525,9 @@ command_rollback() {
                     restarted=true
                     break
                 fi
-                sleep 0.25
+                escalate_stuck_restart "$attempt" "$old_pid" "$current_pid" \
+                    "$old_keymon_pid" "$current_keymon_pid"
+                "$SLEEP" 0.25
                 attempt=$((attempt + 1))
             done
             [ "$restarted" = true ] || fail "bytes restored but runtime restart timed out"
@@ -549,6 +560,20 @@ single_keymon_pid() {
     printf '%s\n' "$1"
 }
 
+escalate_stuck_restart() {
+    local attempt_number=$1 old_shell_pid=$2 current_shell_pid=$3
+    local old_keymon_process=$4 current_keymon_process=$5
+    [ "$attempt_number" -eq "$RESTART_TERM_GRACE_ATTEMPTS" ] || return 0
+    if [ -n "$current_shell_pid" ] && [ "$current_shell_pid" = "$old_shell_pid" ]; then
+        "$KILL" -KILL "$old_shell_pid" 2>/dev/null ||
+            fail "cannot stop wedged Bloom Shell after bounded TERM grace"
+    fi
+    if [ -n "$old_keymon_process" ] && [ "$current_keymon_process" = "$old_keymon_process" ]; then
+        "$KILL" -KILL "$old_keymon_process" 2>/dev/null ||
+            fail "cannot stop wedged keymon after bounded TERM grace"
+    fi
+}
+
 command_restart() {
     id=$1
     safe_id "$id" || fail "deployment id is unsafe"
@@ -576,12 +601,12 @@ command_restart() {
     [ -x "$SHELL_GUARD" ] || fail "Bloom Shell guard is unavailable"
     request_shell_restart "$id" "$component"
     if [ -n "$old_keymon_pid" ]; then
-        kill -TERM "$old_keymon_pid" 2>/dev/null || {
+        "$KILL" -TERM "$old_keymon_pid" 2>/dev/null || {
             rm -f "$RESTART_REQUEST"
             fail "cannot request keymon restart"
         }
     fi
-    kill -TERM "$old_pid" 2>/dev/null || { rm -f "$RESTART_REQUEST"; fail "cannot request Bloom Shell restart"; }
+    "$KILL" -TERM "$old_pid" 2>/dev/null || { rm -f "$RESTART_REQUEST"; fail "cannot request Bloom Shell restart"; }
     attempt=0
     while [ "$attempt" -lt 80 ]; do
         current_pid="$(single_shell_pid 2>/dev/null || true)"
@@ -597,7 +622,9 @@ command_restart() {
             printf '{"schema":1,"operation":"restart","deployment_id":"%s","component":"%s","restarted":true,"ready":true}\n' "$id" "$component"
             return 0
         fi
-        sleep 0.25
+        escalate_stuck_restart "$attempt" "$old_pid" "$current_pid" \
+            "$old_keymon_pid" "$current_keymon_pid"
+        "$SLEEP" 0.25
         attempt=$((attempt + 1))
     done
 
@@ -617,11 +644,11 @@ command_restart() {
     current_pid="$(single_shell_pid 2>/dev/null || true)"
     current_keymon_pid="$(single_keymon_pid 2>/dev/null || true)"
     if [ "$component" = launch-session ] && [ -n "$current_keymon_pid" ]; then
-        kill -TERM "$current_keymon_pid" 2>/dev/null || true
+        "$KILL" -TERM "$current_keymon_pid" 2>/dev/null || true
     fi
     if [ -n "$current_pid" ]; then
         request_shell_restart "$rollback_id" "$rollback_component"
-        kill -TERM "$current_pid" 2>/dev/null || true
+        "$KILL" -TERM "$current_pid" 2>/dev/null || true
     fi
     fail "restart readiness timed out and the deployment was rolled back"
 }

test/shell/dev_deploy.bats

@@ -447,6 +447,54 @@ EOF
     [[ "$output" == *'"ready":true'* ]]
 }
 
+@test "wedged Bloom Shell restart escalates only the exact retained PID after bounded TERM grace" {
+    id=wedged-shell
+    slot="$DEV_ROOT/deployments/$id"
+    mkdir -p "$slot"
+    make_shell_manifest "$id" "$slot/manifest.json"
+    : >"$slot/sealed"
+    printf '%s\n' '{"schema":1,"deployment_id":"wedged-shell","component":"bloom-shell"}' \
+        >"$DEV_ROOT/active.json"
+
+    export BLOOM_PIDOF_BIN="$BATS_TEST_TMPDIR/pidof"
+    export BLOOM_KILL_BIN="$BATS_TEST_TMPDIR/kill"
+    export BLOOM_SLEEP_BIN="$BATS_TEST_TMPDIR/sleep"
+    export BLOOM_SHELL_GUARD_BIN="$BATS_TEST_TMPDIR/shell-guard"
+    export BLOOM_DEV_RESTART_TERM_GRACE_ATTEMPTS=2
+    export BLOOM_TEST_PID_STATE="$BATS_TEST_TMPDIR/shell-pid"
+    export BLOOM_TEST_KILL_LOG="$BATS_TEST_TMPDIR/kill.log"
+    printf '%s\n' 111 >"$BLOOM_TEST_PID_STATE"
+
+    cat >"$BLOOM_PIDOF_BIN" <<'EOF'
+#!/bin/sh
+[ "$1" = bloom-shell ] || exit 1
+cat "$BLOOM_TEST_PID_STATE"
+EOF
+    cat >"$BLOOM_KILL_BIN" <<'EOF'
+#!/bin/sh
+printf '%s %s\n' "$1" "$2" >>"$BLOOM_TEST_KILL_LOG"
+if [ "$1:$2" = -KILL:111 ]; then printf '%s\n' 222 >"$BLOOM_TEST_PID_STATE"; fi
+EOF
+    cat >"$BLOOM_SLEEP_BIN" <<'EOF'
+#!/bin/sh
+exit 0
+EOF
+    cat >"$BLOOM_SHELL_GUARD_BIN" <<'EOF'
+#!/bin/sh
+printf '%s\n' '{"schema":1,"phase":"ready"}'
+EOF
+    chmod +x "$BLOOM_PIDOF_BIN" "$BLOOM_KILL_BIN" "$BLOOM_SLEEP_BIN" \
+        "$BLOOM_SHELL_GUARD_BIN"
+
+    run "$SERVICE" restart "$id"
+
+    [ "$status" -eq 0 ]
+    [[ "$output" == *'"restarted":true'* ]]
+    [ "$(sed -n '1p' "$BLOOM_TEST_KILL_LOG")" = '-TERM 111' ]
+    [ "$(sed -n '2p' "$BLOOM_TEST_KILL_LOG")" = '-KILL 111' ]
+    [ "$(wc -l <"$BLOOM_TEST_KILL_LOG" | tr -d ' ')" = 2 ]
+}
+
 @test "seed reuses an exact live file and missing reports only changed bytes" {
     live="$SYSTEM_ROOT/bin/bloom-listener-audit"
     changed="$BATS_TEST_TMPDIR/changed-seed"