Add systemd/launchd supervision and uploader liveness monitoring (fixes #223)

crates/standx-cli/src/commands/maker/mod.rs

@@ -32,6 +32,7 @@ use feed::{market_snapshot, spawn_market_feed};
 #[cfg(test)]
 use model::is_order_rejection;
 use model::{is_maker_order, position_for_symbol, MakerExit, PendingPlace};
+pub use model::{FailSafeShutdown, FAIL_SAFE_EXIT_CODE};
 #[cfg(test)]
 use notify::webhook_body;
 use notify::{MakerNotifier, PositionChange, RiskNotice};

crates/standx-cli/src/commands/maker/model.rs

@@ -1,6 +1,37 @@
 use standx_sdk::error::Error as StandxError;
 use standx_sdk::models::{Order, OrderSide, Position};
 
+/// Process exit code emitted when the maker performs an *intentional*
+/// fail-safe shutdown: the order-response stream was lost, three maker
+/// cycles failed in a row, position reconciliation failed, or residual
+/// maker-owned orders could not be cancelled on the way out.
+///
+/// Supervisors must treat this as "stop, do NOT auto-restart, notify a
+/// human" (systemd `RestartPreventExitStatus=`). It is deliberately
+/// distinct from `0` (a clean Ctrl+C / SIGTERM stop: no restart, no alert),
+/// from `1` (a generic startup/config/validation error), and from a panic
+/// (`101`) or a fatal signal (e.g. SIGKILL -> `137`), so that an
+/// *unexpected* death remains restartable while a designed fail-safe exit
+/// does not trigger a restart loop.
+pub const FAIL_SAFE_EXIT_CODE: i32 = 75;
+
+/// Typed marker for an intentional maker fail-safe shutdown. Carrying the
+/// reason as a concrete error (rather than a bare `anyhow::anyhow!`) lets
+/// `main` downcast it and map it to [`FAIL_SAFE_EXIT_CODE`] while still
+/// printing the message through the normal error path.
+#[derive(Debug)]
+pub struct FailSafeShutdown {
+    pub message: String,
+}
+
+impl std::fmt::Display for FailSafeShutdown {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for FailSafeShutdown {}
+
 pub(super) enum MakerExit {
     CtrlC,
     OrderResponse(String),

crates/standx-cli/src/commands/maker/runtime.rs

@@ -1716,7 +1716,8 @@ pub(super) async fn run_maker(
                         if let (Some(equity), Some(available)) = (equity, available) {
                             let fired = alerts.evaluate_account(equity, available);
                             for alert in fired {
-                                notifier.alert(&alert, &symbol);
+                                let await_delivery = alert.firing;
+                                notifier.alert(&alert, &symbol, await_delivery).await;
                             }
                         }
                     }
@@ -2189,14 +2190,17 @@ pub(super) async fn run_maker(
         .await;
 
     if let Some(error) = cleanup_error {
-        return Err(anyhow::anyhow!(
-            "maker stopped but maker-owned order cleanup failed: {}",
-            error
-        ));
+        // A residual-order cleanup failure is an intentional fail-safe stop
+        // that needs a human, not an automatic restart.
+        return Err(anyhow::Error::new(FailSafeShutdown {
+            message: format!(
+                "maker stopped (fail-safe) but maker-owned order cleanup failed: {error}"
+            ),
+        }));
     }
 
     match exit.terminal_error() {
-        Some(message) => Err(anyhow::anyhow!(message)),
+        Some(message) => Err(anyhow::Error::new(FailSafeShutdown { message })),
         None => Ok(()),
     }
 }

crates/standx-cli/src/commands/mod.rs

@@ -24,7 +24,7 @@ pub use block::handle_block;
 pub use config::handle_config;
 pub use dashboard::handle_dashboard;
 pub use leverage::handle_leverage;
-pub use maker::{handle_maker, panic_webhook_body};
+pub use maker::{handle_maker, panic_webhook_body, FailSafeShutdown, FAIL_SAFE_EXIT_CODE};
 pub use margin::handle_margin;
 pub use market::handle_market;
 pub use order::handle_order;

crates/standx-cli/src/main.rs

@@ -1,6 +1,7 @@
 use clap::Parser;
 use standx_cli::cli::{AlertWebhookFormat, Cli, Commands, MakerCommands, OutputFormat};
 use standx_cli::commands;
+use standx_cli::commands::{FailSafeShutdown, FAIL_SAFE_EXIT_CODE};
 use standx_cli::telemetry::Telemetry;
 
 /// Print cool splash screen
@@ -116,11 +117,25 @@ async fn main() {
         Err(e) => {
             print_error(&e, output);
             telemetry.track_command_complete(command_name, false, Some(&e.to_string()));
-            std::process::exit(1);
+            std::process::exit(exit_code_for(e.as_ref()));
         }
     }
 }
 
+/// Map a command error to a process exit code.
+///
+/// An intentional maker fail-safe shutdown gets its own
+/// [`FAIL_SAFE_EXIT_CODE`] so supervisors can tell it apart from a generic
+/// failure (`1`) and from an unexpected crash, and refuse to auto-restart
+/// it. Everything else keeps the generic error code `1`.
+fn exit_code_for(error: &(dyn std::error::Error + 'static)) -> i32 {
+    if error.downcast_ref::<FailSafeShutdown>().is_some() {
+        FAIL_SAFE_EXIT_CODE
+    } else {
+        1
+    }
+}
+
 /// Stable, non-sensitive telemetry label for the top-level command.
 fn command_name(command: &Commands) -> &'static str {
     match command {
@@ -245,7 +260,16 @@ async fn execute_command(
             commands::handle_block(command, output).await?;
         }
         Commands::Maker { command } => {
-            commands::handle_maker(*command, output, verbose).await?;
+            // anyhow flattens the concrete error type when it is boxed into a
+            // `Box<dyn Error>`, so downcast the fail-safe marker here (where the
+            // original `anyhow::Error` is still intact) and re-box it concretely
+            // so `exit_code_for` can recognise it and pick the fail-safe code.
+            if let Err(err) = commands::handle_maker(*command, output, verbose).await {
+                return Err(match err.downcast::<FailSafeShutdown>() {
+                    Ok(fail_safe) => Box::new(fail_safe) as Box<dyn std::error::Error>,
+                    Err(other) => other.into(),
+                });
+            }
         }
     }
     Ok(())
@@ -324,3 +348,31 @@ fn print_error(error: &Box<dyn std::error::Error>, output: OutputFormat) {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fail_safe_error_maps_to_fail_safe_exit_code() {
+        // Mirror how `execute_command` hands a maker fail-safe error back to
+        // `main`: built as an anyhow error, downcast to the concrete marker,
+        // then re-boxed concretely so the type survives.
+        let err = anyhow::Error::new(FailSafeShutdown {
+            message: "maker stopped immediately (fail-safe): order-response stream unavailable"
+                .to_string(),
+        });
+        let boxed: Box<dyn std::error::Error> = match err.downcast::<FailSafeShutdown>() {
+            Ok(fail_safe) => Box::new(fail_safe),
+            Err(other) => other.into(),
+        };
+        assert_eq!(exit_code_for(boxed.as_ref()), FAIL_SAFE_EXIT_CODE);
+    }
+
+    #[test]
+    fn generic_error_maps_to_exit_code_one() {
+        let boxed: Box<dyn std::error::Error> =
+            anyhow::anyhow!("could not load maker config").into();
+        assert_eq!(exit_code_for(boxed.as_ref()), 1);
+    }
+}

deploy/README.md

@@ -0,0 +1,118 @@
+# StandX maker process supervision
+
+Supervision units for the maker bot and its OpenObserve log uploader, plus a
+crash/boot catch-up. The repo targets both Linux (prod) and macOS (dev), so
+there are `systemd/` and `launchd/` variants with matching semantics.
+
+Without these units the maker's foreground wrapper (`scripts/run_maker_observed.sh`)
+has no supervisor: an unexpected death is neither restarted nor alerted, and a
+uploader that dies mid-run only shows up as a stale dashboard. These units fix
+all three gaps (see issue #223).
+
+## Exit-code policy
+
+The maker distinguishes an **intentional fail-safe shutdown** from an
+**unexpected death** with a dedicated process exit code. Supervision keys off it:
+
+| Exit code | Meaning | Restart? | Notify? |
+|-----------|---------|----------|---------|
+| `0` | Clean stop (Ctrl+C / SIGTERM) | No | No |
+| `75` | Intentional maker fail-safe (order-response stream lost, 3 consecutive cycle errors, position-reconciliation failure, or residual-order cleanup failure) | **No** | **Yes** |
+| `1` | Startup / config / validation error | Yes (loop-limited) | Yes |
+| `101`, signal (`137` = SIGKILL, …) | Panic, OOM kill, unexpected death | **Yes** | **Yes** |
+
+`75` is defined in Rust as `FAIL_SAFE_EXIT_CODE`
+(`crates/standx-cli/src/commands/maker/model.rs`) and returned via the typed
+`FailSafeShutdown` error; `main` maps that error to the code. A fail-safe is a
+deliberate, safe stop that needs a human, so it must **not** auto-restart — but
+it still alerts. An unexpected death is restartable.
+
+- **systemd** enforces this directly: `RestartPreventExitStatus=75` +
+  `Restart=on-failure` + `OnFailure=standx-notify@…`.
+- **launchd** has no per-code exclusion, so `standx-maker-supervise.sh`
+  translates exit `75` to `0` (after notifying) and pairs with
+  `KeepAlive.SuccessfulExit=false` (restart only on non-zero).
+
+## What manages the uploader
+
+`run_maker_observed.sh` co-supervises the live uploader: it polls the uploader
+while the maker runs and relaunches it (with a stderr + optional webhook notice)
+if it dies mid-run. Because the units run the maker *through* the wrapper, a
+single maker unit covers both processes. Set `STANDX_SUPERVISOR_WEBHOOK` (and,
+optionally, `STANDX_SUPERVISE_INTERVAL`, default 5s) to receive those notices.
+
+## Crash/boot catch-up
+
+`scripts/openobserve_catchup.sh` re-uploads any un-ingested log tail left by a
+SIGKILL or power loss. It runs the ingester in incremental (non-follow) mode
+with `--run-id <run_id>` per `<run_id>.ndjson`, reusing the exact checkpoint key
+the live follow uploader used — so it resumes without re-sending events. It is
+idempotent and best-effort (always exits 0). It runs at boot via
+`standx-openobserve-catchup.service` (ordered before the maker) / the
+`com.standx.openobserve-catchup` launchd agent (`RunAtLoad`), and can also be
+run by hand.
+
+---
+
+## Linux (systemd)
+
+Assumes an install root of `/opt/standx`. If you install elsewhere, edit the
+absolute paths in the three unit files.
+
+```sh
+# 1. Config + secrets
+sudo install -d -m 700 /etc/standx
+sudo install -m 600 /opt/standx/deploy/systemd/maker.env.example /etc/standx/maker.env
+sudoedit /etc/standx/maker.env        # set symbol, config, OpenObserve creds, webhook
+
+# 2. Install units + helper script
+sudo cp /opt/standx/deploy/systemd/standx-maker.service \
+        /opt/standx/deploy/systemd/standx-openobserve-catchup.service \
+        /opt/standx/deploy/systemd/standx-notify@.service \
+        /etc/systemd/system/
+sudo chmod +x /opt/standx/deploy/systemd/standx-notify.sh \
+              /opt/standx/scripts/run_maker_observed.sh \
+              /opt/standx/scripts/openobserve_catchup.sh
+sudo systemctl daemon-reload
+
+# 3. Enable
+sudo systemctl enable --now standx-openobserve-catchup.service
+sudo systemctl enable --now standx-maker.service
+
+# Observe
+journalctl -u standx-maker.service -f
+```
+
+Stop cleanly with `sudo systemctl stop standx-maker.service` (SIGTERM → the
+maker runs its fail-safe cleanup and exits 0 → no restart).
+
+## macOS (launchd)
+
+Install as a per-user LaunchAgent. Edit the absolute paths in both plists first.
+
+```sh
+cp /opt/standx/deploy/launchd/com.standx.openobserve-catchup.plist \
+   /opt/standx/deploy/launchd/com.standx.maker.plist \
+   ~/Library/LaunchAgents/
+chmod +x /opt/standx/deploy/launchd/standx-maker-supervise.sh \
+         /opt/standx/scripts/run_maker_observed.sh \
+         /opt/standx/scripts/openobserve_catchup.sh
+
+launchctl load ~/Library/LaunchAgents/com.standx.openobserve-catchup.plist
+launchctl load ~/Library/LaunchAgents/com.standx.maker.plist
+
+# Logs
+tail -f /opt/standx/var/standx/launchd-maker.err.log
+# Stop / remove
+launchctl unload ~/Library/LaunchAgents/com.standx.maker.plist
+```
+
+Provide OpenObserve credentials and the optional `STANDX_SUPERVISOR_WEBHOOK`
+via `deploy/openobserve/.env` (pointed at by `STANDX_ENV_FILE`).
+
+## Live trading
+
+All units default to **paper**. Live trading requires BOTH `--live` in
+`STANDX_MAKER_ARGS` (systemd) / the plist `ProgramArguments`, AND
+`STANDX_ENABLE_LIVE_MAKER=1`. Read `docs/14-maker-live-gate.md` first. Setting
+the env var alone changes nothing.

deploy/launchd/com.standx.maker.plist

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!--
+  StandX maker (macOS dev). Install to ~/Library/LaunchAgents/ and edit the
+  absolute paths below to your checkout. The supervise wrapper enforces the
+  exit-code policy (see deploy/README.md): fail-safe (75) is translated to a
+  clean exit so KeepAlive.SuccessfulExit=false does NOT restart it, while an
+  unexpected non-zero exit is passed through so launchd DOES restart.
+
+  Paper mode by default: there is no --live flag here and STANDX_ENABLE_LIVE_MAKER
+  is not set. See docs/14-maker-live-gate.md before enabling live trading.
+-->
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>com.standx.maker</string>
+
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/bash</string>
+    <string>/opt/standx/deploy/launchd/standx-maker-supervise.sh</string>
+    <string>/opt/standx/bin/standx</string>
+    <string>--output</string>
+    <string>json</string>
+    <string>maker</string>
+    <string>run</string>
+    <string>BTC-USD</string>
+    <string>--maker-config</string>
+    <string>/opt/standx/examples/maker.toml</string>
+  </array>
+
+  <key>WorkingDirectory</key>
+  <string>/opt/standx</string>
+
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>OPENOBSERVE_AUTO_UPLOAD</key>
+    <string>1</string>
+    <key>STANDX_LOG_DIR</key>
+    <string>/opt/standx/var/standx</string>
+    <key>STANDX_ENV_FILE</key>
+    <string>/opt/standx/deploy/openobserve/.env</string>
+  </dict>
+
+  <key>RunAtLoad</key>
+  <true/>
+
+  <!-- Restart only on a non-zero (unexpected) exit; the wrapper maps the
+       intentional fail-safe to 0 so it is left stopped. -->
+  <key>KeepAlive</key>
+  <dict>
+    <key>SuccessfulExit</key>
+    <false/>
+  </dict>
+
+  <!-- Throttle restarts so an early crash cannot busy-loop. -->
+  <key>ThrottleInterval</key>
+  <integer>10</integer>
+
+  <key>StandardOutPath</key>
+  <string>/opt/standx/var/standx/launchd-maker.out.log</string>
+  <key>StandardErrorPath</key>
+  <string>/opt/standx/var/standx/launchd-maker.err.log</string>
+</dict>
+</plist>

deploy/launchd/com.standx.openobserve-catchup.plist

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!--
+  StandX OpenObserve crash/boot catch-up (macOS). RunAtLoad fires once when the
+  agent is loaded (e.g. at login/boot), re-uploading any un-ingested tail of
+  prior runs. No KeepAlive: this is a one-shot. Idempotent and best-effort.
+  Load it BEFORE com.standx.maker so a stale tail is flushed before the next run.
+-->
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>com.standx.openobserve-catchup</string>
+
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/bash</string>
+    <string>/opt/standx/scripts/openobserve_catchup.sh</string>
+  </array>
+
+  <key>WorkingDirectory</key>
+  <string>/opt/standx</string>
+
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>STANDX_LOG_DIR</key>
+    <string>/opt/standx/var/standx</string>
+    <key>OPENOBSERVE_ENV_FILE</key>
+    <string>/opt/standx/deploy/openobserve/.env</string>
+  </dict>
+
+  <key>RunAtLoad</key>
+  <true/>
+
+  <key>StandardOutPath</key>
+  <string>/opt/standx/var/standx/launchd-catchup.out.log</string>
+  <key>StandardErrorPath</key>
+  <string>/opt/standx/var/standx/launchd-catchup.err.log</string>
+</dict>
+</plist>