From 08dccfdcbb57dd62f4739496fb3709fa7643dffb Mon Sep 17 00:00:00 2001 From: BossX Date: Mon, 13 Jul 2026 15:32:13 +0800 Subject: [PATCH] Add process supervision and uploader liveness monitoring (fixes #223) systemd + launchd units for the maker and uploader with Restart=on-failure, an OnFailure notify path, and a distinct fail-safe exit code (75) so an intentional safe shutdown is never auto-restarted (only notified). Wrapper now supervises and relaunches the uploader; a boot/crash catch-up re-uploads any un-ingested log tail. Rebased onto main after #218-#221 merged. Merged the commands re-export with #220's panic_webhook_body. Also repairs the same latent main build break that #231 fixes: #219's equity/margin `notifier.alert(&alert, &symbol)` call did not update to the 3-arg `await_delivery` form #221 introduced; corrected here too so this branch compiles standalone. Co-Authored-By: Claude Opus 4.8 --- crates/standx-cli/src/commands/maker/mod.rs | 1 + crates/standx-cli/src/commands/maker/model.rs | 31 +++++ .../standx-cli/src/commands/maker/runtime.rs | 16 ++- crates/standx-cli/src/commands/mod.rs | 2 +- crates/standx-cli/src/main.rs | 56 ++++++++- deploy/README.md | 118 ++++++++++++++++++ deploy/launchd/com.standx.maker.plist | 65 ++++++++++ .../com.standx.openobserve-catchup.plist | 39 ++++++ deploy/launchd/standx-maker-supervise.sh | 55 ++++++++ deploy/systemd/maker.env.example | 29 +++++ deploy/systemd/standx-maker.service | 43 +++++++ deploy/systemd/standx-notify.sh | 25 ++++ deploy/systemd/standx-notify@.service | 9 ++ .../standx-openobserve-catchup.service | 17 +++ scripts/openobserve_catchup.sh | 71 +++++++++++ scripts/run_maker_observed.sh | 74 ++++++++--- 16 files changed, 628 insertions(+), 23 deletions(-) create mode 100644 deploy/README.md create mode 100644 deploy/launchd/com.standx.maker.plist create mode 100644 deploy/launchd/com.standx.openobserve-catchup.plist create mode 100755 deploy/launchd/standx-maker-supervise.sh create mode 100644 deploy/systemd/maker.env.example create mode 100644 deploy/systemd/standx-maker.service create mode 100755 deploy/systemd/standx-notify.sh create mode 100644 deploy/systemd/standx-notify@.service create mode 100644 deploy/systemd/standx-openobserve-catchup.service create mode 100755 scripts/openobserve_catchup.sh diff --git a/crates/standx-cli/src/commands/maker/mod.rs b/crates/standx-cli/src/commands/maker/mod.rs index db993c3..00150ba 100644 --- a/crates/standx-cli/src/commands/maker/mod.rs +++ b/crates/standx-cli/src/commands/maker/mod.rs @@ -32,6 +32,7 @@ use feed::{market_snapshot, spawn_market_feed}; #[cfg(test)] use model::is_order_rejection; use model::{is_maker_order, position_for_symbol, MakerExit, PendingPlace}; +pub use model::{FailSafeShutdown, FAIL_SAFE_EXIT_CODE}; #[cfg(test)] use notify::webhook_body; use notify::{MakerNotifier, PositionChange, RiskNotice}; diff --git a/crates/standx-cli/src/commands/maker/model.rs b/crates/standx-cli/src/commands/maker/model.rs index d07e81d..0dd66f4 100644 --- a/crates/standx-cli/src/commands/maker/model.rs +++ b/crates/standx-cli/src/commands/maker/model.rs @@ -1,6 +1,37 @@ use standx_sdk::error::Error as StandxError; use standx_sdk::models::{Order, OrderSide, Position}; +/// Process exit code emitted when the maker performs an *intentional* +/// fail-safe shutdown: the order-response stream was lost, three maker +/// cycles failed in a row, position reconciliation failed, or residual +/// maker-owned orders could not be cancelled on the way out. +/// +/// Supervisors must treat this as "stop, do NOT auto-restart, notify a +/// human" (systemd `RestartPreventExitStatus=`). It is deliberately +/// distinct from `0` (a clean Ctrl+C / SIGTERM stop: no restart, no alert), +/// from `1` (a generic startup/config/validation error), and from a panic +/// (`101`) or a fatal signal (e.g. SIGKILL -> `137`), so that an +/// *unexpected* death remains restartable while a designed fail-safe exit +/// does not trigger a restart loop. +pub const FAIL_SAFE_EXIT_CODE: i32 = 75; + +/// Typed marker for an intentional maker fail-safe shutdown. Carrying the +/// reason as a concrete error (rather than a bare `anyhow::anyhow!`) lets +/// `main` downcast it and map it to [`FAIL_SAFE_EXIT_CODE`] while still +/// printing the message through the normal error path. +#[derive(Debug)] +pub struct FailSafeShutdown { + pub message: String, +} + +impl std::fmt::Display for FailSafeShutdown { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for FailSafeShutdown {} + pub(super) enum MakerExit { CtrlC, OrderResponse(String), diff --git a/crates/standx-cli/src/commands/maker/runtime.rs b/crates/standx-cli/src/commands/maker/runtime.rs index efe389c..7d34b83 100644 --- a/crates/standx-cli/src/commands/maker/runtime.rs +++ b/crates/standx-cli/src/commands/maker/runtime.rs @@ -1716,7 +1716,8 @@ pub(super) async fn run_maker( if let (Some(equity), Some(available)) = (equity, available) { let fired = alerts.evaluate_account(equity, available); for alert in fired { - notifier.alert(&alert, &symbol); + let await_delivery = alert.firing; + notifier.alert(&alert, &symbol, await_delivery).await; } } } @@ -2189,14 +2190,17 @@ pub(super) async fn run_maker( .await; if let Some(error) = cleanup_error { - return Err(anyhow::anyhow!( - "maker stopped but maker-owned order cleanup failed: {}", - error - )); + // A residual-order cleanup failure is an intentional fail-safe stop + // that needs a human, not an automatic restart. + return Err(anyhow::Error::new(FailSafeShutdown { + message: format!( + "maker stopped (fail-safe) but maker-owned order cleanup failed: {error}" + ), + })); } match exit.terminal_error() { - Some(message) => Err(anyhow::anyhow!(message)), + Some(message) => Err(anyhow::Error::new(FailSafeShutdown { message })), None => Ok(()), } } diff --git a/crates/standx-cli/src/commands/mod.rs b/crates/standx-cli/src/commands/mod.rs index 0d00939..545a344 100644 --- a/crates/standx-cli/src/commands/mod.rs +++ b/crates/standx-cli/src/commands/mod.rs @@ -24,7 +24,7 @@ pub use block::handle_block; pub use config::handle_config; pub use dashboard::handle_dashboard; pub use leverage::handle_leverage; -pub use maker::{handle_maker, panic_webhook_body}; +pub use maker::{handle_maker, panic_webhook_body, FailSafeShutdown, FAIL_SAFE_EXIT_CODE}; pub use margin::handle_margin; pub use market::handle_market; pub use order::handle_order; diff --git a/crates/standx-cli/src/main.rs b/crates/standx-cli/src/main.rs index 1a895c6..529428a 100644 --- a/crates/standx-cli/src/main.rs +++ b/crates/standx-cli/src/main.rs @@ -1,6 +1,7 @@ use clap::Parser; use standx_cli::cli::{AlertWebhookFormat, Cli, Commands, MakerCommands, OutputFormat}; use standx_cli::commands; +use standx_cli::commands::{FailSafeShutdown, FAIL_SAFE_EXIT_CODE}; use standx_cli::telemetry::Telemetry; /// Print cool splash screen @@ -116,11 +117,25 @@ async fn main() { Err(e) => { print_error(&e, output); telemetry.track_command_complete(command_name, false, Some(&e.to_string())); - std::process::exit(1); + std::process::exit(exit_code_for(e.as_ref())); } } } +/// Map a command error to a process exit code. +/// +/// An intentional maker fail-safe shutdown gets its own +/// [`FAIL_SAFE_EXIT_CODE`] so supervisors can tell it apart from a generic +/// failure (`1`) and from an unexpected crash, and refuse to auto-restart +/// it. Everything else keeps the generic error code `1`. +fn exit_code_for(error: &(dyn std::error::Error + 'static)) -> i32 { + if error.downcast_ref::().is_some() { + FAIL_SAFE_EXIT_CODE + } else { + 1 + } +} + /// Stable, non-sensitive telemetry label for the top-level command. fn command_name(command: &Commands) -> &'static str { match command { @@ -245,7 +260,16 @@ async fn execute_command( commands::handle_block(command, output).await?; } Commands::Maker { command } => { - commands::handle_maker(*command, output, verbose).await?; + // anyhow flattens the concrete error type when it is boxed into a + // `Box`, so downcast the fail-safe marker here (where the + // original `anyhow::Error` is still intact) and re-box it concretely + // so `exit_code_for` can recognise it and pick the fail-safe code. + if let Err(err) = commands::handle_maker(*command, output, verbose).await { + return Err(match err.downcast::() { + Ok(fail_safe) => Box::new(fail_safe) as Box, + Err(other) => other.into(), + }); + } } } Ok(()) @@ -324,3 +348,31 @@ fn print_error(error: &Box, output: OutputFormat) { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fail_safe_error_maps_to_fail_safe_exit_code() { + // Mirror how `execute_command` hands a maker fail-safe error back to + // `main`: built as an anyhow error, downcast to the concrete marker, + // then re-boxed concretely so the type survives. + let err = anyhow::Error::new(FailSafeShutdown { + message: "maker stopped immediately (fail-safe): order-response stream unavailable" + .to_string(), + }); + let boxed: Box = match err.downcast::() { + Ok(fail_safe) => Box::new(fail_safe), + Err(other) => other.into(), + }; + assert_eq!(exit_code_for(boxed.as_ref()), FAIL_SAFE_EXIT_CODE); + } + + #[test] + fn generic_error_maps_to_exit_code_one() { + let boxed: Box = + anyhow::anyhow!("could not load maker config").into(); + assert_eq!(exit_code_for(boxed.as_ref()), 1); + } +} diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..4f92ac1 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,118 @@ +# StandX maker process supervision + +Supervision units for the maker bot and its OpenObserve log uploader, plus a +crash/boot catch-up. The repo targets both Linux (prod) and macOS (dev), so +there are `systemd/` and `launchd/` variants with matching semantics. + +Without these units the maker's foreground wrapper (`scripts/run_maker_observed.sh`) +has no supervisor: an unexpected death is neither restarted nor alerted, and a +uploader that dies mid-run only shows up as a stale dashboard. These units fix +all three gaps (see issue #223). + +## Exit-code policy + +The maker distinguishes an **intentional fail-safe shutdown** from an +**unexpected death** with a dedicated process exit code. Supervision keys off it: + +| Exit code | Meaning | Restart? | Notify? | +|-----------|---------|----------|---------| +| `0` | Clean stop (Ctrl+C / SIGTERM) | No | No | +| `75` | Intentional maker fail-safe (order-response stream lost, 3 consecutive cycle errors, position-reconciliation failure, or residual-order cleanup failure) | **No** | **Yes** | +| `1` | Startup / config / validation error | Yes (loop-limited) | Yes | +| `101`, signal (`137` = SIGKILL, …) | Panic, OOM kill, unexpected death | **Yes** | **Yes** | + +`75` is defined in Rust as `FAIL_SAFE_EXIT_CODE` +(`crates/standx-cli/src/commands/maker/model.rs`) and returned via the typed +`FailSafeShutdown` error; `main` maps that error to the code. A fail-safe is a +deliberate, safe stop that needs a human, so it must **not** auto-restart — but +it still alerts. An unexpected death is restartable. + +- **systemd** enforces this directly: `RestartPreventExitStatus=75` + + `Restart=on-failure` + `OnFailure=standx-notify@…`. +- **launchd** has no per-code exclusion, so `standx-maker-supervise.sh` + translates exit `75` to `0` (after notifying) and pairs with + `KeepAlive.SuccessfulExit=false` (restart only on non-zero). + +## What manages the uploader + +`run_maker_observed.sh` co-supervises the live uploader: it polls the uploader +while the maker runs and relaunches it (with a stderr + optional webhook notice) +if it dies mid-run. Because the units run the maker *through* the wrapper, a +single maker unit covers both processes. Set `STANDX_SUPERVISOR_WEBHOOK` (and, +optionally, `STANDX_SUPERVISE_INTERVAL`, default 5s) to receive those notices. + +## Crash/boot catch-up + +`scripts/openobserve_catchup.sh` re-uploads any un-ingested log tail left by a +SIGKILL or power loss. It runs the ingester in incremental (non-follow) mode +with `--run-id ` per `.ndjson`, reusing the exact checkpoint key +the live follow uploader used — so it resumes without re-sending events. It is +idempotent and best-effort (always exits 0). It runs at boot via +`standx-openobserve-catchup.service` (ordered before the maker) / the +`com.standx.openobserve-catchup` launchd agent (`RunAtLoad`), and can also be +run by hand. + +--- + +## Linux (systemd) + +Assumes an install root of `/opt/standx`. If you install elsewhere, edit the +absolute paths in the three unit files. + +```sh +# 1. Config + secrets +sudo install -d -m 700 /etc/standx +sudo install -m 600 /opt/standx/deploy/systemd/maker.env.example /etc/standx/maker.env +sudoedit /etc/standx/maker.env # set symbol, config, OpenObserve creds, webhook + +# 2. Install units + helper script +sudo cp /opt/standx/deploy/systemd/standx-maker.service \ + /opt/standx/deploy/systemd/standx-openobserve-catchup.service \ + /opt/standx/deploy/systemd/standx-notify@.service \ + /etc/systemd/system/ +sudo chmod +x /opt/standx/deploy/systemd/standx-notify.sh \ + /opt/standx/scripts/run_maker_observed.sh \ + /opt/standx/scripts/openobserve_catchup.sh +sudo systemctl daemon-reload + +# 3. Enable +sudo systemctl enable --now standx-openobserve-catchup.service +sudo systemctl enable --now standx-maker.service + +# Observe +journalctl -u standx-maker.service -f +``` + +Stop cleanly with `sudo systemctl stop standx-maker.service` (SIGTERM → the +maker runs its fail-safe cleanup and exits 0 → no restart). + +## macOS (launchd) + +Install as a per-user LaunchAgent. Edit the absolute paths in both plists first. + +```sh +cp /opt/standx/deploy/launchd/com.standx.openobserve-catchup.plist \ + /opt/standx/deploy/launchd/com.standx.maker.plist \ + ~/Library/LaunchAgents/ +chmod +x /opt/standx/deploy/launchd/standx-maker-supervise.sh \ + /opt/standx/scripts/run_maker_observed.sh \ + /opt/standx/scripts/openobserve_catchup.sh + +launchctl load ~/Library/LaunchAgents/com.standx.openobserve-catchup.plist +launchctl load ~/Library/LaunchAgents/com.standx.maker.plist + +# Logs +tail -f /opt/standx/var/standx/launchd-maker.err.log +# Stop / remove +launchctl unload ~/Library/LaunchAgents/com.standx.maker.plist +``` + +Provide OpenObserve credentials and the optional `STANDX_SUPERVISOR_WEBHOOK` +via `deploy/openobserve/.env` (pointed at by `STANDX_ENV_FILE`). + +## Live trading + +All units default to **paper**. Live trading requires BOTH `--live` in +`STANDX_MAKER_ARGS` (systemd) / the plist `ProgramArguments`, AND +`STANDX_ENABLE_LIVE_MAKER=1`. Read `docs/14-maker-live-gate.md` first. Setting +the env var alone changes nothing. diff --git a/deploy/launchd/com.standx.maker.plist b/deploy/launchd/com.standx.maker.plist new file mode 100644 index 0000000..a1d0ff7 --- /dev/null +++ b/deploy/launchd/com.standx.maker.plist @@ -0,0 +1,65 @@ + + + + + + Label + com.standx.maker + + ProgramArguments + + /bin/bash + /opt/standx/deploy/launchd/standx-maker-supervise.sh + /opt/standx/bin/standx + --output + json + maker + run + BTC-USD + --maker-config + /opt/standx/examples/maker.toml + + + WorkingDirectory + /opt/standx + + EnvironmentVariables + + OPENOBSERVE_AUTO_UPLOAD + 1 + STANDX_LOG_DIR + /opt/standx/var/standx + STANDX_ENV_FILE + /opt/standx/deploy/openobserve/.env + + + RunAtLoad + + + + KeepAlive + + SuccessfulExit + + + + + ThrottleInterval + 10 + + StandardOutPath + /opt/standx/var/standx/launchd-maker.out.log + StandardErrorPath + /opt/standx/var/standx/launchd-maker.err.log + + diff --git a/deploy/launchd/com.standx.openobserve-catchup.plist b/deploy/launchd/com.standx.openobserve-catchup.plist new file mode 100644 index 0000000..362ae52 --- /dev/null +++ b/deploy/launchd/com.standx.openobserve-catchup.plist @@ -0,0 +1,39 @@ + + + + + + Label + com.standx.openobserve-catchup + + ProgramArguments + + /bin/bash + /opt/standx/scripts/openobserve_catchup.sh + + + WorkingDirectory + /opt/standx + + EnvironmentVariables + + STANDX_LOG_DIR + /opt/standx/var/standx + OPENOBSERVE_ENV_FILE + /opt/standx/deploy/openobserve/.env + + + RunAtLoad + + + StandardOutPath + /opt/standx/var/standx/launchd-catchup.out.log + StandardErrorPath + /opt/standx/var/standx/launchd-catchup.err.log + + diff --git a/deploy/launchd/standx-maker-supervise.sh b/deploy/launchd/standx-maker-supervise.sh new file mode 100755 index 0000000..3992da1 --- /dev/null +++ b/deploy/launchd/standx-maker-supervise.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# launchd exit-code translator for the StandX maker. +# +# launchd's KeepAlive cannot exclude a single exit code from restart the way +# systemd's RestartPreventExitStatus= can. So we run the observed wrapper and +# translate its exit code to match the intended policy, paired with +# `KeepAlive.SuccessfulExit=false` in the plist (restart only on non-zero): +# +# exit 75 (intentional fail-safe) -> notify, then exit 0 => launchd does NOT restart +# exit !=0 (panic / kill / config) -> notify, then pass through => launchd DOES restart +# exit 0 (clean stop) -> exit 0 => launchd does NOT restart +# +# Usage (from the plist): standx-maker-supervise.sh --output json maker run ... +set -uo pipefail + +FAIL_SAFE_EXIT_CODE=75 +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../.." && pwd)" + +# Optional env file (OpenObserve creds, STANDX_SUPERVISOR_WEBHOOK, etc.). +env_file="${STANDX_ENV_FILE:-$repo_root/deploy/openobserve/.env}" +if [[ -f "$env_file" ]]; then + set -a + # shellcheck disable=SC1090 + . "$env_file" + set +a +fi + +notify() { + local message="$1" + printf '%s\n' "$message" >&2 + if [[ -n "${STANDX_SUPERVISOR_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then + local escaped="${message//\\/\\\\}" + escaped="${escaped//\"/\\\"}" + curl -fsS -m 5 -X POST \ + -H 'Content-Type: application/json' \ + --data "{\"text\":\"$escaped\"}" \ + "$STANDX_SUPERVISOR_WEBHOOK" >/dev/null 2>&1 || + printf 'supervisor webhook post failed (message logged above)\n' >&2 + fi +} + +host="$(hostname 2>/dev/null || echo unknown-host)" +"$repo_root/scripts/run_maker_observed.sh" "$@" +status=$? + +if ((status == FAIL_SAFE_EXIT_CODE)); then + notify "StandX maker fail-safe shutdown (exit ${FAIL_SAFE_EXIT_CODE}) on ${host}; NOT restarting (intentional). Check the maker logs before relaunching." + exit 0 +elif ((status != 0)); then + notify "StandX maker unexpected exit (status ${status}) on ${host}; launchd will restart it." + exit "$status" +fi + +exit 0 diff --git a/deploy/systemd/maker.env.example b/deploy/systemd/maker.env.example new file mode 100644 index 0000000..bfc44d9 --- /dev/null +++ b/deploy/systemd/maker.env.example @@ -0,0 +1,29 @@ +# StandX maker supervision environment. +# Copy to /etc/standx/maker.env (root-owned, chmod 600) and edit. +# The systemd units assume an install root of /opt/standx; if you install +# elsewhere, edit the paths in the .service files too. + +# Maker tunables (consumed by standx-maker.service ExecStart). +STANDX_SYMBOL=BTC-USD +STANDX_MAKER_CONFIG=/opt/standx/examples/maker.toml +# Extra maker flags, word-split into arguments. Leave empty for paper defaults. +# DO NOT add --live here unless you have read docs/14-maker-live-gate.md. +STANDX_MAKER_ARGS= + +# Persistent log directory so the crash/boot catch-up can find prior runs. +STANDX_LOG_DIR=/opt/standx/var/standx + +# OpenObserve live upload (mirror deploy/openobserve/.env.example). +OPENOBSERVE_AUTO_UPLOAD=1 +OPENOBSERVE_URL=http://127.0.0.1:5080 +OPENOBSERVE_ORG=default +OPENOBSERVE_STREAM=standx_maker +OPENOBSERVE_USER=standx@example.com +OPENOBSERVE_PASSWORD='ReplaceMe-123!' + +# Optional webhook for uploader-death notices (wrapper) and OnFailure alerts. +#STANDX_SUPERVISOR_WEBHOOK=https://hooks.example.com/services/XXXX + +# LIVE TRADING GATE — leave unset for paper. Setting this alone does NOTHING +# unless --live is also passed in STANDX_MAKER_ARGS. See docs/14-maker-live-gate.md. +#STANDX_ENABLE_LIVE_MAKER=1 diff --git a/deploy/systemd/standx-maker.service b/deploy/systemd/standx-maker.service new file mode 100644 index 0000000..15dd54d --- /dev/null +++ b/deploy/systemd/standx-maker.service @@ -0,0 +1,43 @@ +[Unit] +Description=StandX maker bot (observed, with live OpenObserve upload) +Documentation=file:/opt/standx/deploy/README.md +After=network-online.target +Wants=network-online.target +# Re-ingest any un-uploaded log tail from a previous crash before starting. +After=standx-openobserve-catchup.service +Wants=standx-openobserve-catchup.service +# Notify a human on ANY failure, including the no-restart fail-safe (exit 75). +OnFailure=standx-notify@%n.service +# Break crash loops (e.g. a persistent config error that keeps exiting 1): +# give up after 5 failures in 5 minutes and enter 'failed' (still fires +# OnFailure) instead of restarting forever. +StartLimitIntervalSec=300 +StartLimitBurst=5 + +[Service] +Type=simple +# Operator-provided config: install paths, symbol, maker flags, OpenObserve +# credentials, the optional STANDX_SUPERVISOR_WEBHOOK, and (only if live +# trading is truly intended) STANDX_ENABLE_LIVE_MAKER. See maker.env.example. +EnvironmentFile=/etc/standx/maker.env +WorkingDirectory=/opt/standx +# The wrapper enforces `maker run --output json`, tees logs, and co-supervises +# the live uploader (relaunching it if it dies mid-run). $STANDX_MAKER_ARGS is +# intentionally unquoted so extra flags are word-split into arguments. +ExecStart=/opt/standx/scripts/run_maker_observed.sh /opt/standx/bin/standx --output json maker run ${STANDX_SYMBOL} --maker-config ${STANDX_MAKER_CONFIG} $STANDX_MAKER_ARGS + +# --- Exit-code policy (see deploy/README.md) --- +# 0 clean stop (Ctrl+C / SIGTERM) -> no restart, no alert +# 75 intentional maker fail-safe shutdown -> notify, NO restart +# 1/101/signal startup error, panic, or unexpected kill -> notify + restart +SuccessExitStatus=0 +RestartPreventExitStatus=75 +Restart=on-failure +RestartSec=5 +# Forward SIGTERM to the wrapper so the maker runs its fail-safe cleanup, and +# give it time to cancel maker-owned orders before SIGKILL. +KillSignal=SIGTERM +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/standx-notify.sh b/deploy/systemd/standx-notify.sh new file mode 100755 index 0000000..15922e3 --- /dev/null +++ b/deploy/systemd/standx-notify.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# OnFailure notifier for StandX supervision units. +# +# Argument $1 is the failed unit name (passed as %i from the +# `OnFailure=standx-notify@%n.service` hook). Always logs to stderr (the +# journal); additionally POSTs to STANDX_SUPERVISOR_WEBHOOK when it is set and +# curl is available. Best-effort: a failed webhook never changes the outcome. +set -uo pipefail + +unit="${1:-unknown.unit}" +host="$(hostname 2>/dev/null || echo unknown-host)" +now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +msg="StandX supervision alert: unit ${unit} entered a failure state on ${host} at ${now}. Inspect with: journalctl -u ${unit} -n 100 --no-pager" + +printf '%s\n' "$msg" >&2 + +if [[ -n "${STANDX_SUPERVISOR_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then + escaped="${msg//\\/\\\\}" + escaped="${escaped//\"/\\\"}" + curl -fsS -m 5 -X POST \ + -H 'Content-Type: application/json' \ + --data "{\"text\":\"$escaped\"}" \ + "$STANDX_SUPERVISOR_WEBHOOK" >/dev/null 2>&1 || + printf 'notify webhook post failed (message logged above)\n' >&2 +fi diff --git a/deploy/systemd/standx-notify@.service b/deploy/systemd/standx-notify@.service new file mode 100644 index 0000000..2273c05 --- /dev/null +++ b/deploy/systemd/standx-notify@.service @@ -0,0 +1,9 @@ +[Unit] +Description=StandX supervision failure notifier for %i + +[Service] +Type=oneshot +# Optional (leading '-'): the notifier still logs to the journal if absent. +EnvironmentFile=-/etc/standx/maker.env +# %i is the failed unit name, passed via OnFailure=standx-notify@%n.service. +ExecStart=/opt/standx/deploy/systemd/standx-notify.sh %i diff --git a/deploy/systemd/standx-openobserve-catchup.service b/deploy/systemd/standx-openobserve-catchup.service new file mode 100644 index 0000000..dea5a84 --- /dev/null +++ b/deploy/systemd/standx-openobserve-catchup.service @@ -0,0 +1,17 @@ +[Unit] +Description=StandX OpenObserve crash/boot catch-up (re-upload un-ingested log tail) +Documentation=file:/opt/standx/deploy/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=/etc/standx/maker.env +WorkingDirectory=/opt/standx +# Idempotent: resumes each run's incremental checkpoint, so already-ingested +# events are not re-sent. The script always exits 0 (best-effort), so a down +# backend never blocks boot or the maker unit that orders after it. +ExecStart=/opt/standx/scripts/openobserve_catchup.sh + +[Install] +WantedBy=multi-user.target diff --git a/scripts/openobserve_catchup.sh b/scripts/openobserve_catchup.sh new file mode 100755 index 0000000..d6023c8 --- /dev/null +++ b/scripts/openobserve_catchup.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Boot/crash catch-up: re-upload any un-ingested tail of local maker logs. +# +# On SIGKILL / power loss the local NDJSON survives but the tail past the last +# checkpoint was never sent, so the dashboard is stale until someone runs a +# manual upload. This script closes that gap: for every `.ndjson` in +# the log directory it runs the ingester in incremental (non-follow) mode with +# `--run-id `, which reuses the exact same checkpoint key the live +# follow uploader used and therefore resumes without re-sending events. +# +# It is safe to run repeatedly (idempotent) and at boot before the maker +# starts. Requires OPENOBSERVE_* to be exported (or sourced from an env file). +set -uo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/.." && pwd)" +log_dir="${STANDX_LOG_DIR:-$repo_root/var/standx}" + +# Optionally source an env file (e.g. deploy/openobserve/.env) so the catch-up +# can run from a boot unit without a pre-populated shell. +env_file="${OPENOBSERVE_ENV_FILE:-}" +if [[ -n "$env_file" && -f "$env_file" ]]; then + set -a + # shellcheck disable=SC1090 + . "$env_file" + set +a +fi + +if [[ -z "${OPENOBSERVE_USER:-}" || -z "${OPENOBSERVE_PASSWORD:-}" ]]; then + printf 'openobserve catch-up skipped: OPENOBSERVE_USER/OPENOBSERVE_PASSWORD not set\n' >&2 + exit 0 +fi + +if [[ ! -d "$log_dir" ]]; then + printf 'openobserve catch-up: no log directory at %s; nothing to do\n' "$log_dir" >&2 + exit 0 +fi + +shopt -s nullglob +logs=("$log_dir"/*.ndjson) +shopt -u nullglob + +if ((${#logs[@]} == 0)); then + printf 'openobserve catch-up: no *.ndjson logs under %s; nothing to do\n' "$log_dir" >&2 + exit 0 +fi + +git_sha="$(git -C "$repo_root" rev-parse --short HEAD 2>/dev/null || true)" +failures=0 + +for log in "${logs[@]}"; do + run_id="$(basename "$log" .ndjson)" + printf 'openobserve catch-up: run_id=%s file=%s\n' "$run_id" "$log" >&2 + catchup_args=( + "$script_dir/openobserve_ingest.py" "$log" + --run-id "$run_id" + --incremental + ) + [[ -n "$git_sha" ]] && catchup_args+=(--git-sha "$git_sha") + # An exit of 1 here means "no new lines" (already fully ingested) or a bad + # file; log it and keep going so one stale file cannot block the rest. + if ! python3 "${catchup_args[@]}"; then + printf 'openobserve catch-up: nothing to upload or upload failed for run_id=%s\n' "$run_id" >&2 + failures=$((failures + 1)) + fi +done + +if ((failures > 0)); then + printf 'openobserve catch-up: completed with %s file(s) reporting no-new-data or error\n' "$failures" >&2 +fi +exit 0 diff --git a/scripts/run_maker_observed.sh b/scripts/run_maker_observed.sh index 45a25e4..21070fd 100755 --- a/scripts/run_maker_observed.sh +++ b/scripts/run_maker_observed.sh @@ -64,6 +64,23 @@ forward_signal() { fi } +# Emit an operational notice to stderr and, if configured, to a webhook. +# The webhook is best-effort: a failed POST never affects the maker. +notify() { + local message="$1" + printf '%s\n' "$message" >&2 + if [[ -n "${STANDX_SUPERVISOR_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then + # Escape backslashes and double quotes so the message is valid JSON. + local escaped="${message//\\/\\\\}" + escaped="${escaped//\"/\\\"}" + curl -fsS -m 5 -X POST \ + -H 'Content-Type: application/json' \ + --data "{\"text\":\"$escaped\"}" \ + "$STANDX_SUPERVISOR_WEBHOOK" >/dev/null 2>&1 || + printf 'supervisor webhook post failed (message logged above)\n' >&2 + fi +} + trap cleanup EXIT trap 'forward_signal INT' INT trap 'forward_signal TERM' TERM @@ -82,29 +99,58 @@ if [[ -n "$config_file" && -f "$config_file" ]]; then config_hash="$(shasum -a 256 "$config_file" | awk '{print $1}')" fi +uploader_enabled=0 + +# Launch (or relaunch) the live follow uploader. Follow mode is incremental +# and resumes from the on-disk checkpoint, so a relaunch never re-sends events +# that were already ingested. +start_uploader() { + local upload_args=( + "$script_dir/openobserve_ingest.py" "$stdout_log" + --run-id "$run_id" + --incremental + --follow + --preflight + --poll-interval "${OPENOBSERVE_UPLOAD_INTERVAL:-2}" + ) + [[ -n "$git_sha" ]] && upload_args+=(--git-sha "$git_sha") + [[ -n "$config_hash" ]] && upload_args+=(--config-hash "$config_hash") + python3 "${upload_args[@]}" >&2 & + uploader_pid=$! +} + if [[ "${OPENOBSERVE_AUTO_UPLOAD:-0}" == "1" ]]; then if [[ -z "${OPENOBSERVE_USER:-}" || -z "${OPENOBSERVE_PASSWORD:-}" ]]; then printf 'OpenObserve live upload skipped: credentials are not exported\n' >&2 else - upload_args=( - "$script_dir/openobserve_ingest.py" "$stdout_log" - --run-id "$run_id" - --incremental - --follow - --preflight - --poll-interval "${OPENOBSERVE_UPLOAD_INTERVAL:-2}" - ) - [[ -n "$git_sha" ]] && upload_args+=(--git-sha "$git_sha") - [[ -n "$config_hash" ]] && upload_args+=(--config-hash "$config_hash") - python3 "${upload_args[@]}" >&2 & - uploader_pid=$! - printf 'OpenObserve live uploader starting: run_id=%s interval=%ss\n' \ - "$run_id" "${OPENOBSERVE_UPLOAD_INTERVAL:-2}" >&2 + uploader_enabled=1 + start_uploader + printf 'OpenObserve live uploader starting: run_id=%s interval=%ss pid=%s\n' \ + "$run_id" "${OPENOBSERVE_UPLOAD_INTERVAL:-2}" "$uploader_pid" >&2 fi fi "${args[@]}" >"$pipe_dir/stdout" 2>"$pipe_dir/stderr" & child_pid=$! + +# Supervise the uploader while the maker runs. If the follow loop dies +# mid-run (an uncaught error, OOM kill, etc.) the only remote symptom is a +# stale dashboard, so relaunch it and emit a notice. We poll instead of +# blocking on `wait "$child_pid"` so the check happens during the run, not +# after the maker already exited. +supervise_interval="${STANDX_SUPERVISE_INTERVAL:-5}" +while kill -0 "$child_pid" 2>/dev/null; do + if ((uploader_enabled == 1)) && [[ -n "$uploader_pid" ]] && + ! kill -0 "$uploader_pid" 2>/dev/null; then + uploader_exit=0 + wait "$uploader_pid" 2>/dev/null || uploader_exit=$? + notify "OpenObserve live uploader died (run_id=$run_id status=$uploader_exit) while maker still running; relaunching" + start_uploader + notify "OpenObserve live uploader relaunched (run_id=$run_id pid=$uploader_pid)" + fi + sleep "$supervise_interval" +done + wait "$child_pid" child_status=$? wait "$stdout_tee_pid" || true