From 54ab44e75f759fd8fdd0a726800b4f7276790fa5 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 18:37:19 +0200 Subject: [PATCH 1/8] feat(health): add health config keys and CLI modes - New allowlisted config keys: HEALTH_{CPU,MEM,DISK}_{WARN,CRIT}, HEALTH_CPU_RUNS, HEALTH_IGNORE_IDS, HEALTH_STATE_DIR, NTFY_URL, NTFY_TOKEN_FILE, HEALTH_MAIL_TO, HEALTH_MAIL_FROM; inline NTFY_TOKEN is ignored and rejected in notification modes. - _validate_health_config runs only in health/check/test-notify modes. - New flags --health, --check, --dry-run, --test-notify with mode resolution and invalid-combination checks; fatal setup errors exit 3 in --check mode. - Extract _fmt_duration and _owner_mode_ok helpers. Co-Authored-By: Claude Opus 5 --- proxmox-manager.sh | 333 +++++++++++++++++++++++++++++++++++++++++++-- tests/run.sh | 74 ++++++++++ 2 files changed, 393 insertions(+), 14 deletions(-) diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 1bb782d..b535149 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -27,6 +27,30 @@ FORCE_MODE=0 # Set to 1 via --force to skip all confirm() prompts FILTER_NAME="" # ERE substring-match against VM/CT name (empty = no filter) declare -A _type_cache=() # ID→type cache populated by main_menu; used by type_of_id() +# Health monitoring (--health / --check). A threshold of 0 disables that level. +HEALTH_CPU_WARN="${HEALTH_CPU_WARN:-85}" +HEALTH_CPU_CRIT="${HEALTH_CPU_CRIT:-95}" +HEALTH_MEM_WARN="${HEALTH_MEM_WARN:-90}" +HEALTH_MEM_CRIT="${HEALTH_MEM_CRIT:-95}" +HEALTH_DISK_WARN="${HEALTH_DISK_WARN:-85}" +HEALTH_DISK_CRIT="${HEALTH_DISK_CRIT:-95}" +HEALTH_CPU_RUNS="${HEALTH_CPU_RUNS:-3}" # consecutive --check runs before a CPU alert +HEALTH_IGNORE_IDS="${HEALTH_IGNORE_IDS:-}" # VMIDs excluded from health checks (comma/space separated) +HEALTH_STATE_DIR="${HEALTH_STATE_DIR:-/var/lib/pman}" +NTFY_URL="${NTFY_URL:-}" # e.g. https://ntfy.sh/my-topic (opt-in) +NTFY_TOKEN_FILE="${NTFY_TOKEN_FILE:-}" # file with the ntfy access token, mode 0600 +HEALTH_MAIL_TO="${HEALTH_MAIL_TO:-}" # recipient(s), comma separated (opt-in, needs sendmail) +HEALTH_MAIL_FROM="${HEALTH_MAIL_FROM:-}" # optional sender address +HEALTH_FLAG=0 +CHECK_FLAG=0 +DRY_RUN=0 +TEST_NOTIFY_FLAG=0 +FATAL_EXIT=1 # exit code for fatal setup errors (3 = UNKNOWN in --check mode) +_CONFIG_INLINE_TOKEN=0 # set when a config file contains an inline NTFY_TOKEN +MSG_TITLE='' # notification message composed by _compose_message +MSG_BODY='' +MSG_PRIORITY='' + # ============================================================================= # COLORS (active only on a real TTY, or when NO_COLOR is unset) # ============================================================================= @@ -207,9 +231,17 @@ _load_config_file() { fi case "$key" in - STOP_TIMEOUT | LOG_FILE | PROXMOX_MANAGER_SPICE_ADDR) + STOP_TIMEOUT | LOG_FILE | PROXMOX_MANAGER_SPICE_ADDR | \ + HEALTH_CPU_WARN | HEALTH_CPU_CRIT | HEALTH_MEM_WARN | HEALTH_MEM_CRIT | \ + HEALTH_DISK_WARN | HEALTH_DISK_CRIT | HEALTH_CPU_RUNS | HEALTH_IGNORE_IDS | \ + HEALTH_STATE_DIR | NTFY_URL | NTFY_TOKEN_FILE | HEALTH_MAIL_TO | HEALTH_MAIL_FROM) printf -v "$key" '%s' "$value" ;; + NTFY_TOKEN) + # Secrets never live in the config file itself; see NTFY_TOKEN_FILE. + _CONFIG_INLINE_TOKEN=1 + _config_warn "Ignoring inline NTFY_TOKEN in $file; store the token in a mode 0600 file and set NTFY_TOKEN_FILE." + ;; *) _config_warn "Ignoring unsupported setting '$key' in $file." ;; @@ -217,6 +249,21 @@ _load_config_file() { done <"$file" } +# _owner_mode_ok PATH MASK — true when PATH is owned by the current user and +# none of the octal permission bits in MASK (e.g. 022, 077) are set. +_owner_mode_ok() { + local path="$1" mask="$2" owner mode + owner="$(stat -Lc '%u' -- "$path" 2>/dev/null || printf 'invalid')" + mode="$(stat -Lc '%a' -- "$path" 2>/dev/null || printf 'invalid')" + if [[ "$owner" != "$EUID" || ! "$mode" =~ ^[0-7]{3,4}$ ]]; then + return 1 + fi + if ((8#$mode & 8#$mask)); then + return 1 + fi + return 0 +} + # _prepare_log_file — require a private regular file in a trusted directory. _prepare_log_file() { [[ -z "$LOG_FILE" ]] && return 0 @@ -226,16 +273,14 @@ _prepare_log_file() { return 1 fi - local parent owner mode + local parent parent="$(dirname -- "$LOG_FILE")" if [[ ! -d "$parent" ]]; then _config_warn "LOG_FILE parent directory does not exist; logging disabled." LOG_FILE='' return 1 fi - owner="$(stat -Lc '%u' -- "$parent" 2>/dev/null || printf 'invalid')" - mode="$(stat -Lc '%a' -- "$parent" 2>/dev/null || printf 'invalid')" - if [[ "$owner" != "$EUID" || ! "$mode" =~ ^[0-7]{3,4}$ ]] || ((8#$mode & 8#022)); then + if ! _owner_mode_ok "$parent" 022; then _config_warn "LOG_FILE parent directory is not private and owner-controlled; logging disabled." LOG_FILE='' return 1 @@ -247,9 +292,7 @@ _prepare_log_file() { LOG_FILE='' return 1 fi - owner="$(stat -Lc '%u' -- "$LOG_FILE" 2>/dev/null || printf 'invalid')" - mode="$(stat -Lc '%a' -- "$LOG_FILE" 2>/dev/null || printf 'invalid')" - if [[ "$owner" != "$EUID" || ! "$mode" =~ ^[0-7]{3,4}$ ]] || ((8#$mode & 8#077)); then + if ! _owner_mode_ok "$LOG_FILE" 077; then _config_warn "LOG_FILE must be owned by the current user with mode 0600; logging disabled." LOG_FILE='' return 1 @@ -266,6 +309,101 @@ _prepare_log_file() { return 0 } +# _valid_mail_addr ADDR — conservative e-mail address check (no quoting, no spaces). +_valid_mail_addr() { + [[ "$1" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*$ ]] +} + +# _validate_health_config [notify] — validate health settings; print errors, return 1 on failure. +# With "notify", the notification channel settings are validated as well. +# Only called in health/check/test-notify modes so bad settings never block the TUI. +_validate_health_config() { + local scope="${1:-}" rc=0 key val metric w c addr + local -a addrs=() + for key in HEALTH_CPU_WARN HEALTH_CPU_CRIT HEALTH_MEM_WARN HEALTH_MEM_CRIT HEALTH_DISK_WARN HEALTH_DISK_CRIT; do + val="${!key}" + if [[ ! "$val" =~ ^[0-9]{1,3}$ ]] || ((10#$val > 100)); then + err "$key must be an integer between 0 and 100 (got '$val')." + rc=1 + else + printf -v "$key" '%d' "$((10#$val))" + fi + done + if ((rc == 0)); then + for metric in CPU MEM DISK; do + key="HEALTH_${metric}_WARN" + w="${!key}" + key="HEALTH_${metric}_CRIT" + c="${!key}" + if ((w > 0 && c > 0 && w >= c)); then + err "HEALTH_${metric}_WARN ($w) must be lower than HEALTH_${metric}_CRIT ($c)." + rc=1 + fi + done + fi + if [[ ! "$HEALTH_CPU_RUNS" =~ ^[0-9]{1,3}$ ]] || ((10#$HEALTH_CPU_RUNS < 1 || 10#$HEALTH_CPU_RUNS > 100)); then + err "HEALTH_CPU_RUNS must be an integer between 1 and 100 (got '$HEALTH_CPU_RUNS')." + rc=1 + else + HEALTH_CPU_RUNS=$((10#$HEALTH_CPU_RUNS)) + fi + if [[ ! "$HEALTH_IGNORE_IDS" =~ ^[0-9,\ ]*$ ]]; then + err "HEALTH_IGNORE_IDS must be a comma or space separated list of VMIDs." + rc=1 + fi + if [[ ! "$HEALTH_STATE_DIR" =~ ^/[A-Za-z0-9._/-]*$ ]]; then + err "HEALTH_STATE_DIR must be an absolute path using only [A-Za-z0-9._/-]." + rc=1 + fi + if [[ "$scope" != "notify" ]]; then + return "$rc" + fi + + if ((_CONFIG_INLINE_TOKEN == 1)); then + err "NTFY_TOKEN must not be set in a config file; use NTFY_TOKEN_FILE (mode 0600)." + rc=1 + fi + if [[ -n "$NTFY_URL" && ! "$NTFY_URL" =~ ^https?://[A-Za-z0-9.-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~-]+)+/?$ ]]; then + err "NTFY_URL must look like https://host[:port]/topic (got an unsupported value)." + rc=1 + fi + if [[ -n "$NTFY_TOKEN_FILE" ]]; then + if [[ -z "$NTFY_URL" ]]; then + err "NTFY_TOKEN_FILE is set but NTFY_URL is empty." + rc=1 + fi + if [[ "$NTFY_TOKEN_FILE" != /* || -L "$NTFY_TOKEN_FILE" || ! -f "$NTFY_TOKEN_FILE" ]]; then + err "NTFY_TOKEN_FILE must be an absolute path to a regular file (no symlink)." + rc=1 + elif ! _owner_mode_ok "$NTFY_TOKEN_FILE" 077; then + err "NTFY_TOKEN_FILE must be owned by the current user with mode 0600." + rc=1 + fi + if [[ "$NTFY_URL" == http://* ]]; then + _config_warn "NTFY_URL uses plain http; the access token is sent unencrypted." + fi + fi + if [[ -n "$HEALTH_MAIL_TO" ]]; then + IFS=',' read -r -a addrs <<<"$HEALTH_MAIL_TO" + if ((${#addrs[@]} == 0)); then + err "HEALTH_MAIL_TO contains no address." + rc=1 + fi + for addr in "${addrs[@]}"; do + if ! _valid_mail_addr "$addr"; then + err "HEALTH_MAIL_TO contains an invalid address (comma separated, no spaces)." + rc=1 + break + fi + done + fi + if [[ -n "$HEALTH_MAIL_FROM" ]] && ! _valid_mail_addr "$HEALTH_MAIL_FROM"; then + err "HEALTH_MAIL_FROM must be a single plain e-mail address." + rc=1 + fi + return "$rc" +} + # _repeat CHAR N — print CHAR repeated N times. _repeat() { local char="$1" n="$2" out='' @@ -320,14 +458,14 @@ require_root() { fi ((EUID == 0)) || { err "Please run as root." - exit 1 + exit "$FATAL_EXIT" } } require_tools() { { have qm || have pct; } || { err "Neither 'qm' nor 'pct' found. Run on a Proxmox VE host." - exit 1 + exit "$FATAL_EXIT" } } @@ -360,6 +498,12 @@ Options: --once Run a single interactive refresh (useful for TTY recording) --timeout SECS Timeout for stop operations in seconds (default: 60) --force Skip all confirmation prompts (use with care) + --health Show CPU/memory/disk health of local VMs/CTs + (combine with --list for plain text or --json; --filter/--name apply) + --check Run health checks for cron; alert on changes; exit 0/1/2/3 + (OK/WARN/CRIT/UNKNOWN) + --dry-run With --check: print the result and message, send nothing, keep state + --test-notify Send a test message through all configured channels --version Print version and exit -h, --help Show this help EOF @@ -427,6 +571,18 @@ parse_args() { --force) FORCE_MODE=1 ;; + --health) + HEALTH_FLAG=1 + ;; + --check) + CHECK_FLAG=1 + ;; + --dry-run) + DRY_RUN=1 + ;; + --test-notify) + TEST_NOTIFY_FLAG=1 + ;; --version) printf 'proxmox-manager.sh %s\n' "$(_script_version)" exit 0 @@ -456,6 +612,41 @@ parse_args() { err "Options --list and --json are not combinable." exit 1 fi + _resolve_mode +} + +# _resolve_mode — derive MODE from the health flags and reject invalid combinations. +_resolve_mode() { + if ((DRY_RUN == 1 && CHECK_FLAG == 0)); then + err "Option --dry-run requires --check." + exit 1 + fi + if ((CHECK_FLAG == 1)); then + if ((LIST_FLAG == 1 || JSON_FLAG == 1 || HEALTH_FLAG == 1 || TEST_NOTIFY_FLAG == 1)); then + err "Option --check is not combinable with --list, --json, --health or --test-notify." + exit 1 + fi + if [[ -n "$FILTER_STATUS" || -n "$FILTER_NAME" ]]; then + err "Option --check always checks all guests; --filter and --name are not supported." + exit 1 + fi + MODE="check" + return 0 + fi + if ((TEST_NOTIFY_FLAG == 1)); then + if ((LIST_FLAG == 1 || JSON_FLAG == 1 || HEALTH_FLAG == 1)); then + err "Option --test-notify is not combinable with --list, --json or --health." + exit 1 + fi + MODE="test_notify" + return 0 + fi + if ((HEALTH_FLAG == 1)); then + MODE="health" + ((LIST_FLAG == 1)) && MODE="health_list" + ((JSON_FLAG == 1)) && MODE="health_json" + fi + return 0 } # ============================================================================= @@ -645,7 +836,14 @@ _box_content() { _uptime_short() { local secs secs="$(cut -d. -f1 /proc/uptime 2>/dev/null || true)" - [[ "$secs" =~ ^[0-9]+$ ]] || return 0 + _fmt_duration "$secs" +} + +# _fmt_duration SECS — compact duration, e.g. "17d 13h 4m"; prints nothing for non-numbers. +_fmt_duration() { + local secs="$1" + [[ "$secs" =~ ^[0-9]{1,12}$ ]] || return 0 + secs=$((10#$secs)) local d=$((secs / 86400)) h=$((secs % 86400 / 3600)) m=$((secs % 3600 / 60)) if ((d > 0)); then printf '%sd %sh %sm' "$d" "$h" "$m" @@ -923,6 +1121,82 @@ print_table() { return 0 } +# ============================================================================= +# HEALTH — DATA LAYER & VIEW +# ============================================================================= + +# _local_node — print the short name of the local node (validated); return 1 if unknown. +_local_node() { + local n + n="$(hostname -s 2>/dev/null || true)" + [[ "$n" =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,62}$ ]] || return 1 + printf '%s' "$n" +} + +print_health_table() { + err "Health view is not implemented yet." + return 1 +} + +print_health_json() { + err "Health view is not implemented yet." + return 1 +} + +# ============================================================================= +# HEALTH — CHECK ENGINE (--check) +# ============================================================================= + +# _check_unknown MESSAGE — report an UNKNOWN result (Nagios style) and exit 3. +_check_unknown() { + err "$*" + printf 'PMAN UNKNOWN - %s\n' "$*" + exit 3 +} + +run_check() { + _check_unknown "--check is not implemented yet." +} + +# ============================================================================= +# HEALTH — NOTIFICATIONS +# ============================================================================= + +# _notify_channels_configured — true when at least one notification channel is set. +_notify_channels_configured() { + [[ -n "$NTFY_URL" || -n "$HEALTH_MAIL_TO" ]] +} + +# _notify_all — deliver MSG_TITLE/MSG_BODY/MSG_PRIORITY through every configured channel. +# Returns 1 only when channels are configured and all of them failed. +# Channel senders (ntfy, sendmail) are added in a later work package. +_notify_all() { + if ! _notify_channels_configured; then + return 0 + fi + log "WARN" "Notification delivery not implemented yet; '${MSG_TITLE}' (${MSG_PRIORITY}, ${#MSG_BODY} bytes) not sent." + return 0 +} + +# run_test_notify — send a test message through all configured channels. +run_test_notify() { + if ! _notify_channels_configured; then + err "No notification channel configured (set NTFY_URL and/or HEALTH_MAIL_TO)." + exit 1 + fi + local node + node="$(_local_node || printf 'unknown')" + MSG_TITLE="pman ${node}: test notification" + MSG_BODY="Test message from proxmox-manager on ${node}." + MSG_PRIORITY="low" + if _notify_all; then + ok "Test notification handed to all configured channels." + exit 0 + fi + err "Test notification failed on all channels." + exit 1 +} + # ============================================================================= # INTERACTIVE MENUS # ============================================================================= @@ -1670,16 +1944,17 @@ main() { _load_config_file /etc/pmanrc _load_config_file "${HOME}/.pmanrc" _prepare_log_file || true + parse_args "$@" + [[ "$MODE" == "check" ]] && FATAL_EXIT=3 # Validate settings after parsing config as data. if [[ ! "$STOP_TIMEOUT" =~ ^[0-9]+$ ]] || ((STOP_TIMEOUT < 1)); then err "STOP_TIMEOUT must be a positive integer (got '$STOP_TIMEOUT'). Check /etc/pmanrc or ~/.pmanrc." - exit 1 + exit "$FATAL_EXIT" fi if [[ -n "$PROXMOX_MANAGER_SPICE_ADDR" && ! "$PROXMOX_MANAGER_SPICE_ADDR" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]*$ ]]; then err "PROXMOX_MANAGER_SPICE_ADDR contains unsupported characters." - exit 1 + exit "$FATAL_EXIT" fi - parse_args "$@" if ((FORCE_MODE == 1)); then warn "--force active: all confirmation prompts will be skipped automatically." fi @@ -1690,6 +1965,16 @@ main() { fi log "INFO" "Starting proxmox-manager (mode=$MODE)" case "$MODE" in + health | health_list | health_json | test_notify) + local scope='' + [[ "$MODE" == "test_notify" ]] && scope="notify" + _validate_health_config "$scope" || exit 1 + ;; + check) + _validate_health_config notify || _check_unknown "Invalid health configuration." + ;; + esac + case "$MODE" in list) if print_table; then exit 0 @@ -1701,6 +1986,26 @@ main() { print_json exit 0 ;; + health | health_list) + if print_health_table; then + exit 0 + else + exit 1 + fi + ;; + health_json) + if print_health_json; then + exit 0 + else + exit 1 + fi + ;; + check) + run_check + ;; + test_notify) + run_test_notify + ;; interactive) while true; do main_menu || true diff --git a/tests/run.sh b/tests/run.sh index 825a7cf..23420b3 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -15,6 +15,12 @@ export TERM=dumb cd "$ROOT_DIR" +# Isolate from the caller's ~/.pmanrc and state; everything lives in one temp tree. +TEST_TMP="$(mktemp -d)" +trap 'rm -rf "$TEST_TMP"' EXIT +export HOME="$TEST_TMP/home" +mkdir -p "$HOME" + PASS=0 FAIL=0 @@ -320,6 +326,74 @@ else _fail "--filter invalid value should exit 1, got $filter_inv_exit" fi +# --------------------------------------------------------------------------- +# Health: CLI modes, config validation, helpers +# --------------------------------------------------------------------------- +# _rc CMD... — run CMD with stdout/stderr discarded and print its exit code. +_rc() { + local rc=0 + "$@" >/dev/null 2>&1 || rc=$? + printf '%s' "$rc" +} + +# _expect_rc LABEL EXPECTED CMD... — assert the exit code of CMD. +_expect_rc() { + local label="$1" expected="$2" actual + shift 2 + actual="$(_rc "$@")" + if [[ "$actual" == "$expected" ]]; then + _pass "$label" + else + _fail "$label (expected exit $expected, got $actual)" + fi +} + +_new_state_dir() { + local d + d="$(mktemp -d "$TEST_TMP/state.XXXXXX")" + chmod 700 "$d" + printf '%s' "$d" +} + +_expect_rc "health flags: --check with --json exits 1" 1 "$SCRIPT" --check --json +_expect_rc "health flags: --check with --health exits 1" 1 "$SCRIPT" --check --health +_expect_rc "health flags: --check with --filter exits 1" 1 "$SCRIPT" --check --filter running +_expect_rc "health flags: --dry-run without --check exits 1" 1 "$SCRIPT" --dry-run +_expect_rc "health flags: --test-notify with --list exits 1" 1 "$SCRIPT" --test-notify --list +_expect_rc "test-notify: no channel configured exits 1" 1 "$SCRIPT" --test-notify + +# Config validation: bad health settings are fatal (exit 3) only for --check. +printf '%s\n' 'HEALTH_MEM_WARN=96' 'HEALTH_MEM_CRIT=95' >"$HOME/.pmanrc" +_expect_rc "health config: WARN >= CRIT makes --check exit 3" 3 \ + env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check +_expect_rc "health config: invalid thresholds do not block --list" 0 "$SCRIPT" --list +printf '%s\n' 'NTFY_URL=https://ntfy.example.invalid/topic' 'NTFY_TOKEN=tk_inline' >"$HOME/.pmanrc" +cfg_out="$(HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check 2>&1)" && cfg_rc=0 || cfg_rc=$? +if [[ "$cfg_rc" == "3" && "$cfg_out" == *NTFY_TOKEN_FILE* && "$cfg_out" != *tk_inline* ]]; then + _pass "health config: inline NTFY_TOKEN rejected without echoing the secret" +else + _fail "health config: inline NTFY_TOKEN not rejected (exit $cfg_rc)" +fi +token_file="$TEST_TMP/ntfy.token" +printf 'tk_file\n' >"$token_file" +chmod 644 "$token_file" +printf '%s\n' 'NTFY_URL=https://ntfy.example.invalid/topic' "NTFY_TOKEN_FILE=$token_file" >"$HOME/.pmanrc" +_expect_rc "health config: token file with mode 0644 makes --check exit 3" 3 \ + env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check +printf '%s\n' 'NTFY_URL=ftp://ntfy.example.invalid/topic' >"$HOME/.pmanrc" +_expect_rc "health config: non-http NTFY_URL makes --check exit 3" 3 \ + env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check +printf '%s\n' 'HEALTH_MAIL_TO=root@localhost,bad address' >"$HOME/.pmanrc" +_expect_rc "health config: invalid HEALTH_MAIL_TO makes --check exit 3" 3 \ + env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check +rm -f "$HOME/.pmanrc" + +if [[ "$(_fmt_duration 90061)" == "1d 1h 1m" && "$(_fmt_duration 3660)" == "1h 1m" && "$(_fmt_duration 59)" == "0m" && -z "$(_fmt_duration abc)" ]]; then + _pass "_fmt_duration: formats days, hours and minutes" +else + _fail "_fmt_duration: unexpected output" +fi + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From 3b5c2eee48d35f2592ea981a3af0814985ff5935 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 18:40:27 +0200 Subject: [PATCH 2/8] feat(health): add health view with pvesh data layer - Read local guests from `pvesh get /cluster/resources` (30 s timeout), parse JSON on stdin with python3 into TSV; templates and guests on other nodes are skipped, names sanitized. - _health_checks is the single source for cpu/mem/disk/down levels; stopped guests with onboot=1 are CRIT. - --health (boxed), --health --list (plain) with findings and totals, --health --json with null for n/a metrics; --filter/--name apply. - Main menu key "h" opens the health overview; the status action adds a "Health:" line and stays silent without pvesh/python3. - Tests: pvesh mock with fixtures, onboot support in qm/pct mocks. Co-Authored-By: Claude Opus 5 --- proxmox-manager.sh | 563 +++++++++++++++++++++- tests/fixtures/resources-200-stopped.json | 6 + tests/fixtures/resources-cpu-hot.json | 6 + tests/fixtures/resources-hot.json | 6 + tests/fixtures/tasks-failed-new.json | 7 + tests/fixtures/tasks-failed-old.json | 4 + tests/fixtures/tasks-shutdown-200.json | 4 + tests/mock-bin/pct | 4 + tests/mock-bin/pvesh | 43 ++ tests/mock-bin/qm | 4 + tests/run.sh | 91 ++++ 11 files changed, 732 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/resources-200-stopped.json create mode 100644 tests/fixtures/resources-cpu-hot.json create mode 100644 tests/fixtures/resources-hot.json create mode 100644 tests/fixtures/tasks-failed-new.json create mode 100644 tests/fixtures/tasks-failed-old.json create mode 100644 tests/fixtures/tasks-shutdown-200.json create mode 100755 tests/mock-bin/pvesh diff --git a/proxmox-manager.sh b/proxmox-manager.sh index b535149..654ed4e 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -50,6 +50,11 @@ _CONFIG_INLINE_TOKEN=0 # set when a config file contains an inline NTFY_TOKEN MSG_TITLE='' # notification message composed by _compose_message MSG_BODY='' MSG_PRIORITY='' +HEALTH_NODE='' # local node name (set by _health_load) +HEALTH_ERR='' # reason for the last _health_load/_health_load_tasks failure +HEALTH_ROWS=() # parsed guest rows (see _health_parse_resources) +declare -A _HV_LEVEL=() # per-guest max health level (see _health_evaluate) +declare -A _HV_FIND=() # per-guest findings (see _health_evaluate) # ============================================================================= # COLORS (active only on a real TTY, or when NO_COLOR is unset) @@ -1133,14 +1138,558 @@ _local_node() { printf '%s' "$n" } -print_health_table() { - err "Health view is not implemented yet." +# _level_name LEVEL — OK / WARN / CRIT for 0 / 1 / 2. +_level_name() { + case "$1" in + 2) printf 'CRIT' ;; + 1) printf 'WARN' ;; + 0) printf 'OK' ;; + *) printf 'n/a' ;; + esac +} + +# _level_color LEVEL TEXT — print TEXT in the colour of LEVEL. +_level_color() { + local lvl="$1" txt="$2" + case "$lvl" in + 2) printf '%b%s%b' "$RED_BRIGHT" "$txt" "$NC" ;; + 1) printf '%b%s%b' "$YELLOW_BRIGHT" "$txt" "$NC" ;; + 0) printf '%b%s%b' "$GREEN_BRIGHT" "$txt" "$NC" ;; + *) printf '%b%s%b' "$DIM" "$txt" "$NC" ;; + esac +} + +# _level_for VALUE WARN CRIT — print 0 (OK), 1 (WARN) or 2 (CRIT); "-" when VALUE is n/a. +# A threshold of 0 disables that level. +_level_for() { + local v="$1" w="$2" c="$3" + if [[ ! "$v" =~ ^[0-9]{1,6}$ ]]; then + printf -- '-' + elif ((c > 0 && 10#$v >= c)); then + printf '2' + elif ((w > 0 && 10#$v >= w)); then + printf '1' + else + printf '0' + fi +} + +# _health_ignored ID — true when ID is listed in HEALTH_IGNORE_IDS. +_health_ignored() { + local id="$1" x + local -a ids=() + IFS=' ' read -r -a ids <<<"${HEALTH_IGNORE_IDS//,/ }" + for x in "${ids[@]}"; do + if [[ "$x" == "$id" ]]; then + return 0 + fi + done return 1 } +# _health_fetch resources|tasks [NODE] — raw pvesh JSON on stdout (30 s timeout). +_health_fetch() { + local kind="$1" node="${2:-}" + have pvesh || return 1 + case "$kind" in + resources) timeout 30 pvesh get /cluster/resources --type vm --output-format json 2>/dev/null ;; + tasks) timeout 30 pvesh get "/nodes/${node}/tasks" --limit 200 --output-format json 2>/dev/null ;; + *) return 1 ;; + esac +} + +# _health_parse_resources NODE — read /cluster/resources JSON on stdin and print TSV rows +# for NODE (templates skipped), sorted by VMID: +# vmid VM|CT status name cpu% mem% disk% uptime-seconds +# cpu/mem/disk are "-" when not applicable (guest not running, no guest-agent disk data). +_health_parse_resources() { + python3 -c ' +import json, sys +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + data = json.loads(sys.stdin.buffer.read()) +except Exception: + sys.exit(1) +if isinstance(data, dict): + data = data.get("data", []) +if not isinstance(data, list): + sys.exit(1) +node = sys.argv[1] + +def num(v): + try: + return float(v) + except (TypeError, ValueError): + return 0.0 + +rows = [] +for r in data: + if not isinstance(r, dict) or str(r.get("node", "")) != node: + continue + if num(r.get("template")) == 1: + continue + kind = {"qemu": "VM", "lxc": "CT"}.get(r.get("type")) + if kind is None: + continue + try: + vmid = int(r.get("vmid")) + except (TypeError, ValueError): + continue + if not 1 <= vmid <= 999999: + continue + status = "".join(c for c in str(r.get("status") or "").lower() if c.isalnum() or c in "-_")[:16] + status = status or "unknown" + name = "".join(c if c.isprintable() else "?" for c in str(r.get("name") or ""))[:64].strip() + name = name or "%s-%d" % (kind, vmid) + running = status == "running" + cpu = mem = disk = "-" + if running: + cpu = str(round(num(r.get("cpu")) * 100)) + maxmem = num(r.get("maxmem")) + if maxmem > 0: + mem = str(round(num(r.get("mem")) * 100 / maxmem)) + used, maxdisk = num(r.get("disk")), num(r.get("maxdisk")) + if used > 0 and maxdisk > 0: + disk = str(round(used * 100 / maxdisk)) + uptime = int(num(r.get("uptime"))) if running else 0 + rows.append((vmid, kind, status, name, cpu, mem, disk, str(uptime))) +rows.sort() +for row in rows: + print("\t".join(str(x) for x in row)) +' "$1" +} + +# _health_parse_tasks — read /nodes//tasks JSON on stdin and print TSV rows sorted +# by end time: endtime upid type id status +# Running tasks have endtime 0 and status "running"; missing ids are "-". +_health_parse_tasks() { + python3 -c ' +import json, sys +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + data = json.loads(sys.stdin.buffer.read()) +except Exception: + sys.exit(1) +if isinstance(data, dict): + data = data.get("data", []) +if not isinstance(data, list): + sys.exit(1) +safe = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") + +def keep(value, extra, limit): + return "".join(c if c in safe or c in extra else "_" for c in str(value or ""))[:limit] + +rows = [] +for t in data: + if not isinstance(t, dict): + continue + upid = keep(t.get("upid"), ":@._!=-", 200) + if not upid: + continue + try: + end = int(t.get("endtime") or 0) + except (TypeError, ValueError): + end = 0 + status = "".join(c if c.isprintable() else "?" for c in str(t.get("status") or ""))[:120].strip() + if end <= 0: + end, status = 0, "running" + status = status or "unknown" + rows.append((end, upid, keep(t.get("type"), "-_", 32) or "-", keep(t.get("id"), "._-", 64) or "-", status)) +rows.sort() +for row in rows: + print("\t".join(str(x) for x in row)) +' +} + +# _health_load — fetch and parse the local guests into HEALTH_ROWS (sets HEALTH_NODE). +# On failure returns 1 with a reason in HEALTH_ERR; prints nothing. +_health_load() { + local raw parsed + HEALTH_ROWS=() + HEALTH_ERR='' + if ! have python3; then + HEALTH_ERR="python3 is required for health checks." + return 1 + fi + if ! have pvesh; then + HEALTH_ERR="'pvesh' not found. Run on a Proxmox VE host." + return 1 + fi + if ! HEALTH_NODE="$(_local_node)"; then + HEALTH_ERR="Could not determine the local node name." + return 1 + fi + if ! raw="$(_health_fetch resources)"; then + HEALTH_ERR="pvesh get /cluster/resources failed or timed out." + return 1 + fi + if ! parsed="$(printf '%s' "$raw" | _health_parse_resources "$HEALTH_NODE")"; then + HEALTH_ERR="Could not parse the pvesh /cluster/resources output." + return 1 + fi + if [[ -n "$parsed" ]]; then + mapfile -t HEALTH_ROWS <<<"$parsed" + fi + return 0 +} + +# _guest_onboot ID TYPE — true when the guest config has "onboot: 1". +_guest_onboot() { + local id="$1" ty="$2" cfg='' + case "$ty" in + CT) cfg="$(pct config "$id" 2>/dev/null || true)" ;; + VM) cfg="$(qm config "$id" 2>/dev/null || true)" ;; + *) return 1 ;; + esac + grep -qE '^onboot:[[:space:]]*1[[:space:]]*$' <<<"$cfg" +} + +# _health_checks — single source of truth for --health and --check. +# For every non-ignored guest in HEALTH_ROWS print one TSV row per check: +# vmid type check(cpu|mem|disk|down) level(0|1|2|-) value name +# "down" is CRIT when the guest is stopped although onboot=1; --check adds +# the "stopped unexpectedly" state on top. +_health_checks() { + local row id ty st nm cpu mem disk up lvl + for row in "${HEALTH_ROWS[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + [[ -z "$id" ]] && continue + if _health_ignored "$id"; then + continue + fi + printf '%s\t%s\tcpu\t%s\t%s\t%s\n' "$id" "$ty" \ + "$(_level_for "$cpu" "$HEALTH_CPU_WARN" "$HEALTH_CPU_CRIT")" "$cpu" "$nm" + printf '%s\t%s\tmem\t%s\t%s\t%s\n' "$id" "$ty" \ + "$(_level_for "$mem" "$HEALTH_MEM_WARN" "$HEALTH_MEM_CRIT")" "$mem" "$nm" + printf '%s\t%s\tdisk\t%s\t%s\t%s\n' "$id" "$ty" \ + "$(_level_for "$disk" "$HEALTH_DISK_WARN" "$HEALTH_DISK_CRIT")" "$disk" "$nm" + lvl=0 + if [[ "$st" == "stopped" ]] && _guest_onboot "$id" "$ty"; then + lvl=2 + fi + printf '%s\t%s\tdown\t%s\t%s\t%s\n' "$id" "$ty" "$lvl" "$st" "$nm" + done +} + +# _health_finding CHECK LEVEL VALUE — human-readable description of a non-OK check. +_health_finding() { + local check="$1" lvl="$2" val="$3" label warn crit thr + case "$check" in + cpu) + label="CPU" + warn="$HEALTH_CPU_WARN" + crit="$HEALTH_CPU_CRIT" + ;; + mem) + label="memory" + warn="$HEALTH_MEM_WARN" + crit="$HEALTH_MEM_CRIT" + ;; + disk) + label="disk" + warn="$HEALTH_DISK_WARN" + crit="$HEALTH_DISK_CRIT" + ;; + down) + if [[ "$lvl" == "2" ]]; then + printf 'stopped although onboot=1' + else + printf 'stopped unexpectedly' + fi + return 0 + ;; + *) + printf '%s' "$check" + return 0 + ;; + esac + thr="$warn" + [[ "$lvl" == "2" ]] && thr="$crit" + if [[ "$val" =~ ^[0-9]+$ ]]; then + printf '%s %s%% (>= %s%%)' "$label" "$val" "$thr" + else + printf '%s above %s%% (no current value)' "$label" "$thr" + fi +} + +# _pct_text VALUE — "12%" or "n/a". +_pct_text() { + if [[ "$1" =~ ^[0-9]+$ ]]; then + printf '%s%%' "$1" + else + printf 'n/a' + fi +} + +# _json_num VALUE — VALUE when it is an integer, otherwise null. +_json_num() { + if [[ "$1" =~ ^[0-9]+$ ]]; then + printf '%s' "$1" + else + printf 'null' + fi +} + +# _health_evaluate — run _health_checks for HEALTH_ROWS and fill the caller-visible +# globals _HV_LEVEL[id] (max level) and _HV_FIND[id] (TAB-joined "level|check|value" items). +_health_evaluate() { + local checks cid chk clvl cval + _HV_LEVEL=() + _HV_FIND=() + checks="$(_health_checks)" + while IFS=$'\t' read -r cid _ chk clvl cval _; do + [[ -z "$cid" ]] && continue + [[ -z "${_HV_LEVEL[$cid]:-}" ]] && _HV_LEVEL[$cid]=0 + [[ "$clvl" =~ ^[0-2]$ ]] || continue + if ((clvl > _HV_LEVEL[$cid])); then + _HV_LEVEL[$cid]=$clvl + fi + if ((clvl > 0)); then + _HV_FIND[$cid]+="${clvl}|${chk}|${cval}"$'\t' + fi + done <<<"$checks" + return 0 +} + +# _health_row_shown STATUS NAME — apply --filter/--name to a health row. +_health_row_shown() { + local st="$1" nm="$2" + if [[ -n "$FILTER_STATUS" && "$st" != "$FILTER_STATUS" ]]; then + return 1 + fi + if [[ -n "$FILTER_NAME" && ! "$nm" =~ $FILTER_NAME ]]; then + return 1 + fi + return 0 +} + +# print_health_table — health overview of local guests (boxed for --health and the +# interactive menu, plain for --health --list). Returns 1 on error or when empty. +print_health_table() { + local draw_boxes=0 + [[ "$MODE" == "health" || "$MODE" == "interactive" ]] && draw_boxes=1 + if ! _health_load; then + err "$HEALTH_ERR" + return 1 + fi + _health_evaluate + + local -a rows=() findings=() + local row id ty st nm cpu mem disk up + for row in "${HEALTH_ROWS[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + if _health_row_shown "$st" "$nm"; then + rows+=("$row") + fi + done + + # │ + 2 + ID 6 + TYPE 5 + STATUS 8 + CPU% 5 + MEM% 5 + DISK% 5 + UPTIME 11 + HEALTH 7 + # + 8 separators + NAME + 2 + │ + local name_w=15 fixed=65 + for row in "${rows[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + (($(_vis_width "$nm") > name_w)) && name_w=$(_vis_width "$nm") + done + if ((draw_boxes)); then + local max_name=$(($(_term_cols) - fixed)) + ((name_w > max_name)) && name_w=$max_name + ((name_w < 10)) && name_w=10 + fi + local W=$((fixed + name_w)) + + local head + printf -v head '%-6s %-5s %-8s %5s %5s %5s %-11s %-7s %s' \ + "ID" "TYPE" "STATUS" "CPU%" "MEM%" "DISK%" "UPTIME" "HEALTH" "NAME" + if ((draw_boxes)); then + _draw_line_top $W + _box_content "${CYAN}" "${LINE_V}" $W " ${BOLD}${WHITE}${head}${NC}" + _draw_line_mid $W + else + printf '%b%s%b\n' "${BOLD}${WHITE}" "$head" "${NC}" + fi + + if ((${#rows[@]} == 0)); then + local msg="No VMs or containers found on node ${HEALTH_NODE}." + [[ -n "$FILTER_STATUS" || -n "$FILTER_NAME" ]] && msg="No VMs or containers match the filter." + if ((draw_boxes)); then + _box_content "${CYAN}" "${LINE_V}" $W " ${RED_BRIGHT}${msg}${NC}" + _draw_line_bot $W + else + printf '%b%s%b\n' "${RED_BRIGHT}" "$msg" "${NC}" + fi + return 1 + fi + + local n_ok=0 n_warn=0 n_crit=0 n_ign=0 n_run=0 lvl hname ty_col line upt item f_lvl f_chk f_val + local -a items=() + for row in "${rows[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + [[ "$st" == "running" ]] && n_run=$((n_run + 1)) + if _health_ignored "$id"; then + lvl='-' + hname="IGN" + n_ign=$((n_ign + 1)) + else + lvl="${_HV_LEVEL[$id]:-0}" + hname="$(_level_name "$lvl")" + case "$lvl" in + 2) n_crit=$((n_crit + 1)) ;; + 1) n_warn=$((n_warn + 1)) ;; + *) n_ok=$((n_ok + 1)) ;; + esac + IFS=$'\t' read -r -a items <<<"${_HV_FIND[$id]:-}" + for item in "${items[@]}"; do + IFS='|' read -r f_lvl f_chk f_val <<<"$item" + findings+=("${f_lvl}"$'\t'"${ty} ${id} (${nm}): $(_health_finding "$f_chk" "$f_lvl" "$f_val")") + done + fi + case "$ty" in + CT) printf -v ty_col '%b%-5s%b' "${MAGENTA_BRIGHT}" "$ty" "${NC}" ;; + VM) printf -v ty_col '%b%-5s%b' "${BLUE_BRIGHT}" "$ty" "${NC}" ;; + *) printf -v ty_col '%-5s' "$ty" ;; + esac + upt='-' + [[ "$st" == "running" ]] && upt="$(_fmt_duration "$up")" + printf -v line '%-6s %s %s %5s %5s %5s %-11s %s %s' \ + "$id" "$ty_col" \ + "$(_status_color "$st" "$(printf '%-8s' "$st")")" \ + "$cpu" "$mem" "$disk" "$(_truncate "$upt" 11)" \ + "$(_level_color "$lvl" "$(printf '%-7s' "$hname")")" \ + "$(_truncate "$nm" "$name_w")" + if ((draw_boxes)); then + _box_content "${CYAN}" "${LINE_V}" $W " ${line}" + else + printf '%s\n' "$line" + fi + done + + local totals f + printf -v totals '%bTotal:%b %s guests, %s running %b%s OK%b %b%s WARN%b %b%s CRIT%b' \ + "${BOLD}" "${NC}" "${#rows[@]}" "$n_run" \ + "${GREEN_BRIGHT}" "$n_ok" "${NC}" "${YELLOW_BRIGHT}" "$n_warn" "${NC}" \ + "${RED_BRIGHT}" "$n_crit" "${NC}" + ((n_ign > 0)) && totals+=" ${n_ign} ignored" + if ((draw_boxes)); then + _draw_line_mid $W + if ((${#findings[@]} > 0)); then + _box_content "${CYAN}" "${LINE_V}" $W " ${BOLD}Findings:${NC}" + for f in "${findings[@]}"; do + _box_content "${CYAN}" "${LINE_V}" $W \ + " $(_level_color "${f%%$'\t'*}" "[$(_level_name "${f%%$'\t'*}")]") $(_truncate "${f#*$'\t'}" $((W - 13)))" + done + _draw_line_mid $W + fi + _box_content "${CYAN}" "${LINE_V}" $W " ${totals}" + _draw_line_bot $W + else + if ((${#findings[@]} > 0)); then + printf '\nFindings:\n' + for f in "${findings[@]}"; do + printf '%s %s\n' "$(_level_color "${f%%$'\t'*}" "[$(_level_name "${f%%$'\t'*}")]")" "${f#*$'\t'}" + done + fi + printf '\n%s\n' "$totals" + fi + return 0 +} + +# print_health_json — health data of local guests as JSON (diagnostics on stderr only). print_health_json() { - err "Health view is not implemented yet." - return 1 + if ! _health_load; then + err "$HEALTH_ERR" + return 1 + fi + _health_evaluate + local row id ty st nm cpu mem disk up lvl hname ign first=1 ffirst item f_lvl f_chk f_val fval + local n=0 n_ok=0 n_warn=0 n_crit=0 n_ign=0 + local -a items=() + printf '{"node":"%s","guests":[' "$(json_escape "$HEALTH_NODE")" + for row in "${HEALTH_ROWS[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + _health_row_shown "$st" "$nm" || continue + n=$((n + 1)) + ign=false + if _health_ignored "$id"; then + ign=true + hname="IGNORED" + n_ign=$((n_ign + 1)) + else + lvl="${_HV_LEVEL[$id]:-0}" + hname="$(_level_name "$lvl")" + case "$lvl" in + 2) n_crit=$((n_crit + 1)) ;; + 1) n_warn=$((n_warn + 1)) ;; + *) n_ok=$((n_ok + 1)) ;; + esac + fi + ((first)) || printf ',' + first=0 + printf '{"id":%s,"type":"%s","status":"%s","name":"%s","cpu":%s,"mem":%s,"disk":%s,"uptime":%s,"health":"%s","ignored":%s,"findings":[' \ + "$id" "$(json_escape "$ty")" "$(json_escape "$st")" "$(json_escape "$nm")" \ + "$(_json_num "$cpu")" "$(_json_num "$mem")" "$(_json_num "$disk")" \ + "$([[ "$st" == "running" ]] && _json_num "$up" || printf 'null')" \ + "$hname" "$ign" + ffirst=1 + if [[ "$ign" == "false" ]]; then + IFS=$'\t' read -r -a items <<<"${_HV_FIND[$id]:-}" + for item in "${items[@]}"; do + IFS='|' read -r f_lvl f_chk f_val <<<"$item" + ((ffirst)) || printf ',' + ffirst=0 + fval="$(_json_num "$f_val")" + [[ "$fval" == "null" ]] && fval="\"$(json_escape "$f_val")\"" + printf '{"check":"%s","level":"%s","value":%s,"text":"%s"}' \ + "$f_chk" "$(_level_name "$f_lvl")" "$fval" \ + "$(json_escape "$(_health_finding "$f_chk" "$f_lvl" "$f_val")")" + done + fi + printf ']}' + done + printf '],"summary":{"guests":%s,"ok":%s,"warn":%s,"crit":%s,"ignored":%s}}\n' \ + "$n" "$n_ok" "$n_warn" "$n_crit" "$n_ign" + return 0 +} + +# _health_status_line ID — one-line health summary for the status action. +# Fails silently (return 1) when pvesh/python3 are missing or the guest is unknown. +_health_status_line() { + local want="$1" row id ty st nm cpu mem disk up found='' lvl tag upt + _validate_health_config >/dev/null 2>&1 || return 1 + _health_load 2>/dev/null || return 1 + for row in "${HEALTH_ROWS[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + if [[ "$id" == "$want" ]]; then + found="$row" + break + fi + done + [[ -n "$found" ]] || return 1 + if _health_ignored "$want"; then + lvl='-' + tag="IGNORED" + else + HEALTH_ROWS=("$found") + _health_evaluate + lvl="${_HV_LEVEL[$want]:-0}" + tag="$(_level_name "$lvl")" + fi + upt='-' + [[ "$st" == "running" ]] && upt="$(_fmt_duration "$up")" + printf ' Health: CPU %s MEM %s DISK %s up %s [%s]\n' \ + "$(_pct_text "$cpu")" "$(_pct_text "$mem")" "$(_pct_text "$disk")" "$upt" \ + "$(_level_color "$lvl" "$tag")" + return 0 +} + +# health_overview — interactive health view (main menu key "h"). +health_overview() { + echo + if _validate_health_config; then + print_health_table || true + fi + printf '\n %bPress Enter to continue...%b ' "${DIM}" "${NC}" + local _dummy + read_line _dummy } # ============================================================================= @@ -1220,13 +1769,14 @@ main_menu() { if ! print_table; then return 1; fi echo printf ' %b%s%b %s\n' "${BOLD}" "Keys:" "${NC}" \ - " = open action menu ${BOLD}r${NC} = refresh ${BOLD}q${NC} = quit" + " = open action menu ${BOLD}h${NC} = health ${BOLD}r${NC} = refresh ${BOLD}q${NC} = quit" printf ' %b→%b ' "${CYAN_BRIGHT}" "${NC}" local choice read_line choice case "$choice" in q | Q) exit 0 ;; r | R | '') return 0 ;; + h | H) health_overview ;; *) if [[ "$choice" =~ ^[0-9]+$ ]]; then if ! validate_vmid "$choice"; then @@ -1247,7 +1797,7 @@ main_menu() { err "VMID $choice not found. Press 'r' to refresh the list." fi else - err "Invalid input: '$choice'. Enter a numeric VMID, 'r', or 'q'." + err "Invalid input: '$choice'. Enter a numeric VMID, 'h', 'r', or 'q'." fi ;; esac @@ -1468,6 +2018,7 @@ do_action() { err "Could not retrieve status for VM $id." fi fi + _health_status_line "$id" 2>/dev/null || true ;; *) err "Unknown action: $act" ;; diff --git a/tests/fixtures/resources-200-stopped.json b/tests/fixtures/resources-200-stopped.json new file mode 100644 index 0000000..3240712 --- /dev/null +++ b/tests/fixtures/resources-200-stopped.json @@ -0,0 +1,6 @@ +[ +{"id": "lxc/100", "type": "lxc", "vmid": 100, "node": "mock-host", "name": "ct-one", "status": "running", "template": 0, "cpu": 0.12, "maxcpu": 2, "mem": 268435456, "maxmem": 1073741824, "disk": 1073741824, "maxdisk": 8589934592, "uptime": 3600}, +{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "mock-host", "name": "ct-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 1, "mem": 0, "maxmem": 536870912, "disk": 0, "maxdisk": 4294967296, "uptime": 0}, +{"id": "qemu/200", "type": "qemu", "vmid": 200, "node": "mock-host", "name": "vm-one", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 4, "mem": 0, "maxmem": 2147483648, "disk": 0, "maxdisk": 34359738368, "uptime": 0}, +{"id": "qemu/201", "type": "qemu", "vmid": 201, "node": "mock-host", "name": "vm-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 2, "mem": 0, "maxmem": 1073741824, "disk": 0, "maxdisk": 10737418240, "uptime": 0} +] diff --git a/tests/fixtures/resources-cpu-hot.json b/tests/fixtures/resources-cpu-hot.json new file mode 100644 index 0000000..e8f0f1f --- /dev/null +++ b/tests/fixtures/resources-cpu-hot.json @@ -0,0 +1,6 @@ +[ +{"id": "lxc/100", "type": "lxc", "vmid": 100, "node": "mock-host", "name": "ct-one", "status": "running", "template": 0, "cpu": 0.12, "maxcpu": 2, "mem": 268435456, "maxmem": 1073741824, "disk": 1073741824, "maxdisk": 8589934592, "uptime": 3600}, +{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "mock-host", "name": "ct-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 1, "mem": 0, "maxmem": 536870912, "disk": 0, "maxdisk": 4294967296, "uptime": 0}, +{"id": "qemu/200", "type": "qemu", "vmid": 200, "node": "mock-host", "name": "vm-one", "status": "running", "template": 0, "cpu": 0.9, "maxcpu": 4, "mem": 536870912, "maxmem": 2147483648, "disk": 0, "maxdisk": 34359738368, "uptime": 90061}, +{"id": "qemu/201", "type": "qemu", "vmid": 201, "node": "mock-host", "name": "vm-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 2, "mem": 0, "maxmem": 1073741824, "disk": 0, "maxdisk": 10737418240, "uptime": 0} +] diff --git a/tests/fixtures/resources-hot.json b/tests/fixtures/resources-hot.json new file mode 100644 index 0000000..fe342ee --- /dev/null +++ b/tests/fixtures/resources-hot.json @@ -0,0 +1,6 @@ +[ +{"id": "lxc/100", "type": "lxc", "vmid": 100, "node": "mock-host", "name": "ct-one", "status": "running", "template": 0, "cpu": 0.12, "maxcpu": 2, "mem": 1030792151, "maxmem": 1073741824, "disk": 7730941132, "maxdisk": 8589934592, "uptime": 3600}, +{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "mock-host", "name": "ct-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 1, "mem": 0, "maxmem": 536870912, "disk": 0, "maxdisk": 4294967296, "uptime": 0}, +{"id": "qemu/200", "type": "qemu", "vmid": 200, "node": "mock-host", "name": "vm-one", "status": "running", "template": 0, "cpu": 0.9, "maxcpu": 4, "mem": 536870912, "maxmem": 2147483648, "disk": 0, "maxdisk": 34359738368, "uptime": 90061}, +{"id": "qemu/201", "type": "qemu", "vmid": 201, "node": "mock-host", "name": "vm-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 2, "mem": 0, "maxmem": 1073741824, "disk": 0, "maxdisk": 10737418240, "uptime": 0} +] diff --git a/tests/fixtures/tasks-failed-new.json b/tests/fixtures/tasks-failed-new.json new file mode 100644 index 0000000..b8a4204 --- /dev/null +++ b/tests/fixtures/tasks-failed-new.json @@ -0,0 +1,7 @@ +[ +{"upid": "UPID:mock-host:00001000:00002000:65000000:vzdump::root@pam:", "node": "mock-host", "type": "vzdump", "id": "", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000100, "status": "job errors"}, +{"upid": "UPID:mock-host:00001001:00002001:65000001:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000200, "endtime": 1700000205, "status": "OK"}, +{"upid": "UPID:mock-host:00001002:00002002:65000002:vzdump:100:root@pam:", "node": "mock-host", "type": "vzdump", "id": "100", "user": "root@pam", "starttime": 1700000300, "endtime": 1700000400, "status": "WARNINGS: 1"}, +{"upid": "UPID:mock-host:00001003:00002003:65000003:qmstart:201:root@pam:", "node": "mock-host", "type": "qmstart", "id": "201", "user": "root@pam", "starttime": 1700000500, "endtime": 1700000501, "status": "start failed: QEMU exited with code 1"}, +{"upid": "UPID:mock-host:00001004:00002004:65000004:vncproxy:200:root@pam:", "node": "mock-host", "type": "vncproxy", "id": "200", "user": "root@pam", "starttime": 1700000600} +] diff --git a/tests/fixtures/tasks-failed-old.json b/tests/fixtures/tasks-failed-old.json new file mode 100644 index 0000000..5675aa0 --- /dev/null +++ b/tests/fixtures/tasks-failed-old.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001000:00002000:65000000:vzdump::root@pam:", "node": "mock-host", "type": "vzdump", "id": "", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000100, "status": "job errors"}, +{"upid": "UPID:mock-host:00001001:00002001:65000001:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000200, "endtime": 1700000205, "status": "OK"} +] diff --git a/tests/fixtures/tasks-shutdown-200.json b/tests/fixtures/tasks-shutdown-200.json new file mode 100644 index 0000000..e7bff47 --- /dev/null +++ b/tests/fixtures/tasks-shutdown-200.json @@ -0,0 +1,4 @@ +[ +{"upid":"UPID:mock-host:00001000:00002000:65000000:qmstart:200:root@pam:","node":"mock-host","type":"qmstart","id":"200","user":"root@pam","starttime":1700000000,"endtime":1700000005,"status":"OK"}, +{"upid":"UPID:mock-host:00001100:00002100:65000100:qmshutdown:200:root@pam:","node":"mock-host","type":"qmshutdown","id":"200","user":"root@pam","starttime":@NOW@,"endtime":@NOW@,"status":"OK"} +] diff --git a/tests/mock-bin/pct b/tests/mock-bin/pct index 6c6056d..1742f97 100755 --- a/tests/mock-bin/pct +++ b/tests/mock-bin/pct @@ -24,6 +24,10 @@ EOL 101) echo "hostname: ct-fallback" ;; *) echo "hostname: ct-${1:-unknown}" ;; esac + # PMAN_MOCK_ONBOOT_IDS: space separated VMIDs that report "onboot: 1". + if [[ " ${PMAN_MOCK_ONBOOT_IDS:-} " == *" ${1:-} "* ]]; then + echo "onboot: 1" + fi ;; status) echo "status: running" diff --git a/tests/mock-bin/pvesh b/tests/mock-bin/pvesh new file mode 100755 index 0000000..211daa9 --- /dev/null +++ b/tests/mock-bin/pvesh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Mock for `pvesh get ... --output-format json`. +# PMAN_MOCK_RESOURCES / PMAN_MOCK_TASKS: fixture files replacing the inline defaults. +# The token @NOW@ in a fixture is replaced with the current epoch. +# PMAN_MOCK_PVESH_FAIL=1 makes every call fail. +set -euo pipefail + +if [[ "${PMAN_MOCK_PVESH_FAIL:-0}" == "1" ]]; then + echo "mock pvesh: simulated failure" >&2 + exit 1 +fi + +emit() { + local file="$1" default="$2" + if [[ -n "$file" ]]; then + sed "s/@NOW@/$(date +%s)/g" "$file" + else + printf '%s\n' "$default" + fi +} + +[[ "${1:-}" == "get" ]] || exit 2 +case "${2:-}" in + /cluster/resources) + emit "${PMAN_MOCK_RESOURCES:-}" '[ +{"id":"lxc/100","type":"lxc","vmid":100,"node":"mock-host","name":"ct-one","status":"running","template":0,"cpu":0.12,"maxcpu":2,"mem":268435456,"maxmem":1073741824,"disk":1073741824,"maxdisk":8589934592,"uptime":3600}, +{"id":"lxc/101","type":"lxc","vmid":101,"node":"mock-host","name":"ct-fallback","status":"stopped","template":0,"cpu":0,"maxcpu":1,"mem":0,"maxmem":536870912,"disk":0,"maxdisk":4294967296,"uptime":0}, +{"id":"qemu/200","type":"qemu","vmid":200,"node":"mock-host","name":"vm-one","status":"running","template":0,"cpu":0.05,"maxcpu":4,"mem":536870912,"maxmem":2147483648,"disk":0,"maxdisk":34359738368,"uptime":90061}, +{"id":"qemu/201","type":"qemu","vmid":201,"node":"mock-host","name":"vm-fallback","status":"stopped","template":0,"cpu":0,"maxcpu":2,"mem":0,"maxmem":1073741824,"disk":0,"maxdisk":10737418240,"uptime":0}, +{"id":"qemu/9000","type":"qemu","vmid":9000,"node":"mock-host","name":"tpl-debian","status":"stopped","template":1,"cpu":0,"maxcpu":1,"mem":0,"maxmem":1073741824,"disk":0,"maxdisk":10737418240,"uptime":0}, +{"id":"qemu/300","type":"qemu","vmid":300,"node":"other-node","name":"vm-elsewhere","status":"running","template":0,"cpu":0.99,"maxcpu":2,"mem":1073741824,"maxmem":1073741824,"disk":0,"maxdisk":10737418240,"uptime":100} +]' + ;; + /nodes/*/tasks) + emit "${PMAN_MOCK_TASKS:-}" '[ +{"upid":"UPID:mock-host:00001000:00002000:65000000:qmstart:200:root@pam:","node":"mock-host","type":"qmstart","id":"200","user":"root@pam","starttime":1700000000,"endtime":1700000005,"status":"OK"} +]' + ;; + *) + echo "mock pvesh: unsupported path ${2:-}" >&2 + exit 2 + ;; +esac diff --git a/tests/mock-bin/qm b/tests/mock-bin/qm index c9deece..6a31caf 100755 --- a/tests/mock-bin/qm +++ b/tests/mock-bin/qm @@ -45,6 +45,10 @@ EOL 201) echo "name: vm-fallback" ;; *) echo "name: vm-${1:-unknown}" ;; esac + # PMAN_MOCK_ONBOOT_IDS: space separated VMIDs that report "onboot: 1". + if [[ " ${PMAN_MOCK_ONBOOT_IDS:-} " == *" ${1:-} "* ]]; then + echo "onboot: 1" + fi ;; status) echo "status: running" diff --git a/tests/run.sh b/tests/run.sh index 23420b3..976f14b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -394,6 +394,97 @@ else _fail "_fmt_duration: unexpected output" fi +# --------------------------------------------------------------------------- +# Health view: --health, --health --list, --health --json, menu key, status line +# --------------------------------------------------------------------------- +FIXTURES="$ROOT_DIR/tests/fixtures" + +health_list_out="$("$SCRIPT" --health --list)" +if grep -qE '^100 +CT +running' <<<"$health_list_out" && grep -qE '^200 +VM +running' <<<"$health_list_out" && + ! grep -qE '^(300|9000) ' <<<"$health_list_out"; then + _pass "--health --list: local guests only, templates and other nodes skipped" +else + _fail "--health --list: unexpected guest set" +fi + +printf '%s\n' 'UNSUPPORTED_KEY=1' >"$HOME/.pmanrc" +health_json_out="$("$SCRIPT" --health --json 2>/dev/null)" +rm -f "$HOME/.pmanrc" +if python3 -c ' +import json, sys +d = json.loads(sys.stdin.read()) +g = {x["id"]: x for x in d["guests"]} +assert d["node"] == "mock-host" +assert sorted(g) == [100, 101, 200, 201] +assert g[200]["disk"] is None and g[200]["cpu"] == 5 and g[200]["uptime"] == 90061 +assert g[101]["cpu"] is None and g[101]["uptime"] is None +assert d["summary"] == {"guests": 4, "ok": 4, "warn": 0, "crit": 0, "ignored": 0} +' <<<"$health_json_out" 2>/dev/null; then + _pass "--health --json: valid JSON on stdout only, n/a metrics are null" +else + _fail "--health --json: invalid JSON or unexpected content" +fi + +hot_out="$(PMAN_MOCK_RESOURCES="$FIXTURES/resources-hot.json" PMAN_MOCK_ONBOOT_IDS=101 "$SCRIPT" --health --list)" +if grep -q '\[CRIT\] CT 100 (ct-one): memory 96%' <<<"$hot_out" && + grep -q '\[WARN\] VM 200 (vm-one): CPU 90%' <<<"$hot_out" && + grep -q '\[CRIT\] CT 101 (ct-fallback): stopped although onboot=1' <<<"$hot_out" && + grep -q '1 OK 1 WARN 2 CRIT' <<<"$hot_out"; then + _pass "--health: thresholds and onboot produce findings and totals" +else + _fail "--health: findings or totals missing" +fi + +ign_out="$(PMAN_MOCK_RESOURCES="$FIXTURES/resources-hot.json" HEALTH_IGNORE_IDS=100 "$SCRIPT" --health --list)" +if grep -qE '^100 .* IGN ' <<<"$ign_out" && ! grep -q 'CT 100 (ct-one)' <<<"$ign_out"; then + _pass "--health: HEALTH_IGNORE_IDS suppresses findings" +else + _fail "--health: ignored guest still reported" +fi + +hfilter_out="$("$SCRIPT" --health --list --filter running --name '^vm')" +if grep -qE '^200 ' <<<"$hfilter_out" && ! grep -qE '^(100|101|201) ' <<<"$hfilter_out"; then + _pass "--health: --filter and --name apply" +else + _fail "--health: --filter/--name not applied" +fi + +_expect_rc "--health: pvesh failure exits 1" 1 env PMAN_MOCK_PVESH_FAIL=1 "$SCRIPT" --health + +if [[ "$(_level_for 84 85 95)" == "0" && "$(_level_for 85 85 95)" == "1" && "$(_level_for 95 85 95)" == "2" && + "$(_level_for 99 0 0)" == "0" && "$(_level_for - 85 95)" == "-" ]]; then + _pass "_level_for: thresholds, disabled levels and n/a" +else + _fail "_level_for: unexpected level" +fi + +parsed_rows="$(pvesh get /cluster/resources | _health_parse_resources mock-host)" +if [[ "$(printf '%s\n' "$parsed_rows" | cut -f1 | tr '\n' ' ')" == "100 101 200 201 " ]] && + grep -qP '^200\tVM\trunning\tvm-one\t5\t25\t-\t90061$' <<<"$parsed_rows"; then + _pass "_health_parse_resources: node/template filter and TSV fields" +else + _fail "_health_parse_resources: unexpected rows" +fi +if ! printf 'not json' | _health_parse_resources mock-host >/dev/null 2>&1; then + _pass "_health_parse_resources: invalid JSON fails" +else + _fail "_health_parse_resources: invalid JSON accepted" +fi + +menu_out="$(printf 'h\n\n' | "$SCRIPT" --once --no-clear 2>&1)" && menu_rc=0 || menu_rc=$? +if [[ "$menu_rc" == "0" ]] && grep -q 'HEALTH' <<<"$menu_out" && grep -q 'Total: 4 guests' <<<"$menu_out"; then + _pass "interactive: key 'h' shows the health overview" +else + _fail "interactive: key 'h' did not show the health overview (exit $menu_rc)" +fi + +status_out="$(do_action 100 CT status ct-one 2>&1)" +if grep -q 'Health: CPU 12% MEM 25% DISK 12% up 1h 0m \[OK\]' <<<"$status_out"; then + _pass "status action: prints a health line" +else + _fail "status action: health line missing" +fi + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From 8a52fff680ed9750071e159562a998c39db9f58d Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 18:45:31 +0200 Subject: [PATCH 3/8] feat(health): add --check engine with state, lock and exit codes - --check evaluates all local guests plus recent node tasks and alerts only on changes: new/escalated, improved, RESOLVED with duration. - CPU needs HEALTH_CPU_RUNS consecutive runs; mem/disk alert at once; a metric that is n/a keeps its previous state; a running guest that stops without a successful stop/shutdown/destroy/migrate task is WARN (latched), onboot=1 and stopped is CRIT. - Failed tasks since the last run are one-shot events (WARNINGS -> WARN, otherwise CRIT); the first run is a baseline; watermark by endtime+UPID. - Versioned TSV state (0600 in a 0700 dir), regex-validated on load, never sourced, written atomically; flock against parallel runs; INT/TERM and unexpected aborts exit 3. - Nagios-style summary; exit 0/1/2/3. --dry-run prints the composed message and writes neither state nor lock nor notifications. - _notify_all is a stub until the ntfy/sendmail senders land; with a channel configured it reports failure so state is kept for a retry. Co-Authored-By: Claude Opus 5 --- proxmox-manager.sh | 494 ++++++++++++++++++++++++++++++++++++++++++- tests/mock-bin/pvesh | 6 +- tests/run.sh | 158 +++++++++++++- 3 files changed, 651 insertions(+), 7 deletions(-) diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 654ed4e..40e8e90 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -55,6 +55,17 @@ HEALTH_ERR='' # reason for the last _health_load/_health_load_tasks fa HEALTH_ROWS=() # parsed guest rows (see _health_parse_resources) declare -A _HV_LEVEL=() # per-guest max health level (see _health_evaluate) declare -A _HV_FIND=() # per-guest findings (see _health_evaluate) +HEALTH_TASKS=() # parsed task rows (see _health_parse_tasks) +HEALTH_STATE_HEADER='# pman-health-state v1' +_CHECK_TMP='' # temp state file removed by the --check EXIT trap +_CHECK_DONE=0 # set right before --check exits normally +declare -A _S_LVL=() _S_SINCE=() _S_STREAK=() _S_RUN=() _S_TASK=() # previous --check state +_S_BASE=0 +_S_RUN_TS=0 +_S_TASK_TS=0 +declare -A _N_LVL=() _N_SINCE=() _N_STREAK=() _N_RUN=() _N_TASK=() # new --check state +_N_RUN_TS=0 +_N_TASK_TS=0 # ============================================================================= # COLORS (active only on a real TTY, or when NO_COLOR is unset) @@ -1333,6 +1344,26 @@ _health_load() { return 0 } +# _health_load_tasks — fetch and parse recent node tasks into HEALTH_TASKS. +# Requires HEALTH_NODE (set by _health_load). Returns 1 with HEALTH_ERR on failure. +_health_load_tasks() { + local raw parsed + HEALTH_TASKS=() + HEALTH_ERR='' + if ! raw="$(_health_fetch tasks "$HEALTH_NODE")"; then + HEALTH_ERR="pvesh get /nodes/${HEALTH_NODE}/tasks failed or timed out." + return 1 + fi + if ! parsed="$(printf '%s' "$raw" | _health_parse_tasks)"; then + HEALTH_ERR="Could not parse the pvesh task list." + return 1 + fi + if [[ -n "$parsed" ]]; then + mapfile -t HEALTH_TASKS <<<"$parsed" + fi + return 0 +} + # _guest_onboot ID TYPE — true when the guest config has "onboot: 1". _guest_onboot() { local id="$1" ty="$2" cfg='' @@ -1703,8 +1734,458 @@ _check_unknown() { exit 3 } +# _check_on_exit — EXIT trap of --check: remove the temp file and map any +# unexpected exit (e.g. a set -e abort) to 3 instead of a misleading OK/WARN. +_check_on_exit() { + local rc=$? + if [[ -n "$_CHECK_TMP" ]]; then + rm -f -- "$_CHECK_TMP" + _CHECK_TMP='' + fi + if ((_CHECK_DONE == 0 && rc != 3)); then + exit 3 + fi +} + +# _check_state_dir DIR — create DIR (0700) if needed; require a private, owned directory. +_check_state_dir() { + local dir="$1" + if [[ -L "$dir" ]]; then + err "HEALTH_STATE_DIR must not be a symlink." + return 1 + fi + if [[ ! -d "$dir" ]]; then + if ! (umask 077 && mkdir -p -- "$dir") 2>/dev/null; then + err "Could not create HEALTH_STATE_DIR ${dir}." + return 1 + fi + fi + if ! _owner_mode_ok "$dir" 077; then + err "HEALTH_STATE_DIR ${dir} must be owned by the current user with mode 0700." + return 1 + fi + return 0 +} + +# _state_load FILE — read the versioned TSV state into the _S_* globals. +# The file is parsed as data (never sourced); malformed lines are skipped and an +# unknown header resets to a fresh baseline. Returns 1 when FILE exists but is unsafe +# or unreadable. +# W run-epoch task-watermark T upid R vmid C check:vmid level since streak +_state_load() { + local file="$1" line first=1 bad=0 + local re_w=$'^W\t([0-9]{1,12})\t([0-9]{1,12})$' + local re_t=$'^T\t([A-Za-z0-9:@._!=-]{1,200})$' + local re_r=$'^R\t([0-9]{1,6})$' + local re_c=$'^C\t((cpu|mem|disk|down):[0-9]{1,6})\t([0-2])\t([0-9]{1,12})\t([0-9]{1,4})$' + _S_LVL=() + _S_SINCE=() + _S_STREAK=() + _S_RUN=() + _S_TASK=() + _S_BASE=0 + _S_RUN_TS=0 + _S_TASK_TS=0 + if [[ ! -e "$file" && ! -L "$file" ]]; then + return 0 + fi + if [[ -L "$file" || ! -f "$file" || ! -r "$file" ]]; then + err "State file ${file} must be a readable regular file (no symlink)." + return 1 + fi + if ! _owner_mode_ok "$file" 077; then + err "State file ${file} must be owned by the current user with mode 0600." + return 1 + fi + while IFS= read -r line || [[ -n "$line" ]]; do + if ((first == 1)); then + first=0 + if [[ "$line" != "$HEALTH_STATE_HEADER" ]]; then + _config_warn "Unknown state file format in ${file}; starting a new baseline." + return 0 + fi + continue + fi + if [[ "$line" =~ $re_c ]]; then + _S_LVL["${BASH_REMATCH[1]}"]="${BASH_REMATCH[3]}" + _S_SINCE["${BASH_REMATCH[1]}"]=$((10#${BASH_REMATCH[4]})) + _S_STREAK["${BASH_REMATCH[1]}"]=$((10#${BASH_REMATCH[5]})) + elif [[ "$line" =~ $re_r ]]; then + _S_RUN["${BASH_REMATCH[1]}"]=1 + elif [[ "$line" =~ $re_t ]]; then + _S_TASK["${BASH_REMATCH[1]}"]=1 + elif [[ "$line" =~ $re_w ]]; then + _S_RUN_TS=$((10#${BASH_REMATCH[1]})) + _S_TASK_TS=$((10#${BASH_REMATCH[2]})) + _S_BASE=1 + else + bad=$((bad + 1)) + fi + done <"$file" + if ((bad > 0)); then + _config_warn "Ignored ${bad} malformed line(s) in ${file}." + fi + return 0 +} + +# _state_save DIR FILE — write the _N_* globals atomically (mktemp + mv, mode 0600). +_state_save() { + local dir="$1" file="$2" k + _CHECK_TMP="$(umask 077 && mktemp -p "$dir" health.state.XXXXXX)" || return 1 + { + printf '%s\n' "$HEALTH_STATE_HEADER" + printf 'W\t%s\t%s\n' "$_N_RUN_TS" "$_N_TASK_TS" + for k in "${!_N_TASK[@]}"; do + printf 'T\t%s\n' "$k" + done + for k in "${!_N_RUN[@]}"; do + printf 'R\t%s\n' "$k" + done + for k in "${!_N_LVL[@]}"; do + printf 'C\t%s\t%s\t%s\t%s\n' "$k" "${_N_LVL[$k]}" "${_N_SINCE[$k]:-0}" "${_N_STREAK[$k]:-0}" + done + } >"$_CHECK_TMP" || return 1 + chmod 600 -- "$_CHECK_TMP" || return 1 + mv -f -- "$_CHECK_TMP" "$file" || return 1 + _CHECK_TMP='' + return 0 +} + +# _check_process_tasks — scan HEALTH_TASKS (uses run_check's "events" and "excused"). +# Advances the task watermark (_N_TASK_TS/_N_TASK), appends failed tasks that finished +# since the last run to "events" (level TAB text) and marks guests with a successful or +# running stop/shutdown/destroy/migrate task in "excused". The first run is a baseline. +_check_process_tasks() { + local row t_end t_upid t_type t_id t_status is_new lvl k + _N_TASK_TS=$_S_TASK_TS + _N_TASK=() + for k in "${!_S_TASK[@]}"; do + _N_TASK["$k"]=1 + done + for row in "${HEALTH_TASKS[@]}"; do + IFS=$'\t' read -r t_end t_upid t_type t_id t_status <<<"$row" + [[ "$t_end" =~ ^[0-9]{1,12}$ && -n "$t_upid" ]] || continue + t_end=$((10#$t_end)) + case "$t_type" in + qmstop | qmshutdown | vzstop | vzshutdown | qmdestroy | vzdestroy | qmigrate | vzmigrate) + if ((t_end == 0)) || { [[ "$t_status" == "OK" ]] && ((t_end >= _S_RUN_TS)); }; then + excused["$t_id"]=1 + fi + ;; + esac + if ((t_end == 0)); then + continue # still running + fi + is_new=0 + if ((t_end > _S_TASK_TS)) || { ((t_end == _S_TASK_TS)) && [[ -z "${_S_TASK[$t_upid]:-}" ]]; }; then + is_new=1 + fi + if ((t_end > _N_TASK_TS)); then + _N_TASK_TS=$t_end + _N_TASK=() + fi + if ((t_end == _N_TASK_TS)); then + _N_TASK["$t_upid"]=1 + fi + if ((is_new == 1 && _S_BASE == 1)) && [[ "$t_status" != "OK" ]] && ! _health_ignored "$t_id"; then + lvl=2 + [[ "$t_status" == WARNINGS* ]] && lvl=1 + k="task ${t_type}" + [[ "$t_id" != "-" ]] && k+=" ${t_id}" + if ((lvl == 2)); then + events+=("${lvl}"$'\t'"${k} failed: ${t_status}") + else + events+=("${lvl}"$'\t'"${k} finished with ${t_status}") + fi + fi + done + return 0 +} + +# _check_process_states NOW — derive the new check state (_N_LVL/_N_SINCE/_N_STREAK/_N_RUN) +# from _health_checks and the previous state. Uses run_check's "excused" and fills its +# "seen", "vals" and "labels" arrays. +# cpu: alerts after HEALTH_CPU_RUNS consecutive runs; mem/disk: immediately; +# n/a metric: previous state kept; down: onboot CRIT, or WARN (latched in the streak +# field) when a guest that was running last time stopped without a matching task. +_check_process_states() { + local now="$1" checks row cid chk clvl cval key prev pstreak psince lvl streak + local id ty st nm cpu mem disk up + _N_LVL=() + _N_SINCE=() + _N_STREAK=() + _N_RUN=() + for row in "${HEALTH_ROWS[@]}"; do + IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" + [[ -z "$id" ]] && continue + labels["$id"]="${ty} ${id} (${nm})" + if [[ "$st" == "running" ]] && ! _health_ignored "$id"; then + _N_RUN["$id"]=1 + fi + done + checks="$(_health_checks)" + while IFS=$'\t' read -r cid _ chk clvl cval _; do + [[ -z "$cid" ]] && continue + key="${chk}:${cid}" + seen["$key"]=1 + vals["$key"]="$cval" + prev="${_S_LVL[$key]:-0}" + pstreak="${_S_STREAK[$key]:-0}" + psince="${_S_SINCE[$key]:-0}" + if [[ "$clvl" == "-" ]]; then + # Metric not available (guest not running): keep the previous state. + if [[ -n "${_S_LVL[$key]:-}" ]]; then + _N_LVL["$key"]=$prev + _N_SINCE["$key"]=$psince + _N_STREAK["$key"]=$pstreak + fi + continue + fi + lvl=$clvl + streak=0 + case "$chk" in + cpu) + if ((clvl > 0)); then + streak=$((pstreak + 1)) + if ((streak > 999)); then + streak=999 + fi + if ((streak < HEALTH_CPU_RUNS && prev == 0)); then + lvl=0 + fi + fi + ;; + down) + # The streak field latches "stopped unexpectedly" until the guest runs again. + if [[ "$cval" == "stopped" ]]; then + if ((pstreak > 0)); then + streak=1 + elif ((_S_BASE == 1)) && [[ -n "${_S_RUN[$cid]:-}" && -z "${excused[$cid]:-}" ]]; then + streak=1 + fi + if ((streak == 1 && lvl < 1)); then + lvl=1 + fi + fi + ;; + esac + if ((lvl > 0 || streak > 0)); then + _N_LVL["$key"]=$lvl + _N_STREAK["$key"]=$streak + _N_SINCE["$key"]=0 + if ((lvl > 0 && prev > 0 && psince > 0)); then + _N_SINCE["$key"]=$psince + elif ((lvl > 0)); then + _N_SINCE["$key"]=$now + fi + fi + done <<<"$checks" + return 0 +} + +# _resolved_text CHECK VALUE — description of a check that returned to OK. +_resolved_text() { + local chk="$1" val="$2" + case "$chk" in + cpu) printf 'CPU back to normal (%s)' "$(_pct_text "$val")" ;; + mem) printf 'memory back to normal (%s)' "$(_pct_text "$val")" ;; + disk) printf 'disk back to normal (%s)' "$(_pct_text "$val")" ;; + down) + if [[ "$val" == "running" ]]; then + printf 'running again' + else + printf 'down alert cleared (status %s)' "${val:-unknown}" + fi + ;; + *) printf '%s OK' "$chk" ;; + esac +} + +# _compose_message NODE LEVEL N_NEW N_IMPROVED N_RESOLVED TOTALS LINE... — build one +# combined notification in MSG_TITLE (ASCII), MSG_BODY (<= ~3500 bytes) and MSG_PRIORITY. +_compose_message() { + local node="$1" lvl="$2" n_new="$3" n_imp="$4" n_res="$5" totals="$6" + shift 6 + local word parts='' line body='' dropped=0 max=3500 + if ((lvl > 0)); then + word="$(_level_name "$lvl")" + elif ((n_res > 0 && n_imp == 0)); then + word="RESOLVED" + else + word="IMPROVED" + fi + if ((n_new > 0)); then + parts+="${n_new} new, " + fi + if ((n_imp > 0)); then + parts+="${n_imp} improved, " + fi + if ((n_res > 0)); then + parts+="${n_res} resolved, " + fi + parts="${parts%, }" + MSG_TITLE="pman ${node}: ${word}" + if [[ -n "$parts" ]]; then + MSG_TITLE+=" (${parts})" + fi + case "$lvl" in + 2) MSG_PRIORITY="urgent" ;; + 1) MSG_PRIORITY="high" ;; + *) MSG_PRIORITY="low" ;; + esac + for line in "$@"; do + if ((${#body} + ${#line} + 1 > max)); then + dropped=$((dropped + 1)) + continue + fi + body+="${line}"$'\n' + done + if ((dropped > 0)); then + body+="... ${dropped} more line(s) omitted"$'\n' + fi + MSG_BODY="${body}"$'\n'"${totals}" + return 0 +} + +# run_check — --check: evaluate all local guests and recent tasks, alert on changes. +# Exit code: 0 OK, 1 WARN, 2 CRIT, 3 UNKNOWN (config, lock, state or pvesh failure). run_check() { - _check_unknown "--check is not implemented yet." + local dir="${HEALTH_STATE_DIR%/}" file lock_fd now old_umask + local -A excused=() seen=() vals=() labels=() + local -a events=() keys=() cur_lines=() new_lines=() imp_lines=() res_lines=() ev_lines=() + [[ -z "$dir" ]] && dir="/" + file="${dir%/}/health.state" + trap 'exit 3' INT TERM + trap '_check_on_exit' EXIT + now="$(date +%s)" + + if ((DRY_RUN == 0)); then + _check_state_dir "$dir" || _check_unknown "State directory ${dir} is not usable." + have flock || _check_unknown "'flock' not found (util-linux)." + old_umask="$(umask)" + umask 077 + if ! { exec {lock_fd}>"${dir%/}/check.lock"; } 2>/dev/null; then + _check_unknown "Could not open the lock file in ${dir}." + fi + umask "$old_umask" + flock -n "$lock_fd" || _check_unknown "Another --check run is still active (lock held)." + fi + if ! _state_load "$file"; then + ((DRY_RUN == 1)) || _check_unknown "Could not read the state file ${file}." + fi + _health_load || _check_unknown "$HEALTH_ERR" + _health_load_tasks || _check_unknown "$HEALTH_ERR" + + _check_process_tasks + _check_process_states "$now" + + local key chk id prev new label since dur text ev ev_lvl rc=0 lvl_new=0 + local n_new=0 n_imp=0 n_res=0 n_warn=0 n_crit=0 n_guests=0 + n_guests=${#HEALTH_ROWS[@]} + mapfile -t keys < <(printf '%s\n' "${!seen[@]}" "${!_S_LVL[@]}" | awk 'NF' | sort -t: -k2,2n -k1,1 -u) + for key in "${keys[@]}"; do + chk="${key%%:*}" + id="${key#*:}" + prev="${_S_LVL[$key]:-0}" + new="${_N_LVL[$key]:-0}" + label="${labels[$id]:-guest ${id}}" + if [[ -z "${seen[$key]:-}" ]]; then + # Guest vanished or is ignored now: resolve (silently when ignored). + if _health_ignored "$id"; then + continue + fi + new=0 + vals["$key"]="gone" + fi + text="${label}: $(_health_finding "$chk" "$new" "${vals[$key]:-}")" + if ((new > 0)); then + cur_lines+=("[$(_level_name "$new")] ${text}") + if ((new == 2)); then + n_crit=$((n_crit + 1)) + else + n_warn=$((n_warn + 1)) + fi + if ((new > rc)); then + rc=$new + fi + fi + if ((new > prev)); then + new_lines+=("[$(_level_name "$new")] ${text}") + n_new=$((n_new + 1)) + if ((new > lvl_new)); then + lvl_new=$new + fi + elif ((new > 0 && new < prev)); then + imp_lines+=("[$(_level_name "$new")] ${text} (improved from $(_level_name "$prev"))") + n_imp=$((n_imp + 1)) + elif ((new == 0 && prev > 0)); then + since="${_S_SINCE[$key]:-0}" + dur='' + if ((since > 0 && now >= since)); then + dur=" after $(_fmt_duration $((now - since)))" + fi + if [[ "${vals[$key]:-}" == "gone" ]]; then + res_lines+=("[RESOLVED] ${label}: no longer present${dur}") + else + res_lines+=("[RESOLVED] ${label}: $(_resolved_text "$chk" "${vals[$key]:-}")${dur}") + fi + n_res=$((n_res + 1)) + fi + done + for ev in "${events[@]}"; do + ev_lvl="${ev%%$'\t'*}" + ev_lines+=("[$(_level_name "$ev_lvl")] ${ev#*$'\t'}") + if ((ev_lvl > rc)); then + rc=$ev_lvl + fi + if ((ev_lvl > lvl_new)); then + lvl_new=$ev_lvl + fi + done + + local state_word summary changes + case "$rc" in + 2) state_word="CRITICAL" ;; + 1) state_word="WARNING" ;; + *) state_word="OK" ;; + esac + printf -v summary '%s critical, %s warning, %s task event(s); %s guests on %s' \ + "$n_crit" "$n_warn" "${#events[@]}" "$n_guests" "$HEALTH_NODE" + printf 'PMAN %s - %s\n' "$state_word" "$summary" + for text in "${cur_lines[@]}" "${ev_lines[@]}" "${res_lines[@]}"; do + printf '%s\n' "$text" + done + log "INFO" "health check: ${state_word} - ${summary}" + + changes=$((n_new + n_imp + n_res + ${#events[@]})) + if ((changes > 0)); then + _compose_message "$HEALTH_NODE" "$lvl_new" $((n_new + ${#events[@]})) "$n_imp" "$n_res" \ + "Now: ${summary}" "${new_lines[@]}" "${ev_lines[@]}" "${imp_lines[@]}" "${res_lines[@]}" + fi + + if ((DRY_RUN == 1)); then + printf '\n--- notification (dry-run: nothing sent, state unchanged) ---\n' + if ((changes > 0)); then + printf 'Title: %s\nPriority: %s\n\n%s\n' "$MSG_TITLE" "$MSG_PRIORITY" "$MSG_BODY" + else + printf 'No changes since the last run; no notification.\n' + fi + _CHECK_DONE=1 + exit "$rc" + fi + + if ((changes > 0)); then + log "INFO" "health alert: ${MSG_TITLE}" + if ! _notify_all; then + err "All notification channels failed; state not saved so the next run retries." + _CHECK_DONE=1 + exit "$rc" + fi + fi + _N_RUN_TS=$now + _state_save "$dir" "$file" || _check_unknown "Could not write the state file ${file}." + _CHECK_DONE=1 + exit "$rc" } # ============================================================================= @@ -1717,14 +2198,17 @@ _notify_channels_configured() { } # _notify_all — deliver MSG_TITLE/MSG_BODY/MSG_PRIORITY through every configured channel. -# Returns 1 only when channels are configured and all of them failed. -# Channel senders (ntfy, sendmail) are added in a later work package. +# Returns 1 only when channels are configured and all of them failed; with no channel +# configured there is nothing to deliver and it returns 0. +# Channel senders (ntfy, sendmail) are added in a later work package; until then a +# configured channel counts as failed so --check keeps its state and retries later. _notify_all() { if ! _notify_channels_configured; then return 0 fi - log "WARN" "Notification delivery not implemented yet; '${MSG_TITLE}' (${MSG_PRIORITY}, ${#MSG_BODY} bytes) not sent." - return 0 + err "Notification delivery is not implemented yet." + log "WARN" "Notification not sent: '${MSG_TITLE}' (${MSG_PRIORITY}, ${#MSG_BODY} bytes)." + return 1 } # run_test_notify — send a test message through all configured channels. diff --git a/tests/mock-bin/pvesh b/tests/mock-bin/pvesh index 211daa9..a40f1a2 100755 --- a/tests/mock-bin/pvesh +++ b/tests/mock-bin/pvesh @@ -2,9 +2,13 @@ # Mock for `pvesh get ... --output-format json`. # PMAN_MOCK_RESOURCES / PMAN_MOCK_TASKS: fixture files replacing the inline defaults. # The token @NOW@ in a fixture is replaced with the current epoch. -# PMAN_MOCK_PVESH_FAIL=1 makes every call fail. +# PMAN_MOCK_PVESH_FAIL=1 makes every call fail; PMAN_MOCK_PVESH_SLEEP=N delays each call. set -euo pipefail +if [[ -n "${PMAN_MOCK_PVESH_SLEEP:-}" ]]; then + sleep "$PMAN_MOCK_PVESH_SLEEP" +fi + if [[ "${PMAN_MOCK_PVESH_FAIL:-0}" == "1" ]]; then echo "mock pvesh: simulated failure" >&2 exit 1 diff --git a/tests/run.sh b/tests/run.sh index 976f14b..9ca980c 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -383,7 +383,7 @@ _expect_rc "health config: token file with mode 0644 makes --check exit 3" 3 \ printf '%s\n' 'NTFY_URL=ftp://ntfy.example.invalid/topic' >"$HOME/.pmanrc" _expect_rc "health config: non-http NTFY_URL makes --check exit 3" 3 \ env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check -printf '%s\n' 'HEALTH_MAIL_TO=root@localhost,bad address' >"$HOME/.pmanrc" +printf '%s\n' 'HEALTH_MAIL_TO="root@localhost,bad address"' >"$HOME/.pmanrc" _expect_rc "health config: invalid HEALTH_MAIL_TO makes --check exit 3" 3 \ env HEALTH_STATE_DIR="$(_new_state_dir)" "$SCRIPT" --check rm -f "$HOME/.pmanrc" @@ -485,6 +485,162 @@ else _fail "status action: health line missing" fi +# --------------------------------------------------------------------------- +# Health check engine: --check / --dry-run (notifications are not sent here) +# --------------------------------------------------------------------------- +# _chk DIR [ENV=VAL...] [-- ARGS] — run --check with HEALTH_STATE_DIR=DIR; sets chk_out/chk_rc. +_chk() { + local dir="$1" + shift + local -a envs=() args=() + while (($# > 0)) && [[ "$1" != "--" ]]; do + envs+=("$1") + shift + done + (($# > 0)) && shift + args=("$@") + chk_rc=0 + chk_out="$(env HEALTH_STATE_DIR="$dir" "${envs[@]}" "$SCRIPT" --check "${args[@]}" 2>/dev/null)" || chk_rc=$? +} + +# _expect_chk LABEL RC PATTERN — assert last _chk exit code and a fixed-string match. +_expect_chk() { + local label="$1" rc="$2" pattern="$3" + if [[ "$chk_rc" == "$rc" ]] && grep -qF -- "$pattern" <<<"$chk_out"; then + _pass "$label" + else + _fail "$label (exit $chk_rc, expected $rc; output: ${chk_out//$'\n'/ | })" + fi +} + +HOT="PMAN_MOCK_RESOURCES=$FIXTURES/resources-hot.json" +CPU_HOT="PMAN_MOCK_RESOURCES=$FIXTURES/resources-cpu-hot.json" +STOPPED_200="PMAN_MOCK_RESOURCES=$FIXTURES/resources-200-stopped.json" + +sd="$(_new_state_dir)" +_chk "$sd" +_expect_chk "--check: all OK exits 0 with Nagios summary" 0 "PMAN OK - 0 critical, 0 warning" +if [[ "$(stat -c '%a' "$sd/health.state")" == "600" && "$(stat -c '%a' "$sd/check.lock")" == "600" && + "$(head -1 "$sd/health.state")" == "# pman-health-state v1" ]]; then + _pass "--check: state and lock files are private (0600) and versioned" +else + _fail "--check: state/lock file mode or header wrong" +fi +_chk "$sd" "$HOT" +_expect_chk "--check: memory CRIT alerts immediately (exit 2)" 2 "[CRIT] CT 100 (ct-one): memory 96% (>= 95%)" +if ! grep -q 'CPU' <<<"$chk_out"; then + _pass "--check: single CPU spike does not alert" +else + _fail "--check: CPU alerted on the first high run" +fi +_chk "$sd" +_expect_chk "--check: recovery reports RESOLVED (exit 0)" 0 "[RESOLVED] CT 100 (ct-one): memory back to normal (25%)" + +sd="$(_new_state_dir)" +_chk "$sd" "$CPU_HOT" +cpu1="$chk_rc" +_chk "$sd" "$CPU_HOT" +cpu2="$chk_rc" +_chk "$sd" "$CPU_HOT" +if [[ "$cpu1$cpu2" == "00" ]]; then + _expect_chk "--check: CPU alerts after HEALTH_CPU_RUNS=3 consecutive runs" 1 "[WARN] VM 200 (vm-one): CPU 90% (>= 85%)" +else + _fail "--check: CPU alerted before 3 runs (exits $cpu1 $cpu2)" +fi + +sd="$(_new_state_dir)" +_chk "$sd" PMAN_MOCK_ONBOOT_IDS=101 +_expect_chk "--check: stopped guest with onboot=1 is CRIT" 2 "[CRIT] CT 101 (ct-fallback): stopped although onboot=1" + +sd="$(_new_state_dir)" +_chk "$sd" +_chk "$sd" "$STOPPED_200" +_expect_chk "--check: running guest stopped without task is WARN" 1 "[WARN] VM 200 (vm-one): stopped unexpectedly" +_chk "$sd" "$STOPPED_200" +_expect_chk "--check: unexpected stop stays latched while stopped" 1 "stopped unexpectedly" +sd="$(_new_state_dir)" +_chk "$sd" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-shutdown-200.json" +_expect_chk "--check: stop with a successful shutdown task is OK" 0 "PMAN OK" + +sd="$(_new_state_dir)" +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-old.json" +_expect_chk "--check: first run is a task baseline (old failures ignored)" 0 "0 task event(s)" +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-new.json" +if [[ "$chk_rc" == "2" ]] && grep -qF '[CRIT] task qmstart 201 failed: start failed' <<<"$chk_out" && + grep -qF '[WARN] task vzdump 100 finished with WARNINGS: 1' <<<"$chk_out" && ! grep -q 'job errors' <<<"$chk_out"; then + _pass "--check: new failed tasks alert once (CRIT/WARN), old ones stay silent" +else + _fail "--check: task events wrong (exit $chk_rc)" +fi +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-new.json" +_expect_chk "--check: task events are deduplicated by watermark" 0 "0 task event(s)" + +sd="$(_new_state_dir)" +_chk "$sd" "$HOT" HEALTH_IGNORE_IDS=100 +_expect_chk "--check: HEALTH_IGNORE_IDS skips a guest" 0 "PMAN OK" + +sd="$(_new_state_dir)" +_chk "$sd" PMAN_MOCK_PVESH_FAIL=1 +_expect_chk "--check: pvesh failure exits 3 (UNKNOWN)" 3 "PMAN UNKNOWN" +lock_rc=0 +( + flock -n 9 + HEALTH_STATE_DIR="$sd" "$SCRIPT" --check >/dev/null 2>&1 +) 9>"$sd/check.lock" || lock_rc=$? +if [[ "$lock_rc" == "3" ]]; then + _pass "--check: concurrent run (lock held) exits 3" +else + _fail "--check: lock not enforced (exit $lock_rc)" +fi +chmod 755 "$sd" +_chk "$sd" +_expect_chk "--check: non-private state directory exits 3" 3 "PMAN UNKNOWN" + +sd="$(_new_state_dir)" +_chk "$sd" "$HOT" -- --dry-run +if [[ "$chk_rc" == "2" ]] && grep -qF 'Title: pman mock-host: CRIT (2 new)' <<<"$chk_out" && + grep -qF 'Priority: urgent' <<<"$chk_out" && [[ ! -e "$sd/health.state" && ! -e "$sd/check.lock" ]]; then + _pass "--check --dry-run: prints the message and writes no state" +else + _fail "--check --dry-run: unexpected output or state written (exit $chk_rc)" +fi + +sd="$(_new_state_dir)" +probe="$TEST_TMP/state-probe" +printf '%s\n' '# pman-health-state v1' "W 1 1" "C mem:100 2 1 0" \ + "X \$(touch $probe)" "C mem:1;touch $probe 2 1 0" >"$sd/health.state" +chmod 600 "$sd/health.state" +_chk "$sd" +if [[ "$chk_rc" == "0" && ! -e "$probe" ]] && grep -qF '[RESOLVED] CT 100 (ct-one): memory back to normal' <<<"$chk_out" && + ! grep -q 'touch' "$sd/health.state"; then + _pass "--check: state file parsed as data, malformed lines dropped" +else + _fail "--check: state file handling unsafe or wrong (exit $chk_rc)" +fi + +sd="$(_new_state_dir)" +HEALTH_STATE_DIR="$sd" PMAN_MOCK_PVESH_SLEEP=1 "$SCRIPT" --check >/dev/null 2>&1 & +term_pid=$! +sleep 0.3 +kill -TERM "$term_pid" 2>/dev/null || true +term_rc=0 +wait "$term_pid" || term_rc=$? +if [[ "$term_rc" == "3" ]]; then + _pass "--check: SIGTERM exits 3 instead of 0" +else + _fail "--check: SIGTERM exit code $term_rc (expected 3)" +fi + +cap_lines=() +for i in $(seq 1 200); do cap_lines+=("[WARN] VM $i (some-guest-name): memory 91% (>= 90%)"); done +_compose_message mock-host 1 200 0 0 "Now: totals" "${cap_lines[@]}" +if ((${#MSG_BODY} <= 3600)) && [[ "$MSG_BODY" == *"more line(s) omitted"* && "$MSG_TITLE" == "pman mock-host: WARN (200 new)" && "$MSG_PRIORITY" == "high" ]]; then + _pass "_compose_message: title, priority and ~3500 byte cap" +else + _fail "_compose_message: cap or title wrong (${#MSG_BODY} bytes, '$MSG_TITLE')" +fi + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From 005a8e91a92d3867abadad1f1cb34d2f8f19d982 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 18:51:28 +0200 Subject: [PATCH 4/8] feat(health): send alerts via ntfy and e-mail - _notify_ntfy: curl with URL, Title/Priority/Tags and the optional Bearer token passed only via the stdin config; body from a private temp file; no redirects; only the host is logged. A token is never sent over plain http (channel fails instead). - _notify_mail: local sendmail (fallback /usr/sbin/sendmail) with validated single-line headers, UTF-8 body and Auto-Submitted. - _notify_all tries every configured channel and fails only when all of them fail, so --check keeps its state and retries next run. - --test-notify sends a real test message. - A running or successful vzdump task explains a stopped guest (backup stop mode). - Tests: curl/sendmail mocks, delivery, dedup, recovery, partial and total failure, token handling, header injection, vzdump. Co-Authored-By: Claude Opus 5 --- proxmox-manager.sh | 163 +++++++++++++++++++++++++-- tests/fixtures/tasks-vzdump-200.json | 4 + tests/mock-bin/curl | 17 +++ tests/mock-bin/sendmail | 10 ++ tests/run.sh | 138 +++++++++++++++++++++++ 5 files changed, 320 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/tasks-vzdump-200.json create mode 100755 tests/mock-bin/curl create mode 100755 tests/mock-bin/sendmail diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 40e8e90..df284e3 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -59,6 +59,7 @@ HEALTH_TASKS=() # parsed task rows (see _health_parse_tasks) HEALTH_STATE_HEADER='# pman-health-state v1' _CHECK_TMP='' # temp state file removed by the --check EXIT trap _CHECK_DONE=0 # set right before --check exits normally +_NOTIFY_TMP='' # temp body file of _notify_ntfy declare -A _S_LVL=() _S_SINCE=() _S_STREAK=() _S_RUN=() _S_TASK=() # previous --check state _S_BASE=0 _S_RUN_TS=0 @@ -1742,6 +1743,10 @@ _check_on_exit() { rm -f -- "$_CHECK_TMP" _CHECK_TMP='' fi + if [[ -n "$_NOTIFY_TMP" ]]; then + rm -f -- "$_NOTIFY_TMP" + _NOTIFY_TMP='' + fi if ((_CHECK_DONE == 0 && rc != 3)); then exit 3 fi @@ -1872,6 +1877,15 @@ _check_process_tasks() { excused["$t_id"]=1 fi ;; + vzdump) + # Backups in stop mode stop the guest; a running multi-guest job (no id) excuses all. + if ((t_end == 0)) || { [[ "$t_status" == "OK" ]] && ((t_end >= _S_RUN_TS)); }; then + excused["$t_id"]=1 + fi + if ((t_end == 0)) && [[ "$t_id" == "-" ]]; then + excused[all]=1 + fi + ;; esac if ((t_end == 0)); then continue # still running @@ -1960,7 +1974,7 @@ _check_process_states() { if [[ "$cval" == "stopped" ]]; then if ((pstreak > 0)); then streak=1 - elif ((_S_BASE == 1)) && [[ -n "${_S_RUN[$cid]:-}" && -z "${excused[$cid]:-}" ]]; then + elif ((_S_BASE == 1)) && [[ -n "${_S_RUN[$cid]:-}" && -z "${excused[$cid]:-}${excused[all]:-}" ]]; then streak=1 fi if ((streak == 1 && lvl < 1)); then @@ -2197,18 +2211,143 @@ _notify_channels_configured() { [[ -n "$NTFY_URL" || -n "$HEALTH_MAIL_TO" ]] } +# _notify_title — MSG_TITLE reduced to one line of printable ASCII (header-safe). +_notify_title() { + local t="${MSG_TITLE//[$'\r\n\t']/ }" + t="${t//[^ -~]/?}" + printf '%s' "${t:0:200}" +} + +# _notify_warn MESSAGE — channel failure: warning on stderr (keeps --check stdout clean) and log. +_notify_warn() { + warn "$*" >&2 +} + +# _notify_ntfy — send the message to NTFY_URL. URL, headers and the optional token are +# passed to curl via its stdin config, never in argv; the body comes from a private temp +# file. Only the host is logged, never the topic path or the token. +_notify_ntfy() { + local host token='' tag title cfg rc=0 out + host="${NTFY_URL#*://}" + host="${host%%/*}" + if ! have curl; then + _notify_warn "ntfy: 'curl' not found." + return 1 + fi + if [[ -n "$NTFY_TOKEN_FILE" ]]; then + if [[ "$NTFY_URL" != https://* ]]; then + _notify_warn "ntfy: refusing to send the access token over plain http to ${host}; use https." + return 1 + fi + if ! IFS= read -r token <"$NTFY_TOKEN_FILE" && [[ -z "$token" ]]; then + token='' + fi + if [[ ! "$token" =~ ^[A-Za-z0-9_.-]{1,256}$ ]]; then + _notify_warn "ntfy: token in NTFY_TOKEN_FILE is empty or has unsupported characters." + return 1 + fi + fi + case "$MSG_PRIORITY" in + urgent) tag="rotating_light" ;; + high) tag="warning" ;; + *) tag="white_check_mark" ;; + esac + title="$(_notify_title)" + title="${title//\\/\\\\}" + title="${title//\"/\\\"}" + _NOTIFY_TMP="$(umask 077 && mktemp -p "${TMPDIR:-/tmp}" pman-ntfy.XXXXXX)" || { + _notify_warn "ntfy: could not create a temporary file." + return 1 + } + printf '%s\n' "$MSG_BODY" >"$_NOTIFY_TMP" + cfg="url = \"${NTFY_URL}\""$'\n' + cfg+="header = \"Title: ${title}\""$'\n' + cfg+="header = \"Priority: ${MSG_PRIORITY:-default}\""$'\n' + cfg+="header = \"Tags: ${tag}\""$'\n' + if [[ -n "$token" ]]; then + cfg+="header = \"Authorization: Bearer ${token}\""$'\n' + fi + out="$(curl -fsS --max-time 10 --retry 2 --proto '=http,https' -o /dev/null \ + --data-binary "@${_NOTIFY_TMP}" --config - <<<"$cfg" 2>&1)" || rc=$? + rm -f -- "$_NOTIFY_TMP" + _NOTIFY_TMP='' + if ((rc != 0)); then + # curl's message may contain the topic URL; keep it out of the terminal and the log. + _notify_warn "ntfy: delivery to ${host} failed (curl exit ${rc})." + return 1 + fi + log "INFO" "ntfy: sent to ${host}" + return 0 +} + +# _notify_mail — send the message to HEALTH_MAIL_TO via the local sendmail (postfix etc.). +# Header values are validated single-line strings, so no header injection is possible. +_notify_mail() { + local sm addr to='' subject rc=0 + local -a addrs=() + sm="$(command -v sendmail 2>/dev/null || true)" + if [[ -z "$sm" && -x /usr/sbin/sendmail ]]; then + sm="/usr/sbin/sendmail" + fi + if [[ -z "$sm" ]]; then + _notify_warn "mail: 'sendmail' not found (install postfix or another MTA)." + return 1 + fi + IFS=',' read -r -a addrs <<<"$HEALTH_MAIL_TO" + for addr in "${addrs[@]}"; do + if ! _valid_mail_addr "$addr"; then + _notify_warn "mail: invalid recipient address in HEALTH_MAIL_TO." + return 1 + fi + to+="${to:+, }${addr}" + done + if [[ -z "$to" ]] || { [[ -n "$HEALTH_MAIL_FROM" ]] && ! _valid_mail_addr "$HEALTH_MAIL_FROM"; }; then + _notify_warn "mail: invalid HEALTH_MAIL_TO/HEALTH_MAIL_FROM." + return 1 + fi + subject="$(_notify_title)" + { + printf 'To: %s\n' "$to" + if [[ -n "$HEALTH_MAIL_FROM" ]]; then + printf 'From: %s\n' "$HEALTH_MAIL_FROM" + fi + printf 'Subject: %s\n' "$subject" + printf 'Date: %s\n' "$(date -R)" + printf 'MIME-Version: 1.0\n' + printf 'Content-Type: text/plain; charset=UTF-8\n' + printf 'Content-Transfer-Encoding: 8bit\n' + printf 'Auto-Submitted: auto-generated\n' + printf '\n%s\n' "$MSG_BODY" + } | timeout 30 "$sm" -t -oi >/dev/null 2>&1 || rc=$? + if ((rc != 0)); then + _notify_warn "mail: sendmail failed (exit ${rc})." + return 1 + fi + log "INFO" "mail: sent to ${#addrs[@]} recipient(s)" + return 0 +} + # _notify_all — deliver MSG_TITLE/MSG_BODY/MSG_PRIORITY through every configured channel. -# Returns 1 only when channels are configured and all of them failed; with no channel -# configured there is nothing to deliver and it returns 0. -# Channel senders (ntfy, sendmail) are added in a later work package; until then a -# configured channel counts as failed so --check keeps its state and retries later. +# Returns 0 when no channel is configured or at least one succeeded, 1 when all failed. _notify_all() { - if ! _notify_channels_configured; then - return 0 + local configured=0 delivered=0 + if [[ -n "$NTFY_URL" ]]; then + configured=$((configured + 1)) + if _notify_ntfy; then + delivered=$((delivered + 1)) + fi fi - err "Notification delivery is not implemented yet." - log "WARN" "Notification not sent: '${MSG_TITLE}' (${MSG_PRIORITY}, ${#MSG_BODY} bytes)." - return 1 + if [[ -n "$HEALTH_MAIL_TO" ]]; then + configured=$((configured + 1)) + if _notify_mail; then + delivered=$((delivered + 1)) + fi + fi + if ((configured > 0 && delivered == 0)); then + log "WARN" "Notification failed on all ${configured} channel(s): ${MSG_TITLE}" + return 1 + fi + return 0 } # run_test_notify — send a test message through all configured channels. @@ -2220,10 +2359,10 @@ run_test_notify() { local node node="$(_local_node || printf 'unknown')" MSG_TITLE="pman ${node}: test notification" - MSG_BODY="Test message from proxmox-manager on ${node}." + MSG_BODY="Test message from proxmox-manager on ${node}. Alerts from --check will arrive here." MSG_PRIORITY="low" if _notify_all; then - ok "Test notification handed to all configured channels." + ok "Test notification sent (at least one channel succeeded)." exit 0 fi err "Test notification failed on all channels." diff --git a/tests/fixtures/tasks-vzdump-200.json b/tests/fixtures/tasks-vzdump-200.json new file mode 100644 index 0000000..71914f3 --- /dev/null +++ b/tests/fixtures/tasks-vzdump-200.json @@ -0,0 +1,4 @@ +[ +{"upid":"UPID:mock-host:00001000:00002000:65000000:qmstart:200:root@pam:","node":"mock-host","type":"qmstart","id":"200","user":"root@pam","starttime":1700000000,"endtime":1700000005,"status":"OK"}, +{"upid":"UPID:mock-host:00001200:00002200:65000200:vzdump:200:root@pam:","node":"mock-host","type":"vzdump","id":"200","user":"root@pam","starttime":@NOW@} +] diff --git a/tests/mock-bin/curl b/tests/mock-bin/curl new file mode 100755 index 0000000..6d908d3 --- /dev/null +++ b/tests/mock-bin/curl @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Mock curl: never touches the network. +# Appends argv to $PMAN_MOCK_CURL_LOG, the stdin config to ${PMAN_MOCK_CURL_LOG}.cfg and +# every @file body to ${PMAN_MOCK_CURL_LOG}.body; exits ${PMAN_MOCK_CURL_RC:-0}. +set -euo pipefail +log="${PMAN_MOCK_CURL_LOG:-}" +cfg="$(cat)" +if [[ -n "$log" ]]; then + printf '%s\n' "$*" >>"$log" + printf '%s\n---\n' "$cfg" >>"${log}.cfg" + for arg in "$@"; do + if [[ "$arg" == @* && -f "${arg#@}" ]]; then + cat -- "${arg#@}" >>"${log}.body" + fi + done +fi +exit "${PMAN_MOCK_CURL_RC:-0}" diff --git a/tests/mock-bin/sendmail b/tests/mock-bin/sendmail new file mode 100755 index 0000000..4b3ec44 --- /dev/null +++ b/tests/mock-bin/sendmail @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Mock sendmail: appends argv and the message (stdin) to $PMAN_MOCK_SENDMAIL_LOG; +# exits ${PMAN_MOCK_SENDMAIL_RC:-0}. +set -euo pipefail +log="${PMAN_MOCK_SENDMAIL_LOG:-}" +msg="$(cat)" +if [[ -n "$log" ]]; then + printf 'ARGV: %s\n%s\n---\n' "$*" "$msg" >>"$log" +fi +exit "${PMAN_MOCK_SENDMAIL_RC:-0}" diff --git a/tests/run.sh b/tests/run.sh index 9ca980c..ca41483 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -641,6 +641,144 @@ else _fail "_compose_message: cap or title wrong (${#MSG_BODY} bytes, '$MSG_TITLE')" fi +# --------------------------------------------------------------------------- +# Notifications: ntfy (curl mock) and e-mail (sendmail mock); no real network +# --------------------------------------------------------------------------- +if [[ "$(command -v curl)" != "$MOCK_BIN/curl" || "$(command -v sendmail)" != "$MOCK_BIN/sendmail" ]]; then + _fail "notify: curl/sendmail mocks are not first in PATH; skipping notification tests" +else + ntfy_tok="$TEST_TMP/ntfy-ok.token" + printf 'tk_secret123\n' >"$ntfy_tok" + chmod 600 "$ntfy_tok" + notify_tmp="$TEST_TMP/notify-tmp" + mkdir -p "$notify_tmp" + + # _notify_env NAME — fresh curl/sendmail logs; sets NENV for _chk. + _notify_env() { + nlog="$TEST_TMP/$1" + rm -f "$nlog".* + NENV=("NTFY_URL=https://ntfy.example.invalid/pman-topic" "NTFY_TOKEN_FILE=$ntfy_tok" + "HEALTH_MAIL_TO=root@localhost" "HEALTH_MAIL_FROM=pman@example.invalid" + "PMAN_MOCK_CURL_LOG=$nlog.curl" "PMAN_MOCK_SENDMAIL_LOG=$nlog.mail" "TMPDIR=$notify_tmp") + } + _count() { if [[ -f "$1" ]]; then grep -c "$2" "$1"; else printf '0'; fi; } + + _notify_env n1 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "$HOT" + if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" && "$(_count "$nlog.mail" '^ARGV: -t -oi$')" == "1" ]] && + grep -qF 'header = "Priority: urgent"' "$nlog.curl.cfg" && grep -qF 'header = "Tags: rotating_light"' "$nlog.curl.cfg" && + grep -q '^Subject: pman mock-host: CRIT' "$nlog.mail" && grep -qF '[CRIT] CT 100 (ct-one): memory 96%' "$nlog.curl.body"; then + _pass "notify: hot run sends one ntfy (urgent) and one mail (CRIT)" + else + _fail "notify: hot run did not send exactly one ntfy + one mail (exit $chk_rc)" + fi + if grep -q 'tk_secret123' "$nlog.curl.cfg" && ! grep -q 'tk_secret123' "$nlog.curl" && ! grep -q 'pman-topic' "$nlog.curl"; then + _pass "notify: token and topic only in curl stdin config, never in argv" + else + _fail "notify: token or topic leaked into curl argv" + fi + if [[ -z "$(ls -A "$notify_tmp")" ]]; then + _pass "notify: ntfy body temp file removed" + else + _fail "notify: ntfy body temp file left behind" + fi + if grep -q '^To: root@localhost$' "$nlog.mail" && grep -q '^From: pman@example.invalid$' "$nlog.mail" && + grep -q '^Content-Type: text/plain; charset=UTF-8$' "$nlog.mail" && grep -q '^Auto-Submitted: auto-generated$' "$nlog.mail"; then + _pass "notify: mail has To/From/Subject/MIME/Auto-Submitted headers" + else + _fail "notify: mail headers missing" + fi + _chk "$sd" "${NENV[@]}" "$HOT" + if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" && "$(_count "$nlog.mail" '^ARGV')" == "1" ]]; then + _pass "notify: unchanged state sends nothing" + else + _fail "notify: repeated alert on unchanged state" + fi + _chk "$sd" "${NENV[@]}" + if [[ "$chk_rc" == "0" && "$(_count "$nlog.curl" .)" == "2" ]] && tail -6 "$nlog.curl.cfg" | grep -qF 'Priority: low' && + grep -qF '[RESOLVED] CT 100 (ct-one)' "$nlog.curl.body" && grep -q '^Subject: pman mock-host: RESOLVED' "$nlog.mail"; then + _pass "notify: recovery sends RESOLVED with priority low" + else + _fail "notify: recovery notification wrong (exit $chk_rc)" + fi + + _notify_env n2 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "$HOT" PMAN_MOCK_CURL_RC=22 + if [[ "$chk_rc" == "2" ]] && grep -q $'^C\tmem:100\t2' "$sd/health.state" 2>/dev/null; then + _pass "notify: ntfy fails, mail works -> state saved" + else + _fail "notify: partial failure handling wrong (exit $chk_rc)" + fi + + _notify_env n3 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "$HOT" PMAN_MOCK_CURL_RC=7 PMAN_MOCK_SENDMAIL_RC=75 + if [[ "$chk_rc" == "2" && ! -e "$sd/health.state" ]]; then + _pass "notify: all channels fail -> exit reflects health, state not saved" + else + _fail "notify: all-fail handling wrong (exit $chk_rc)" + fi + rm -f "$nlog".* + _chk "$sd" "${NENV[@]}" "$HOT" + if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" && -e "$sd/health.state" ]] && grep -qF 'Priority: urgent' "$nlog.curl.cfg"; then + _pass "notify: next run resends after a total failure" + else + _fail "notify: alert not resent after a total failure" + fi + + _notify_env n4 + tn_rc=0 + env "${NENV[@]}" "$SCRIPT" --test-notify >/dev/null 2>&1 || tn_rc=$? + if [[ "$tn_rc" == "0" && "$(_count "$nlog.curl" .)" == "1" && "$(_count "$nlog.mail" '^ARGV')" == "1" ]] && + grep -q '^Subject: pman mock-host: test notification$' "$nlog.mail"; then + _pass "--test-notify: sends through both channels (exit 0)" + else + _fail "--test-notify: expected one ntfy + one mail (exit $tn_rc)" + fi + tn_rc=0 + env "${NENV[@]}" PMAN_MOCK_CURL_RC=22 PMAN_MOCK_SENDMAIL_RC=1 "$SCRIPT" --test-notify >/dev/null 2>&1 || tn_rc=$? + if [[ "$tn_rc" == "1" ]]; then + _pass "--test-notify: all channels failing exits 1" + else + _fail "--test-notify: all channels failing exited $tn_rc" + fi + + _notify_env n5 + tn_rc=0 + env "PMAN_MOCK_CURL_LOG=$nlog.curl" NTFY_URL=http://ntfy.example.invalid/pman-topic "NTFY_TOKEN_FILE=$ntfy_tok" \ + "$SCRIPT" --test-notify >/dev/null 2>&1 || tn_rc=$? + if [[ "$tn_rc" == "1" && ! -e "$nlog.curl" ]]; then + _pass "notify: token is never sent over plain http (channel fails)" + else + _fail "notify: token sent over http or wrong exit ($tn_rc)" + fi + + # Header injection: CR/LF in the title must not create extra headers. + _notify_env n6 + inj_rc=0 + ( + export PMAN_MOCK_SENDMAIL_LOG="$nlog.mail" + HEALTH_MAIL_TO="root@localhost" + HEALTH_MAIL_FROM="" + MSG_TITLE=$'pman x: CRIT\r\nBcc: evil@example.invalid' + MSG_BODY="body" + MSG_PRIORITY="urgent" + _notify_mail >/dev/null 2>&1 + ) || inj_rc=$? + if [[ "$inj_rc" == "0" ]] && ! grep -q '^Bcc:' "$nlog.mail" && grep -q '^Subject: pman x: CRIT Bcc: evil@example.invalid$' "$nlog.mail"; then + _pass "notify: CR/LF in the title cannot inject mail headers" + else + _fail "notify: mail header injection possible (exit $inj_rc)" + fi +fi + +sd="$(_new_state_dir)" +_chk "$sd" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-vzdump-200.json" +_expect_chk "--check: stop during a running vzdump (stop mode) is OK" 0 "PMAN OK" + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From ae5576f8e029eb6697acd974771a54bb86ab4f34 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 18:52:44 +0200 Subject: [PATCH 5/8] docs(release): prepare v2.13.0 with health monitoring and alerts - Version header 2.13.0 with changelog line; CHANGELOG [2.13.0] with Added, Changed and Security. - README: features and usage rows, new "Health monitoring & alerts" section (checks, thresholds, exit codes, state, ntfy/mail setup, cron example, limitations), full config key list with chmod 600 advice, updated security note, requirements, test count and banner version. - Bash/Zsh completions: --name, --health, --check, --dry-run, --test-notify. - CI runs tests/run.sh after shfmt. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 3 ++ CHANGELOG.md | 33 ++++++++++++ README.md | 109 ++++++++++++++++++++++++++++++++++++--- completions/pman.bash | 7 ++- completions/pman.zsh | 7 ++- proxmox-manager.sh | 3 +- 6 files changed, 151 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e55445f..a846068 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,3 +51,6 @@ jobs: printf '%s %s\n' "${SHFMT_SHA256}" "${shfmt_path}" | sha256sum --check --strict chmod 0755 "${shfmt_path}" "${shfmt_path}" -d proxmox-manager.sh + + - name: Run tests + run: bash tests/run.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index fe36b34..892e862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,39 @@ Versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [2.13.0] - 2026-09-19 + +### Added +- `--health` (boxed), `--health --list` and `--health --json`: CPU, memory and disk usage, uptime + and a health level per local VM/CT, read from `pvesh get /cluster/resources`; templates and guests + on other cluster nodes are skipped. `--filter` and `--name` apply. +- Main menu key `h` opens the health overview; the status action prints an extra `Health:` line. +- `--check` for cron: alerts only on changes (new, escalated, improved, resolved) for resource + thresholds (CPU after `HEALTH_CPU_RUNS` consecutive runs), guests stopped with `onboot=1`, guests + that stopped without a stop/shutdown/migrate/destroy/backup task, and failed node tasks. + Nagios-style summary and exit codes 0 OK, 1 WARN, 2 CRIT, 3 UNKNOWN. +- `--dry-run` (with `--check`) prints the result and the composed message without sending or + writing state; `--test-notify` sends a test message. +- Notifications via ntfy (`NTFY_URL`, optional `NTFY_TOKEN_FILE`) and e-mail through the local + `sendmail` (`HEALTH_MAIL_TO`, `HEALTH_MAIL_FROM`); one combined message per run. +- New config keys `HEALTH_{CPU,MEM,DISK}_{WARN,CRIT}` (`0` disables a level), `HEALTH_CPU_RUNS`, + `HEALTH_IGNORE_IDS`, `HEALTH_STATE_DIR` (default `/var/lib/pman`) and the notification keys above. +- CI runs the test suite (`tests/run.sh`); shell completions gained `--name`, `--health`, + `--check`, `--dry-run` and `--test-notify`. + +### Changed +- `STOP_TIMEOUT` and `PROXMOX_MANAGER_SPICE_ADDR` are validated after the command line is parsed. +- Outbound traffic is now possible, but only when notifications are configured (opt-in). + +### Security +- The ntfy access token is read from a `0600` file owned by the caller; an inline `NTFY_TOKEN` in a + config file is rejected. The token, URL and headers reach `curl` only via its stdin config, never + argv; redirects are not followed, and the token is never sent over plain `http://`. +- Logs name only the ntfy host, never the topic path or the token. +- Mail headers are built from validated single-line values (no header injection). +- The `--check` state lives in a `0700` directory, is written atomically with mode `0600`, is parsed + as validated data (never sourced), and parallel runs are prevented with `flock`. + ## [2.12.1] - 2026-09-19 ### Fixed diff --git a/README.md b/README.md index 02a1238..88f80f1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ║ ██████╔╝ ██╔████╔██║ ███████║ ██╔██╗ ██║ ║ ║ ██╔═══╝ ██║╚██╔╝██║ ██╔══██║ ██║╚██╗██║ ║ ║ ██║ ██║ ╚═╝ ██║ ██║ ██║ ██║ ╚████║ ║ -║ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═══╝ v2.12.1 ║ +║ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═══╝ v2.13.0 ║ ║ ║ ║ Proxmox VM/CT Manager · Single Bash · No Dependencies ║ ╚══════════════════════════════════════════════════════════╝ @@ -61,6 +61,7 @@ Check out how others are using `proxmox-manager`: | 🌐 | **IP Address Lookup** | Shows current IPv4 addresses for running VMs and CTs — VM via QEMU Guest Agent, CT via `pct exec` | | 📦 | **Snapshot Management** | List, create, rollback, delete — with name validation and snapshot preview before destructive actions | | 🖱️ | **SPICE Integration** | Enable SPICE for VMs and retrieve `.vv` connection files. Auto-launches `virt-viewer` when installed | +| 🩺 | **Health & Alerts** | CPU / RAM / disk per guest, stopped-with-onboot, unexpected stops and failed tasks — `--health` view, `--check` for cron with ntfy / e-mail alerts (opt-in) | | 🤖 | **Automation-Ready** | `--json` output, `--filter` by status, `--name` ERE filter, `--force` mode, structured logging via `LOG_FILE` | | ⚙️ | **Config File** | Persistent allowlisted defaults via `/etc/pmanrc` or `~/.pmanrc` — CLI flags always win | @@ -97,13 +98,14 @@ A single `proxmox-manager.sh` script — no build step, no service, no config fi └───────────────────────────────┘ ``` -> ✅ No network calls · ✅ No background process · ✅ Bash ≥ 4.0 · ✅ PVE 7.x / 8.x / 9.x +> ✅ No network calls (unless alerts are configured) · ✅ No background process · ✅ Bash ≥ 4.0 · ✅ PVE 7.x / 8.x / 9.x --- ## 🚀 Installation -**Requirements:** Proxmox VE host · Bash ≥ 4.0 · Root privileges · `qm` / `pct` (bundled with PVE) +**Requirements:** Proxmox VE host · Bash ≥ 4.0 · Root privileges · `qm` / `pct` / `pvesh` and +`python3` (all bundled with PVE) · optional: `sendmail` (e.g. postfix) for e-mail alerts ### Step 1 — Clone @@ -142,6 +144,10 @@ Installs an atomic, root-owned copy at `/usr/local/bin/pman` so later edits to t | `pman --timeout 30` | Custom stop timeout in seconds (default: 60) | | `pman --no-clear` | Don't clear screen in interactive mode | | `pman --once` | Run a single interactive refresh cycle (useful for TTY recording) | +| `pman --health` | Health overview: CPU / MEM / DISK %, uptime and level per guest (`--list`, `--json` combinable) | +| `pman --check` | Health check for cron: alerts on changes, exit `0` OK · `1` WARN · `2` CRIT · `3` UNKNOWN | +| `pman --check --dry-run` | Show the check result and the message that would be sent; nothing is sent or saved | +| `pman --test-notify` | Send a test message through all configured channels | | `pman --version` | Print version and exit | | `pman -h, --help` | Show usage information and exit | @@ -157,7 +163,7 @@ Displays a table of all VMs/containers. Enter a VMID to open the action menu. [+] running [-] stopped [~] paused [?] unknown ``` -Press `r` to refresh · `q` to quit +Press `h` for the health overview · `r` to refresh · `q` to quit ### JSON output @@ -183,6 +189,77 @@ apt install virt-viewer When `virt-viewer` is **not** installed, the path to the generated `.vv` file is printed together with the install hint. The SPICE bind address defaults to `127.0.0.1` and can be overridden via `PROXMOX_MANAGER_SPICE_ADDR` (env var or `~/.pmanrc`). +### Health monitoring & alerts + +`pman --health` shows every guest on the **local node** (templates skipped) with CPU, memory and +disk usage, uptime and a health level; `--list` prints plain text, `--json` machine-readable +data (`null` where a value is not available). The same data backs the `h` key in the TUI and +the `Health:` line of the status action. + +`pman --check` is meant for cron. It evaluates: + +| Check | Level | Notes | +|---|---|---| +| CPU | WARN / CRIT | Must stay above the threshold for `HEALTH_CPU_RUNS` runs in a row | +| Memory, disk | WARN / CRIT | Immediately. VM disk usage needs the QEMU guest agent, otherwise n/a | +| Stopped with `onboot=1` | CRIT | Guest is configured to start at boot but is not running | +| Stopped unexpectedly | WARN | Was running at the last check, no stop/shutdown/migrate/destroy/backup task | +| Failed task | WARN / CRIT | Any node task that ended not `OK` since the last run (`WARNINGS` → WARN) | + +Thresholds (percent; `0` disables that level): + +| Key | Default | Key | Default | +|---|---|---|---| +| `HEALTH_CPU_WARN` | 85 | `HEALTH_CPU_CRIT` | 95 | +| `HEALTH_MEM_WARN` | 90 | `HEALTH_MEM_CRIT` | 95 | +| `HEALTH_DISK_WARN` | 85 | `HEALTH_DISK_CRIT` | 95 | +| `HEALTH_CPU_RUNS` | 3 | `HEALTH_IGNORE_IDS` | _(empty)_ e.g. `105,210` | + +A notification is sent only when something changes: new or escalated problems, improvements and +`RESOLVED` (with duration). All changes of one run go into one message. The first run only +records a baseline, so old failed tasks are not reported. State is kept in `HEALTH_STATE_DIR` +(default `/var/lib/pman`, mode `0700`); if every channel fails, the state is not saved and the next +run tries again. Exit codes: `0` OK · `1` WARN · `2` CRIT · `3` UNKNOWN (config, lock or `pvesh` +error) — the output is a Nagios-style summary line plus one line per problem. + +**ntfy** — push to phone/desktop via [ntfy](https://ntfy.sh) (public or self-hosted): + +```bash +# /etc/pmanrc +NTFY_URL="https://ntfy.sh/pman-alerts-CHANGE-ME" # use a hard-to-guess topic +NTFY_TOKEN_FILE="/etc/pman/ntfy.token" # optional, for protected topics + +# store the token in a private file (never inline in pmanrc) +install -d -m 700 /etc/pman +install -m 600 /dev/null /etc/pman/ntfy.token && nano /etc/pman/ntfy.token +``` + +The token is only sent over `https://`. **E-mail** uses the local `sendmail` (postfix, which PVE +installs for its own notifications, works out of the box when it can relay): + +```bash +# /etc/pmanrc +HEALTH_MAIL_TO="admin@example.com" # comma separated, no spaces +HEALTH_MAIL_FROM="pman@pve.example.com" # optional +``` + +Test the channels, then add the cron job (set `PATH`, cron's default lacks `/usr/sbin`): + +```bash +pman --test-notify +pman --check --dry-run; echo "exit $?" +``` + +``` +# /etc/cron.d/pman-health +PATH=/usr/sbin:/usr/bin:/sbin:/bin +*/5 * * * * root /usr/local/bin/pman --check >/dev/null 2>&1 +``` + +Limitations: a shutdown from **inside** a guest has no Proxmox task and is reported as +"stopped unexpectedly"; in a cluster, run the cron job on **every node** (each checks only its own +guests); failed tasks are read from the last 200 node tasks. + ### Configuration `proxmox-manager` checks the following files at startup (in this order): @@ -199,9 +276,17 @@ CLI flags are applied last and always win over config file values. STOP_TIMEOUT=120 # stop timeout in seconds (default: 60) LOG_FILE="/var/log/proxmox-manager.log" # structured log file (empty = disabled) PROXMOX_MANAGER_SPICE_ADDR="spice.example.invalid" # SPICE bind address +HEALTH_MEM_WARN=80 # health thresholds, see "Health monitoring & alerts" +NTFY_URL="https://ntfy.sh/pman-alerts-CHANGE-ME" ``` -Configuration files are parsed as data, not executed as shell code. Only the three settings shown above are accepted. When enabled, `LOG_FILE` must be an absolute, non-symlinked regular file owned by the current user with mode `0600`; its parent directory must also be owner-controlled and not group/world-writable. +Configuration files are parsed as data, not executed as shell code. Accepted keys: +`STOP_TIMEOUT`, `LOG_FILE`, `PROXMOX_MANAGER_SPICE_ADDR`, `HEALTH_CPU_WARN`, `HEALTH_CPU_CRIT`, +`HEALTH_MEM_WARN`, `HEALTH_MEM_CRIT`, `HEALTH_DISK_WARN`, `HEALTH_DISK_CRIT`, `HEALTH_CPU_RUNS`, +`HEALTH_IGNORE_IDS`, `HEALTH_STATE_DIR`, `NTFY_URL`, `NTFY_TOKEN_FILE`, `HEALTH_MAIL_TO` and +`HEALTH_MAIL_FROM`; anything else is ignored with a warning. Health settings are validated only +by `--health`, `--check` and `--test-notify`, so a typo never blocks the TUI. Because the file +may name alert targets, keep it private: `chmod 600 /etc/pmanrc`. When enabled, `LOG_FILE` must be an absolute, non-symlinked regular file owned by the current user with mode `0600`; its parent directory must also be owner-controlled and not group/world-writable. ### Shell Completions @@ -220,14 +305,20 @@ cp completions/pman.zsh ~/.zsh/completions/_pman - **Root required:** `qm` and `pct` need elevated privileges — there's no workaround. - **No credentials stored:** Relies entirely on Proxmox host authentication. -- **No outbound traffic:** All operations are local to the node. +- **No outbound traffic unless notifications are configured:** alerts via ntfy / e-mail are opt-in; + the ntfy token is read from a `0600` file, passed to `curl` via stdin and never sent over plain http. - **CI hardening:** ShellCheck on every push · Gitleaks scan for accidental secrets. --- ## 📋 Changelog -### 🆕 [v2.12.1](CHANGELOG.md) — 2026-09-19 +### 🆕 [v2.13.0](CHANGELOG.md) — 2026-09-19 + +> Health view (`--health`, key `h`) · `--check` for cron with ntfy / e-mail alerts on changes · +> exit codes 0/1/2/3 · `--test-notify` + +### [v2.12.1](CHANGELOG.md) — 2026-09-19 > Action and snapshot menus draw a closed frame · README screenshots regenerated @@ -270,7 +361,9 @@ shellcheck proxmox-manager.sh tests/run.sh ``` -31 tests covering `validate_vmid`, `validate_snapshot_name`, `ip_info`, `--filter`, and CLI flags. +100 tests covering `validate_vmid`, `validate_snapshot_name`, `ip_info`, `--filter`, CLI flags, config +parsing, the health view, the `--check` engine and notifications (mocked `pvesh`, `curl`, `sendmail`; +no network, no root). --- diff --git a/completions/pman.bash b/completions/pman.bash index 43bb4d0..3aade28 100644 --- a/completions/pman.bash +++ b/completions/pman.bash @@ -11,13 +11,18 @@ _pman() { prev="${COMP_WORDS[COMP_CWORD-1]}" } - local all_flags="--list --json --filter --no-clear --once --timeout --force --version --help" + local all_flags="--list --json --filter --name --no-clear --once --timeout --force --health --check --dry-run --test-notify --version --help" case "$prev" in --filter) COMPREPLY=($(compgen -W "running stopped paused" -- "$cur")) return 0 ;; + --name) + # Free-form ERE pattern; nothing sensible to complete + COMPREPLY=() + return 0 + ;; --timeout) # Suggest common timeout values; user can type any positive integer COMPREPLY=($(compgen -W "10 30 60 120 300" -- "$cur")) diff --git a/completions/pman.zsh b/completions/pman.zsh index e5afb0f..73f5c12 100644 --- a/completions/pman.zsh +++ b/completions/pman.zsh @@ -19,7 +19,12 @@ _pman() { '--no-clear[Do not clear the screen in interactive mode]' \ '--once[Run a single interactive refresh cycle]' \ '--timeout[Timeout in seconds for stop operations]:seconds:(10 30 60 120 300)' \ - '--force[Skip all confirmation prompts]' + '--name[Filter by VM/CT name (ERE substring-match)]:pattern:' \ + '--force[Skip all confirmation prompts]' \ + '--health[Show CPU/memory/disk health of local VMs/CTs]' \ + '--check[Run health checks for cron (exit 0/1/2/3)]' \ + '--dry-run[With --check: print result and message, send nothing]' \ + '--test-notify[Send a test message through all configured channels]' } _pman "$@" diff --git a/proxmox-manager.sh b/proxmox-manager.sh index df284e3..0059c06 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Proxmox VM/CT Management Tool -# Version 2.12.1 — 2026-09-19 +# Version 2.13.0 — 2026-09-19 +# - feat: health view (--health, menu key h) and cron checks (--check) with ntfy/e-mail alerts # - fix: action and snapshot menus draw a closed frame (right border, padded rows) # - fix: #33 TUI frames aligned by visible width; long guest names fit/truncate; PVE version parsed correctly # - feat: #28 show current IP addresses for running VMs/CTs From 479820a57999e35a1a8468a2b178b8d2266f0104 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 19:01:30 +0200 Subject: [PATCH 6/8] fix(health): address review findings - Metric alerts resolve when a guest stops (CPU counter restarts); an n/a metric on a running guest keeps its state without being reported. - Unexpected-stop detection uses the most recent lifecycle task per guest (no time window, OK or WARNINGS; suspend, reboot in progress and stop-mode backups included); a stop task clears an existing latch. onboot=1 guests stopped by a task are WARN instead of CRIT. - Tasks are read with --source all so running tasks are visible. - --force warning goes to stderr for --json/--health --json/--check. - --check: INT/TERM and usage errors exit 3 from the very start. - Total delivery failure saves the state but keeps the unsent changes (and the task watermark) pending, so they fire once on the next run. - Lock on fd 9, closed for pvesh/python3/qm/pct/curl/sendmail. - onboot read from /etc/pve/{qemu-server,lxc}/ID.conf (current section) with qm/pct fallback; node name from `uname -n` up to the first dot. - Findings shorten the guest name, never the text; narrow terminals drop UPTIME/DISK%/TYPE; ignored guests are not counted; an empty node is not an error for --health. - Regression tests for each item. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 8 +- README.md | 11 +- proxmox-manager.sh | 304 +++++++++++++----- tests/fixtures/resources-100-stopped.json | 6 + tests/fixtures/resources-empty.json | 1 + tests/fixtures/tasks-failed-new.json | 2 +- .../tasks-restart-after-stop-200.json | 4 + tests/fixtures/tasks-shutdown-201.json | 4 + tests/fixtures/tasks-shutdown-old-200.json | 4 + tests/fixtures/tasks-suspend-200.json | 4 + tests/fixtures/tasks-vzdump-200.json | 2 +- tests/fixtures/tasks-vzshutdown-100.json | 4 + tests/mock-bin/curl | 5 + tests/mock-bin/pvesh | 10 + tests/mock-bin/sendmail | 5 + tests/mock-bin/uname | 14 + tests/run.sh | 174 +++++++++- 17 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 tests/fixtures/resources-100-stopped.json create mode 100644 tests/fixtures/resources-empty.json create mode 100644 tests/fixtures/tasks-restart-after-stop-200.json create mode 100644 tests/fixtures/tasks-shutdown-201.json create mode 100644 tests/fixtures/tasks-shutdown-old-200.json create mode 100644 tests/fixtures/tasks-suspend-200.json create mode 100644 tests/fixtures/tasks-vzshutdown-100.json create mode 100755 tests/mock-bin/uname diff --git a/CHANGELOG.md b/CHANGELOG.md index 892e862..62b78db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,10 @@ Versions follow [Semantic Versioning](https://semver.org/). on other cluster nodes are skipped. `--filter` and `--name` apply. - Main menu key `h` opens the health overview; the status action prints an extra `Health:` line. - `--check` for cron: alerts only on changes (new, escalated, improved, resolved) for resource - thresholds (CPU after `HEALTH_CPU_RUNS` consecutive runs), guests stopped with `onboot=1`, guests - that stopped without a stop/shutdown/migrate/destroy/backup task, and failed node tasks. - Nagios-style summary and exit codes 0 OK, 1 WARN, 2 CRIT, 3 UNKNOWN. + thresholds (CPU after `HEALTH_CPU_RUNS` consecutive runs; resolved when the guest stops), guests + stopped with `onboot=1`, guests whose latest lifecycle task does not explain a stop (stop, shutdown, + suspend, migrate, destroy, backup), and failed node tasks (read with `--source all`). + Nagios-style summary and exit codes 0 OK, 1 WARN, 2 CRIT, 3 UNKNOWN (also for usage errors). - `--dry-run` (with `--check`) prints the result and the composed message without sending or writing state; `--test-notify` sends a test message. - Notifications via ntfy (`NTFY_URL`, optional `NTFY_TOKEN_FILE`) and e-mail through the local @@ -30,6 +31,7 @@ Versions follow [Semantic Versioning](https://semver.org/). ### Changed - `STOP_TIMEOUT` and `PROXMOX_MANAGER_SPICE_ADDR` are validated after the command line is parsed. +- With `--json` the `--force` warning goes to stderr, so the JSON output stays valid. - Outbound traffic is now possible, but only when notifications are configured (opt-in). ### Security diff --git a/README.md b/README.md index 88f80f1..9c30bfd 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,9 @@ the `Health:` line of the status action. |---|---|---| | CPU | WARN / CRIT | Must stay above the threshold for `HEALTH_CPU_RUNS` runs in a row | | Memory, disk | WARN / CRIT | Immediately. VM disk usage needs the QEMU guest agent, otherwise n/a | -| Stopped with `onboot=1` | CRIT | Guest is configured to start at boot but is not running | -| Stopped unexpectedly | WARN | Was running at the last check, no stop/shutdown/migrate/destroy/backup task | +| _(guest stops)_ | — | CPU / memory / disk alerts of a stopped guest are resolved | +| Stopped with `onboot=1` | CRIT | Configured to start at boot but not running (WARN if stopped by a task) | +| Stopped unexpectedly | WARN | Was running at the last check; its latest task is no stop/shutdown/suspend/migrate/destroy/backup | | Failed task | WARN / CRIT | Any node task that ended not `OK` since the last run (`WARNINGS` → WARN) | Thresholds (percent; `0` disables that level): @@ -218,9 +219,9 @@ Thresholds (percent; `0` disables that level): A notification is sent only when something changes: new or escalated problems, improvements and `RESOLVED` (with duration). All changes of one run go into one message. The first run only records a baseline, so old failed tasks are not reported. State is kept in `HEALTH_STATE_DIR` -(default `/var/lib/pman`, mode `0700`); if every channel fails, the state is not saved and the next -run tries again. Exit codes: `0` OK · `1` WARN · `2` CRIT · `3` UNKNOWN (config, lock or `pvesh` -error) — the output is a Nagios-style summary line plus one line per problem. +(default `/var/lib/pman`, mode `0700`); if every channel fails, the unsent changes stay pending and +are sent by the next run. Exit codes: `0` OK · `1` WARN · `2` CRIT · `3` UNKNOWN (usage, config, +lock or `pvesh` error) — the output is a Nagios-style summary line plus one line per problem. **ntfy** — push to phone/desktop via [ntfy](https://ntfy.sh) (public or self-hosted): diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 0059c06..3b8ba95 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -58,6 +58,7 @@ declare -A _HV_LEVEL=() # per-guest max health level (see _health_evaluate) declare -A _HV_FIND=() # per-guest findings (see _health_evaluate) HEALTH_TASKS=() # parsed task rows (see _health_parse_tasks) HEALTH_STATE_HEADER='# pman-health-state v1' +PVE_ETC_DIR='/etc/pve' # guest configs for the onboot lookup (overridden only by tests) _CHECK_TMP='' # temp state file removed by the --check EXIT trap _CHECK_DONE=0 # set right before --check exits normally _NOTIFY_TMP='' # temp body file of _notify_ntfy @@ -547,14 +548,14 @@ parse_args() { --filter) if [[ $# -lt 2 ]]; then err "--filter requires a value: running, stopped, or paused." - exit 1 + exit "$FATAL_EXIT" fi FILTER_STATUS="$2" case "$FILTER_STATUS" in running | stopped | paused) ;; *) err "Invalid --filter value '$FILTER_STATUS'. Valid: running, stopped, paused." - exit 1 + exit "$FATAL_EXIT" ;; esac shift # consume the STATUS value; outer shift consumes --filter @@ -562,19 +563,19 @@ parse_args() { --timeout) if [[ $# -lt 2 ]]; then err "--timeout requires a value in seconds (e.g., --timeout 30)." - exit 1 + exit "$FATAL_EXIT" fi STOP_TIMEOUT="$2" if [[ ! "$STOP_TIMEOUT" =~ ^[0-9]+$ ]] || ((STOP_TIMEOUT < 1)); then err "--timeout requires a positive integer (seconds), got '$STOP_TIMEOUT'." - exit 1 + exit "$FATAL_EXIT" fi shift # consume the SECS value; outer shift consumes --timeout ;; --name) if [[ $# -lt 2 ]]; then err "--name requires an ERE pattern value." - exit 1 + exit "$FATAL_EXIT" fi FILTER_NAME="$2" # Validate ERE: grep exit code ≥2 means invalid pattern @@ -582,7 +583,7 @@ parse_args() { echo '' | grep -E -- "$FILTER_NAME" >/dev/null 2>&1 || _grep_rc=$? if ((_grep_rc >= 2)); then err "--name pattern '$FILTER_NAME' is not a valid ERE." - exit 1 + exit "$FATAL_EXIT" fi shift ;; @@ -616,19 +617,19 @@ parse_args() { -*) err "Unknown option: $1" usage - exit 1 + exit "$FATAL_EXIT" ;; *) err "Unexpected argument: $1" usage - exit 1 + exit "$FATAL_EXIT" ;; esac shift done if ((LIST_FLAG == 1 && JSON_FLAG == 1)); then err "Options --list and --json are not combinable." - exit 1 + exit "$FATAL_EXIT" fi _resolve_mode } @@ -637,16 +638,16 @@ parse_args() { _resolve_mode() { if ((DRY_RUN == 1 && CHECK_FLAG == 0)); then err "Option --dry-run requires --check." - exit 1 + exit "$FATAL_EXIT" fi if ((CHECK_FLAG == 1)); then if ((LIST_FLAG == 1 || JSON_FLAG == 1 || HEALTH_FLAG == 1 || TEST_NOTIFY_FLAG == 1)); then err "Option --check is not combinable with --list, --json, --health or --test-notify." - exit 1 + exit "$FATAL_EXIT" fi if [[ -n "$FILTER_STATUS" || -n "$FILTER_NAME" ]]; then err "Option --check always checks all guests; --filter and --name are not supported." - exit 1 + exit "$FATAL_EXIT" fi MODE="check" return 0 @@ -654,7 +655,7 @@ _resolve_mode() { if ((TEST_NOTIFY_FLAG == 1)); then if ((LIST_FLAG == 1 || JSON_FLAG == 1 || HEALTH_FLAG == 1)); then err "Option --test-notify is not combinable with --list, --json or --health." - exit 1 + exit "$FATAL_EXIT" fi MODE="test_notify" return 0 @@ -1144,9 +1145,11 @@ print_table() { # ============================================================================= # _local_node — print the short name of the local node (validated); return 1 if unknown. +# Uses PVE::INotify::nodename semantics: `uname -n` up to the first dot. _local_node() { local n - n="$(hostname -s 2>/dev/null || true)" + n="$(uname -n 2>/dev/null || true)" + n="${n%%.*}" [[ "$n" =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,62}$ ]] || return 1 printf '%s' "$n" } @@ -1205,8 +1208,8 @@ _health_fetch() { local kind="$1" node="${2:-}" have pvesh || return 1 case "$kind" in - resources) timeout 30 pvesh get /cluster/resources --type vm --output-format json 2>/dev/null ;; - tasks) timeout 30 pvesh get "/nodes/${node}/tasks" --limit 200 --output-format json 2>/dev/null ;; + resources) timeout 30 pvesh get /cluster/resources --type vm --output-format json 2>/dev/null 9>&- ;; + tasks) timeout 30 pvesh get "/nodes/${node}/tasks" --source all --limit 200 --output-format json 2>/dev/null 9>&- ;; *) return 1 ;; esac } @@ -1269,7 +1272,7 @@ for r in data: rows.sort() for row in rows: print("\t".join(str(x) for x in row)) -' "$1" +' "$1" 9>&- } # _health_parse_tasks — read /nodes//tasks JSON on stdin and print TSV rows sorted @@ -1311,7 +1314,7 @@ for t in data: rows.sort() for row in rows: print("\t".join(str(x) for x in row)) -' +' 9>&- } # _health_load — fetch and parse the local guests into HEALTH_ROWS (sets HEALTH_NODE). @@ -1367,13 +1370,25 @@ _health_load_tasks() { } # _guest_onboot ID TYPE — true when the guest config has "onboot: 1". +# Reads the current config section of PVE_ETC_DIR/{qemu-server,lxc}/ID.conf (snapshot +# sections start at the first "[" line); falls back to qm/pct config when unreadable. _guest_onboot() { - local id="$1" ty="$2" cfg='' + local id="$1" ty="$2" cfg='' conf case "$ty" in - CT) cfg="$(pct config "$id" 2>/dev/null || true)" ;; - VM) cfg="$(qm config "$id" 2>/dev/null || true)" ;; + CT) conf="${PVE_ETC_DIR}/lxc/${id}.conf" ;; + VM) conf="${PVE_ETC_DIR}/qemu-server/${id}.conf" ;; *) return 1 ;; esac + if [[ "$id" =~ ^[0-9]+$ && -f "$conf" && -r "$conf" ]]; then + if awk '/^\[/ { exit } /^onboot:[[:space:]]*1[[:space:]]*$/ { found = 1 } END { exit !found }' "$conf"; then + return 0 + fi + return 1 + fi + case "$ty" in + CT) cfg="$(pct config "$id" 2>/dev/null 9>&- || true)" ;; + VM) cfg="$(qm config "$id" 2>/dev/null 9>&- || true)" ;; + esac grep -qE '^onboot:[[:space:]]*1[[:space:]]*$' <<<"$cfg" } @@ -1426,6 +1441,8 @@ _health_finding() { down) if [[ "$lvl" == "2" ]]; then printf 'stopped although onboot=1' + elif [[ "$val" == "onboot-task" ]]; then + printf 'stopped by a Proxmox task although onboot=1' else printf 'stopped unexpectedly' fi @@ -1496,8 +1513,25 @@ _health_row_shown() { return 0 } +# _health_finding_line WIDTH ENTRY — render one findings entry ("lvl TAB type id TAB name +# TAB text") within WIDTH columns: the guest name is shortened, the finding text never. +_health_finding_line() { + local width="$1" entry="$2" f_lvl f_guest f_name f_text tag avail + IFS=$'\t' read -r f_lvl f_guest f_name f_text <<<"$entry" + tag="[$(_level_name "$f_lvl")]" + avail=$((width - ${#tag} - 1 - ${#f_guest} - 4 - $(_vis_width "$f_text"))) + if ((width <= 0 || avail >= $(_vis_width "$f_name"))); then + printf '%s %s (%s): %s' "$(_level_color "$f_lvl" "$tag")" "$f_guest" "$f_name" "$f_text" + elif ((avail >= 4)); then + printf '%s %s (%s): %s' "$(_level_color "$f_lvl" "$tag")" "$f_guest" "$(_truncate "$f_name" "$avail")" "$f_text" + else + printf '%s %s: %s' "$(_level_color "$f_lvl" "$tag")" "$f_guest" "$f_text" + fi +} + # print_health_table — health overview of local guests (boxed for --health and the -# interactive menu, plain for --health --list). Returns 1 on error or when empty. +# interactive menu, plain for --health --list). Narrow terminals drop UPTIME, DISK% and +# TYPE (in that order) before the NAME column shrinks. Returns 1 only on errors. print_health_table() { local draw_boxes=0 [[ "$MODE" == "health" || "$MODE" == "interactive" ]] && draw_boxes=1 @@ -1516,23 +1550,40 @@ print_health_table() { fi done - # │ + 2 + ID 6 + TYPE 5 + STATUS 8 + CPU% 5 + MEM% 5 + DISK% 5 + UPTIME 11 + HEALTH 7 - # + 8 separators + NAME + 2 + │ - local name_w=15 fixed=65 + # 2 borders + 4 margin + ID 7 + TYPE 6 + STATUS 9 + CPU% 6 + MEM% 6 + DISK% 6 + # + UPTIME 12 + HEALTH 8 + NAME + local name_w=15 fixed=65 show_type=1 show_disk=1 show_up=1 cols for row in "${rows[@]}"; do IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" (($(_vis_width "$nm") > name_w)) && name_w=$(_vis_width "$nm") done if ((draw_boxes)); then - local max_name=$(($(_term_cols) - fixed)) + cols="$(_term_cols)" + if ((fixed + 10 > cols)); then + show_up=0 + fixed=$((fixed - 12)) + fi + if ((fixed + 10 > cols)); then + show_disk=0 + fixed=$((fixed - 6)) + fi + if ((fixed + 10 > cols)); then + show_type=0 + fixed=$((fixed - 6)) + fi + local max_name=$((cols - fixed)) ((name_w > max_name)) && name_w=$max_name - ((name_w < 10)) && name_w=10 + ((name_w < 6)) && name_w=6 fi local W=$((fixed + name_w)) local head - printf -v head '%-6s %-5s %-8s %5s %5s %5s %-11s %-7s %s' \ - "ID" "TYPE" "STATUS" "CPU%" "MEM%" "DISK%" "UPTIME" "HEALTH" "NAME" + printf -v head '%-6s ' "ID" + ((show_type)) && head+="$(printf '%-5s ' "TYPE")" + head+="$(printf '%-8s %5s %5s ' "STATUS" "CPU%" "MEM%")" + ((show_disk)) && head+="$(printf '%5s ' "DISK%")" + ((show_up)) && head+="$(printf '%-11s ' "UPTIME")" + head+="$(printf '%-7s %s' "HEALTH" "NAME")" if ((draw_boxes)); then _draw_line_top $W _box_content "${CYAN}" "${LINE_V}" $W " ${BOLD}${WHITE}${head}${NC}" @@ -1542,15 +1593,15 @@ print_health_table() { fi if ((${#rows[@]} == 0)); then - local msg="No VMs or containers found on node ${HEALTH_NODE}." - [[ -n "$FILTER_STATUS" || -n "$FILTER_NAME" ]] && msg="No VMs or containers match the filter." + local msg="No guests found on node ${HEALTH_NODE}." + [[ -n "$FILTER_STATUS" || -n "$FILTER_NAME" ]] && msg="No guests match the filter." if ((draw_boxes)); then - _box_content "${CYAN}" "${LINE_V}" $W " ${RED_BRIGHT}${msg}${NC}" + _box_content "${CYAN}" "${LINE_V}" $W " ${DIM}$(_truncate "$msg" $((W - 6)))${NC}" _draw_line_bot $W else - printf '%b%s%b\n' "${RED_BRIGHT}" "$msg" "${NC}" + printf '%s\n' "$msg" fi - return 1 + return 0 fi local n_ok=0 n_warn=0 n_crit=0 n_ign=0 n_run=0 lvl hname ty_col line upt item f_lvl f_chk f_val @@ -1573,22 +1624,22 @@ print_health_table() { IFS=$'\t' read -r -a items <<<"${_HV_FIND[$id]:-}" for item in "${items[@]}"; do IFS='|' read -r f_lvl f_chk f_val <<<"$item" - findings+=("${f_lvl}"$'\t'"${ty} ${id} (${nm}): $(_health_finding "$f_chk" "$f_lvl" "$f_val")") + findings+=("${f_lvl}"$'\t'"${ty} ${id}"$'\t'"${nm}"$'\t'"$(_health_finding "$f_chk" "$f_lvl" "$f_val")") done fi case "$ty" in - CT) printf -v ty_col '%b%-5s%b' "${MAGENTA_BRIGHT}" "$ty" "${NC}" ;; - VM) printf -v ty_col '%b%-5s%b' "${BLUE_BRIGHT}" "$ty" "${NC}" ;; - *) printf -v ty_col '%-5s' "$ty" ;; + CT) printf -v ty_col '%b%-5s%b ' "${MAGENTA_BRIGHT}" "$ty" "${NC}" ;; + VM) printf -v ty_col '%b%-5s%b ' "${BLUE_BRIGHT}" "$ty" "${NC}" ;; + *) printf -v ty_col '%-5s ' "$ty" ;; esac + ((show_type)) || ty_col='' upt='-' [[ "$st" == "running" ]] && upt="$(_fmt_duration "$up")" - printf -v line '%-6s %s %s %5s %5s %5s %-11s %s %s' \ - "$id" "$ty_col" \ - "$(_status_color "$st" "$(printf '%-8s' "$st")")" \ - "$cpu" "$mem" "$disk" "$(_truncate "$upt" 11)" \ - "$(_level_color "$lvl" "$(printf '%-7s' "$hname")")" \ - "$(_truncate "$nm" "$name_w")" + printf -v line '%-6s %s%s %5s %5s ' "$id" "$ty_col" \ + "$(_status_color "$st" "$(printf '%-8s' "$st")")" "$cpu" "$mem" + ((show_disk)) && line+="$(printf '%5s ' "$disk")" + ((show_up)) && line+="$(printf '%-11s ' "$(_truncate "$upt" 11)")" + line+="$(_level_color "$lvl" "$(printf '%-7s' "$hname")") $(_truncate "$nm" "$name_w")" if ((draw_boxes)); then _box_content "${CYAN}" "${LINE_V}" $W " ${line}" else @@ -1602,13 +1653,17 @@ print_health_table() { "${GREEN_BRIGHT}" "$n_ok" "${NC}" "${YELLOW_BRIGHT}" "$n_warn" "${NC}" \ "${RED_BRIGHT}" "$n_crit" "${NC}" ((n_ign > 0)) && totals+=" ${n_ign} ignored" + if ((draw_boxes && $(_vis_width "$totals") > W - 6)); then + printf -v totals '%s guests %b%s OK%b %b%s WARN%b %b%s CRIT%b' "${#rows[@]}" \ + "${GREEN_BRIGHT}" "$n_ok" "${NC}" "${YELLOW_BRIGHT}" "$n_warn" "${NC}" \ + "${RED_BRIGHT}" "$n_crit" "${NC}" + fi if ((draw_boxes)); then _draw_line_mid $W if ((${#findings[@]} > 0)); then _box_content "${CYAN}" "${LINE_V}" $W " ${BOLD}Findings:${NC}" for f in "${findings[@]}"; do - _box_content "${CYAN}" "${LINE_V}" $W \ - " $(_level_color "${f%%$'\t'*}" "[$(_level_name "${f%%$'\t'*}")]") $(_truncate "${f#*$'\t'}" $((W - 13)))" + _box_content "${CYAN}" "${LINE_V}" $W " $(_health_finding_line $((W - 6)) "$f")" done _draw_line_mid $W fi @@ -1618,7 +1673,7 @@ print_health_table() { if ((${#findings[@]} > 0)); then printf '\nFindings:\n' for f in "${findings[@]}"; do - printf '%s %s\n' "$(_level_color "${f%%$'\t'*}" "[$(_level_name "${f%%$'\t'*}")]")" "${f#*$'\t'}" + printf '%s\n' "$(_health_finding_line 0 "$f")" done fi printf '\n%s\n' "$totals" @@ -1858,11 +1913,14 @@ _state_save() { } # _check_process_tasks — scan HEALTH_TASKS (uses run_check's "events" and "excused"). -# Advances the task watermark (_N_TASK_TS/_N_TASK), appends failed tasks that finished -# since the last run to "events" (level TAB text) and marks guests with a successful or -# running stop/shutdown/destroy/migrate task in "excused". The first run is a baseline. +# Advances the task watermark (_N_TASK_TS/_N_TASK) and appends failed tasks that finished +# since the last run to "events" (level TAB text); the first run is a baseline. +# A guest is "excused" (stopped on purpose) when its most recent lifecycle task is a +# running or successful stop/shutdown/suspend/destroy/migrate/backup, or a running +# reboot. No time window: /cluster/resources may lag behind the task list. _check_process_tasks() { local row t_end t_upid t_type t_id t_status is_new lvl k + local -A lc_type=() lc_status=() lc_end=() _N_TASK_TS=$_S_TASK_TS _N_TASK=() for k in "${!_S_TASK[@]}"; do @@ -1873,18 +1931,20 @@ _check_process_tasks() { [[ "$t_end" =~ ^[0-9]{1,12}$ && -n "$t_upid" ]] || continue t_end=$((10#$t_end)) case "$t_type" in - qmstop | qmshutdown | vzstop | vzshutdown | qmdestroy | vzdestroy | qmigrate | vzmigrate) - if ((t_end == 0)) || { [[ "$t_status" == "OK" ]] && ((t_end >= _S_RUN_TS)); }; then - excused["$t_id"]=1 - fi - ;; - vzdump) - # Backups in stop mode stop the guest; a running multi-guest job (no id) excuses all. - if ((t_end == 0)) || { [[ "$t_status" == "OK" ]] && ((t_end >= _S_RUN_TS)); }; then - excused["$t_id"]=1 - fi - if ((t_end == 0)) && [[ "$t_id" == "-" ]]; then - excused[all]=1 + qmstart | vzstart | qmresume | vzresume | qmreboot | vzreboot | qmstop | qmshutdown | \ + vzstop | vzshutdown | qmsuspend | vzsuspend | qmdestroy | vzdestroy | qmigrate | \ + vzmigrate | vzdump) + if [[ "$t_id" == "-" ]]; then + # A running multi-guest backup job (no single id) may stop any guest. + if [[ "$t_type" == "vzdump" ]] && ((t_end == 0)); then + excused[all]=1 + fi + elif [[ -z "${lc_end[$t_id]:-}" ]] || ((lc_end[$t_id] != 0)); then + # Rows are sorted by end time with running tasks (0) first; a running task + # stays the most recent one, otherwise later rows replace earlier ones. + lc_type["$t_id"]="$t_type" + lc_status["$t_id"]="$t_status" + lc_end["$t_id"]=$t_end fi ;; esac @@ -1914,6 +1974,21 @@ _check_process_tasks() { fi fi done + for k in "${!lc_type[@]}"; do + case "${lc_type[$k]}" in + qmstop | qmshutdown | vzstop | vzshutdown | qmsuspend | vzsuspend | qmdestroy | vzdestroy | \ + qmigrate | vzmigrate | vzdump) + if ((lc_end[$k] == 0)) || [[ "${lc_status[$k]}" == "OK" || "${lc_status[$k]}" == WARNINGS* ]]; then + excused["$k"]=1 + fi + ;; + qmreboot | vzreboot) + if ((lc_end[$k] == 0)); then + excused["$k"]=1 + fi + ;; + esac + done return 0 } @@ -1921,11 +1996,15 @@ _check_process_tasks() { # from _health_checks and the previous state. Uses run_check's "excused" and fills its # "seen", "vals" and "labels" arrays. # cpu: alerts after HEALTH_CPU_RUNS consecutive runs; mem/disk: immediately; -# n/a metric: previous state kept; down: onboot CRIT, or WARN (latched in the streak -# field) when a guest that was running last time stopped without a matching task. +# metrics of a stopped guest resolve (CPU counter restarts); a metric that is n/a on a +# running guest keeps its previous state silently (marked in run_check's "stale"); +# down: stopped with onboot=1 is CRIT (WARN when stopped by a task), and a guest that +# was running last time and stopped without a task is WARN, latched in the streak +# field until it runs again or a stop task explains it. _check_process_states() { local now="$1" checks row cid chk clvl cval key prev pstreak psince lvl streak local id ty st nm cpu mem disk up + local -A gst=() _N_LVL=() _N_SINCE=() _N_STREAK=() @@ -1934,6 +2013,7 @@ _check_process_states() { IFS=$'\t' read -r id ty st nm cpu mem disk up <<<"$row" [[ -z "$id" ]] && continue labels["$id"]="${ty} ${id} (${nm})" + gst["$id"]="$st" if [[ "$st" == "running" ]] && ! _health_ignored "$id"; then _N_RUN["$id"]=1 fi @@ -1948,11 +2028,18 @@ _check_process_states() { pstreak="${_S_STREAK[$key]:-0}" psince="${_S_SINCE[$key]:-0}" if [[ "$clvl" == "-" ]]; then - # Metric not available (guest not running): keep the previous state. + if [[ "${gst[$cid]:-}" != "running" ]]; then + # Guest not running: its metric alerts resolve and the CPU counter restarts. + vals["$key"]="guest-stopped" + continue + fi + # Running, but the metric is n/a (e.g. VM disk without guest agent): keep the + # previous state without reporting or counting it. if [[ -n "${_S_LVL[$key]:-}" ]]; then _N_LVL["$key"]=$prev _N_SINCE["$key"]=$psince _N_STREAK["$key"]=$pstreak + stale["$key"]=1 fi continue fi @@ -1973,9 +2060,15 @@ _check_process_states() { down) # The streak field latches "stopped unexpectedly" until the guest runs again. if [[ "$cval" == "stopped" ]]; then - if ((pstreak > 0)); then + if [[ -n "${excused[$cid]:-}${excused[all]:-}" ]]; then + # Stopped on purpose: no "unexpected" latch; onboot=1 is only a WARN then. + if ((lvl == 2)); then + lvl=1 + vals["$key"]="onboot-task" + fi + elif ((pstreak > 0)); then streak=1 - elif ((_S_BASE == 1)) && [[ -n "${_S_RUN[$cid]:-}" && -z "${excused[$cid]:-}${excused[all]:-}" ]]; then + elif ((_S_BASE == 1)) && [[ -n "${_S_RUN[$cid]:-}" ]]; then streak=1 fi if ((streak == 1 && lvl < 1)); then @@ -2001,6 +2094,10 @@ _check_process_states() { # _resolved_text CHECK VALUE — description of a check that returned to OK. _resolved_text() { local chk="$1" val="$2" + if [[ "$val" == "guest-stopped" ]]; then + printf '%s alert cleared (guest stopped)' "$chk" + return 0 + fi case "$chk" in cpu) printf 'CPU back to normal (%s)' "$(_pct_text "$val")" ;; mem) printf 'memory back to normal (%s)' "$(_pct_text "$val")" ;; @@ -2062,12 +2159,40 @@ _compose_message() { return 0 } +# _check_keep_unsent — after a total delivery failure keep run_check's "trans_keys" at +# their previous level (counters and latches advance) and keep the old task watermark, +# so exactly these alerts fire again on the next run. +_check_keep_unsent() { + local key + for key in "${trans_keys[@]}"; do + if [[ -n "${_S_LVL[$key]:-}" ]]; then + _N_STREAK["$key"]="${_N_STREAK[$key]:-${_S_STREAK[$key]:-0}}" + _N_LVL["$key"]="${_S_LVL[$key]}" + _N_SINCE["$key"]="${_S_SINCE[$key]:-0}" + elif ((${_N_STREAK[$key]:-0} > 0)); then + _N_LVL["$key"]=0 + _N_SINCE["$key"]=0 + else + unset "_N_LVL[$key]" "_N_SINCE[$key]" "_N_STREAK[$key]" + fi + done + if ((${#events[@]} > 0)); then + _N_TASK_TS=$_S_TASK_TS + _N_TASK=() + for key in "${!_S_TASK[@]}"; do + _N_TASK["$key"]=1 + done + fi + return 0 +} + # run_check — --check: evaluate all local guests and recent tasks, alert on changes. # Exit code: 0 OK, 1 WARN, 2 CRIT, 3 UNKNOWN (config, lock, state or pvesh failure). run_check() { - local dir="${HEALTH_STATE_DIR%/}" file lock_fd now old_umask - local -A excused=() seen=() vals=() labels=() + local dir="${HEALTH_STATE_DIR%/}" file now old_umask + local -A excused=() seen=() vals=() labels=() stale=() local -a events=() keys=() cur_lines=() new_lines=() imp_lines=() res_lines=() ev_lines=() + local -a trans_keys=() [[ -z "$dir" ]] && dir="/" file="${dir%/}/health.state" trap 'exit 3' INT TERM @@ -2079,11 +2204,13 @@ run_check() { have flock || _check_unknown "'flock' not found (util-linux)." old_umask="$(umask)" umask 077 - if ! { exec {lock_fd}>"${dir%/}/check.lock"; } 2>/dev/null; then + # The lock lives on fd 9; child processes get "9>&-" so e.g. a forking MTA + # cannot keep holding it after this run has finished. + if ! { exec 9>"${dir%/}/check.lock"; } 2>/dev/null; then _check_unknown "Could not open the lock file in ${dir}." fi umask "$old_umask" - flock -n "$lock_fd" || _check_unknown "Another --check run is still active (lock held)." + flock -n 9 || _check_unknown "Another --check run is still active (lock held)." fi if ! _state_load "$file"; then ((DRY_RUN == 1)) || _check_unknown "Could not read the state file ${file}." @@ -2096,7 +2223,11 @@ run_check() { local key chk id prev new label since dur text ev ev_lvl rc=0 lvl_new=0 local n_new=0 n_imp=0 n_res=0 n_warn=0 n_crit=0 n_guests=0 - n_guests=${#HEALTH_ROWS[@]} + for key in "${HEALTH_ROWS[@]}"; do + if ! _health_ignored "${key%%$'\t'*}"; then + n_guests=$((n_guests + 1)) + fi + done mapfile -t keys < <(printf '%s\n' "${!seen[@]}" "${!_S_LVL[@]}" | awk 'NF' | sort -t: -k2,2n -k1,1 -u) for key in "${keys[@]}"; do chk="${key%%:*}" @@ -2104,6 +2235,9 @@ run_check() { prev="${_S_LVL[$key]:-0}" new="${_N_LVL[$key]:-0}" label="${labels[$id]:-guest ${id}}" + if [[ -n "${stale[$key]:-}" ]]; then + continue # n/a on a running guest: state kept, not reported + fi if [[ -z "${seen[$key]:-}" ]]; then # Guest vanished or is ignored now: resolve (silently when ignored). if _health_ignored "$id"; then @@ -2124,6 +2258,9 @@ run_check() { rc=$new fi fi + if ((new != prev)); then + trans_keys+=("$key") + fi if ((new > prev)); then new_lines+=("[$(_level_name "$new")] ${text}") n_new=$((n_new + 1)) @@ -2192,9 +2329,8 @@ run_check() { if ((changes > 0)); then log "INFO" "health alert: ${MSG_TITLE}" if ! _notify_all; then - err "All notification channels failed; state not saved so the next run retries." - _CHECK_DONE=1 - exit "$rc" + err "All notification channels failed; the changes stay pending for the next run." + _check_keep_unsent fi fi _N_RUN_TS=$now @@ -2269,7 +2405,7 @@ _notify_ntfy() { cfg+="header = \"Authorization: Bearer ${token}\""$'\n' fi out="$(curl -fsS --max-time 10 --retry 2 --proto '=http,https' -o /dev/null \ - --data-binary "@${_NOTIFY_TMP}" --config - <<<"$cfg" 2>&1)" || rc=$? + --data-binary "@${_NOTIFY_TMP}" --config - <<<"$cfg" 2>&1 9>&-)" || rc=$? rm -f -- "$_NOTIFY_TMP" _NOTIFY_TMP='' if ((rc != 0)); then @@ -2319,7 +2455,7 @@ _notify_mail() { printf 'Content-Transfer-Encoding: 8bit\n' printf 'Auto-Submitted: auto-generated\n' printf '\n%s\n' "$MSG_BODY" - } | timeout 30 "$sm" -t -oi >/dev/null 2>&1 || rc=$? + } | timeout 30 "$sm" -t -oi >/dev/null 2>&1 9>&- || rc=$? if ((rc != 0)); then _notify_warn "mail: sendmail failed (exit ${rc})." return 1 @@ -3115,6 +3251,15 @@ ip_info() { # ============================================================================= main() { + local arg + # --check reports UNKNOWN (3) for every setup problem, including signals and usage + # errors that happen before run_check installs its own traps. + for arg in "$@"; do + if [[ "$arg" == "--check" ]]; then + FATAL_EXIT=3 + trap 'exit 3' INT TERM + fi + done # Load config files — CLI flags set by parse_args below will override these values. _load_config_file /etc/pmanrc _load_config_file "${HOME}/.pmanrc" @@ -3131,7 +3276,10 @@ main() { exit "$FATAL_EXIT" fi if ((FORCE_MODE == 1)); then - warn "--force active: all confirmation prompts will be skipped automatically." + case "$MODE" in + json | health_json | check) warn "--force active: all confirmation prompts will be skipped automatically." >&2 ;; + *) warn "--force active: all confirmation prompts will be skipped automatically." ;; + esac fi require_root require_tools diff --git a/tests/fixtures/resources-100-stopped.json b/tests/fixtures/resources-100-stopped.json new file mode 100644 index 0000000..c4b7d08 --- /dev/null +++ b/tests/fixtures/resources-100-stopped.json @@ -0,0 +1,6 @@ +[ +{"id": "lxc/100", "type": "lxc", "vmid": 100, "node": "mock-host", "name": "ct-one", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 2, "mem": 0, "maxmem": 1073741824, "disk": 0, "maxdisk": 8589934592, "uptime": 0}, +{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "mock-host", "name": "ct-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 1, "mem": 0, "maxmem": 536870912, "disk": 0, "maxdisk": 4294967296, "uptime": 0}, +{"id": "qemu/200", "type": "qemu", "vmid": 200, "node": "mock-host", "name": "vm-one", "status": "running", "template": 0, "cpu": 0.05, "maxcpu": 4, "mem": 536870912, "maxmem": 2147483648, "disk": 0, "maxdisk": 34359738368, "uptime": 90061}, +{"id": "qemu/201", "type": "qemu", "vmid": 201, "node": "mock-host", "name": "vm-fallback", "status": "stopped", "template": 0, "cpu": 0, "maxcpu": 2, "mem": 0, "maxmem": 1073741824, "disk": 0, "maxdisk": 10737418240, "uptime": 0} +] diff --git a/tests/fixtures/resources-empty.json b/tests/fixtures/resources-empty.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/fixtures/resources-empty.json @@ -0,0 +1 @@ +[] diff --git a/tests/fixtures/tasks-failed-new.json b/tests/fixtures/tasks-failed-new.json index b8a4204..1d9eaa3 100644 --- a/tests/fixtures/tasks-failed-new.json +++ b/tests/fixtures/tasks-failed-new.json @@ -3,5 +3,5 @@ {"upid": "UPID:mock-host:00001001:00002001:65000001:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000200, "endtime": 1700000205, "status": "OK"}, {"upid": "UPID:mock-host:00001002:00002002:65000002:vzdump:100:root@pam:", "node": "mock-host", "type": "vzdump", "id": "100", "user": "root@pam", "starttime": 1700000300, "endtime": 1700000400, "status": "WARNINGS: 1"}, {"upid": "UPID:mock-host:00001003:00002003:65000003:qmstart:201:root@pam:", "node": "mock-host", "type": "qmstart", "id": "201", "user": "root@pam", "starttime": 1700000500, "endtime": 1700000501, "status": "start failed: QEMU exited with code 1"}, -{"upid": "UPID:mock-host:00001004:00002004:65000004:vncproxy:200:root@pam:", "node": "mock-host", "type": "vncproxy", "id": "200", "user": "root@pam", "starttime": 1700000600} +{"upid": "UPID:mock-host:00001004:00002004:65000004:vncproxy:200:root@pam:", "node": "mock-host", "type": "vncproxy", "id": "200", "user": "root@pam", "starttime": 1700000600, "status": "RUNNING"} ] diff --git a/tests/fixtures/tasks-restart-after-stop-200.json b/tests/fixtures/tasks-restart-after-stop-200.json new file mode 100644 index 0000000..cfacaa4 --- /dev/null +++ b/tests/fixtures/tasks-restart-after-stop-200.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001100:00003000:65001000:qmshutdown:200:root@pam:", "node": "mock-host", "type": "qmshutdown", "id": "200", "user": "root@pam", "starttime": 1700000095, "endtime": 1700000100, "status": "OK"}, +{"upid": "UPID:mock-host:00001500:00003000:65001000:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000195, "endtime": 1700000200, "status": "OK"} +] diff --git a/tests/fixtures/tasks-shutdown-201.json b/tests/fixtures/tasks-shutdown-201.json new file mode 100644 index 0000000..b41e1d9 --- /dev/null +++ b/tests/fixtures/tasks-shutdown-201.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001000:00003000:65001000:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000005, "status": "OK"}, +{"upid": "UPID:mock-host:00001300:00003000:65001000:qmshutdown:201:root@pam:", "node": "mock-host", "type": "qmshutdown", "id": "201", "user": "root@pam", "starttime": 1700000095, "endtime": 1700000100, "status": "OK"} +] diff --git a/tests/fixtures/tasks-shutdown-old-200.json b/tests/fixtures/tasks-shutdown-old-200.json new file mode 100644 index 0000000..49105b8 --- /dev/null +++ b/tests/fixtures/tasks-shutdown-old-200.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001000:00003000:65001000:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000005, "status": "OK"}, +{"upid": "UPID:mock-host:00001100:00003000:65001000:qmshutdown:200:root@pam:", "node": "mock-host", "type": "qmshutdown", "id": "200", "user": "root@pam", "starttime": 1700000095, "endtime": 1700000100, "status": "OK"} +] diff --git a/tests/fixtures/tasks-suspend-200.json b/tests/fixtures/tasks-suspend-200.json new file mode 100644 index 0000000..b7ec5f4 --- /dev/null +++ b/tests/fixtures/tasks-suspend-200.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001000:00003000:65001000:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000005, "status": "OK"}, +{"upid": "UPID:mock-host:00001200:00003000:65001000:qmsuspend:200:root@pam:", "node": "mock-host", "type": "qmsuspend", "id": "200", "user": "root@pam", "starttime": 1700000095, "endtime": 1700000100, "status": "WARNINGS: 1"} +] diff --git a/tests/fixtures/tasks-vzdump-200.json b/tests/fixtures/tasks-vzdump-200.json index 71914f3..fea1ce9 100644 --- a/tests/fixtures/tasks-vzdump-200.json +++ b/tests/fixtures/tasks-vzdump-200.json @@ -1,4 +1,4 @@ [ {"upid":"UPID:mock-host:00001000:00002000:65000000:qmstart:200:root@pam:","node":"mock-host","type":"qmstart","id":"200","user":"root@pam","starttime":1700000000,"endtime":1700000005,"status":"OK"}, -{"upid":"UPID:mock-host:00001200:00002200:65000200:vzdump:200:root@pam:","node":"mock-host","type":"vzdump","id":"200","user":"root@pam","starttime":@NOW@} +{"upid":"UPID:mock-host:00001200:00002200:65000200:vzdump:200:root@pam:","node":"mock-host","type":"vzdump","id":"200","user":"root@pam","starttime":@NOW@,"status":"RUNNING"} ] diff --git a/tests/fixtures/tasks-vzshutdown-100.json b/tests/fixtures/tasks-vzshutdown-100.json new file mode 100644 index 0000000..2dc95c0 --- /dev/null +++ b/tests/fixtures/tasks-vzshutdown-100.json @@ -0,0 +1,4 @@ +[ +{"upid": "UPID:mock-host:00001000:00003000:65001000:qmstart:200:root@pam:", "node": "mock-host", "type": "qmstart", "id": "200", "user": "root@pam", "starttime": 1700000000, "endtime": 1700000005, "status": "OK"}, +{"upid": "UPID:mock-host:00001400:00003000:65001000:vzshutdown:100:root@pam:", "node": "mock-host", "type": "vzshutdown", "id": "100", "user": "root@pam", "starttime": 1700000095, "endtime": 1700000100, "status": "OK"} +] diff --git a/tests/mock-bin/curl b/tests/mock-bin/curl index 6d908d3..785320e 100755 --- a/tests/mock-bin/curl +++ b/tests/mock-bin/curl @@ -3,6 +3,11 @@ # Appends argv to $PMAN_MOCK_CURL_LOG, the stdin config to ${PMAN_MOCK_CURL_LOG}.cfg and # every @file body to ${PMAN_MOCK_CURL_LOG}.body; exits ${PMAN_MOCK_CURL_RC:-0}. set -euo pipefail +if [[ -n "${PMAN_MOCK_FD_LOG:-}" ]]; then + for fd in /proc/"$$"/fd/*; do + printf 'curl %s\n' "$(readlink "$fd" || true)" + done >>"$PMAN_MOCK_FD_LOG" +fi log="${PMAN_MOCK_CURL_LOG:-}" cfg="$(cat)" if [[ -n "$log" ]]; then diff --git a/tests/mock-bin/pvesh b/tests/mock-bin/pvesh index a40f1a2..548b7ce 100755 --- a/tests/mock-bin/pvesh +++ b/tests/mock-bin/pvesh @@ -3,8 +3,18 @@ # PMAN_MOCK_RESOURCES / PMAN_MOCK_TASKS: fixture files replacing the inline defaults. # The token @NOW@ in a fixture is replaced with the current epoch. # PMAN_MOCK_PVESH_FAIL=1 makes every call fail; PMAN_MOCK_PVESH_SLEEP=N delays each call. +# PMAN_MOCK_PVESH_LOG: append argv; PMAN_MOCK_FD_LOG: append the targets of open fds. set -euo pipefail +if [[ -n "${PMAN_MOCK_PVESH_LOG:-}" ]]; then + printf '%s\n' "$*" >>"$PMAN_MOCK_PVESH_LOG" +fi +if [[ -n "${PMAN_MOCK_FD_LOG:-}" ]]; then + for fd in /proc/"$$"/fd/*; do + printf 'pvesh %s\n' "$(readlink "$fd" || true)" + done >>"$PMAN_MOCK_FD_LOG" +fi + if [[ -n "${PMAN_MOCK_PVESH_SLEEP:-}" ]]; then sleep "$PMAN_MOCK_PVESH_SLEEP" fi diff --git a/tests/mock-bin/sendmail b/tests/mock-bin/sendmail index 4b3ec44..5286fd6 100755 --- a/tests/mock-bin/sendmail +++ b/tests/mock-bin/sendmail @@ -2,6 +2,11 @@ # Mock sendmail: appends argv and the message (stdin) to $PMAN_MOCK_SENDMAIL_LOG; # exits ${PMAN_MOCK_SENDMAIL_RC:-0}. set -euo pipefail +if [[ -n "${PMAN_MOCK_FD_LOG:-}" ]]; then + for fd in /proc/"$$"/fd/*; do + printf 'sendmail %s\n' "$(readlink "$fd" || true)" + done >>"$PMAN_MOCK_FD_LOG" +fi log="${PMAN_MOCK_SENDMAIL_LOG:-}" msg="$(cat)" if [[ -n "$log" ]]; then diff --git a/tests/mock-bin/uname b/tests/mock-bin/uname new file mode 100755 index 0000000..b8e4cd2 --- /dev/null +++ b/tests/mock-bin/uname @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Mock uname: "-n" reports the mock node name (with a domain part, like a FQDN hostname); +# every other invocation is passed through to the real uname. +set -euo pipefail +if [[ "${1:-}" == "-n" ]]; then + echo "mock-host.example.invalid" + exit 0 +fi +for real in /usr/bin/uname /bin/uname; do + if [[ -x "$real" ]]; then + exec "$real" "$@" + fi +done +exit 1 diff --git a/tests/run.sh b/tests/run.sh index ca41483..44d5ff0 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -355,9 +355,10 @@ _new_state_dir() { printf '%s' "$d" } -_expect_rc "health flags: --check with --json exits 1" 1 "$SCRIPT" --check --json -_expect_rc "health flags: --check with --health exits 1" 1 "$SCRIPT" --check --health -_expect_rc "health flags: --check with --filter exits 1" 1 "$SCRIPT" --check --filter running +_expect_rc "health flags: --check with --json exits 3 (UNKNOWN)" 3 "$SCRIPT" --check --json +_expect_rc "health flags: --check with --health exits 3 (UNKNOWN)" 3 "$SCRIPT" --check --health +_expect_rc "health flags: --check with --filter exits 3 (UNKNOWN)" 3 "$SCRIPT" --check --filter running +_expect_rc "health flags: --check with an unknown option exits 3" 3 "$SCRIPT" --check --bogus _expect_rc "health flags: --dry-run without --check exits 1" 1 "$SCRIPT" --dry-run _expect_rc "health flags: --test-notify with --list exits 1" 1 "$SCRIPT" --test-notify --list _expect_rc "test-notify: no channel configured exits 1" 1 "$SCRIPT" --test-notify @@ -715,18 +716,69 @@ else _notify_env n3 sd="$(_new_state_dir)" _chk "$sd" "${NENV[@]}" "$HOT" PMAN_MOCK_CURL_RC=7 PMAN_MOCK_SENDMAIL_RC=75 - if [[ "$chk_rc" == "2" && ! -e "$sd/health.state" ]]; then - _pass "notify: all channels fail -> exit reflects health, state not saved" + if [[ "$chk_rc" == "2" && -e "$sd/health.state" ]] && ! grep -q $'^C\tmem:100\t2' "$sd/health.state" && + grep -q $'^R\t100$' "$sd/health.state"; then + _pass "notify: all channels fail -> exit reflects health, state saved, alerts kept pending" else _fail "notify: all-fail handling wrong (exit $chk_rc)" fi rm -f "$nlog".* _chk "$sd" "${NENV[@]}" "$HOT" - if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" && -e "$sd/health.state" ]] && grep -qF 'Priority: urgent' "$nlog.curl.cfg"; then + if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" ]] && grep -qF 'Priority: urgent' "$nlog.curl.cfg" && + grep -qF '[CRIT] CT 100 (ct-one): memory 96%' "$nlog.curl.body"; then _pass "notify: next run resends after a total failure" else _fail "notify: alert not resent after a total failure" fi + _chk "$sd" "${NENV[@]}" "$HOT" + if [[ "$(_count "$nlog.curl.body" 'memory 96%')" == "1" ]]; then + _pass "notify: resent alert is delivered exactly once" + else + _fail "notify: resent alert delivered more than once" + fi + + # CPU counter keeps advancing while delivery fails; the WARN fires once delivery works. + _notify_env n7 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "$CPU_HOT" + _chk "$sd" "${NENV[@]}" "$CPU_HOT" + _chk "$sd" "${NENV[@]}" "$CPU_HOT" PMAN_MOCK_CURL_RC=7 PMAN_MOCK_SENDMAIL_RC=75 + if [[ "$chk_rc" == "1" ]] && grep -q $'^C\tcpu:200\t0\t0\t3$' "$sd/health.state"; then + _pass "notify: CPU counter advances although delivery failed" + else + _fail "notify: CPU counter not saved after delivery failure (exit $chk_rc)" + fi + rm -f "$nlog".* + _chk "$sd" "${NENV[@]}" "$CPU_HOT" + if [[ "$(_count "$nlog.curl" .)" == "1" ]] && grep -qF '[WARN] VM 200 (vm-one): CPU 90%' "$nlog.curl.body"; then + _pass "notify: pending CPU alert delivered on the next run" + else + _fail "notify: pending CPU alert not delivered" + fi + + # Tasks: failed task events are resent after a total delivery failure. + _notify_env n8 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-old.json" + _chk "$sd" "${NENV[@]}" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-new.json" PMAN_MOCK_CURL_RC=7 PMAN_MOCK_SENDMAIL_RC=75 + rm -f "$nlog".* + _chk "$sd" "${NENV[@]}" "PMAN_MOCK_TASKS=$FIXTURES/tasks-failed-new.json" + if [[ "$chk_rc" == "2" && "$(_count "$nlog.curl" .)" == "1" ]] && grep -qF 'task qmstart 201 failed' "$nlog.curl.body"; then + _pass "notify: unsent task events are resent on the next run" + else + _fail "notify: unsent task events lost (exit $chk_rc)" + fi + + # The --check lock (fd 9) must not leak into child processes. + _notify_env n9 + sd="$(_new_state_dir)" + _chk "$sd" "${NENV[@]}" "$HOT" "PMAN_MOCK_FD_LOG=$nlog.fds" + if [[ -s "$nlog.fds" ]] && grep -q '^sendmail ' "$nlog.fds" && grep -q '^curl ' "$nlog.fds" && + grep -q '^pvesh ' "$nlog.fds" && ! grep -q 'check.lock' "$nlog.fds"; then + _pass "notify: lock fd is closed for pvesh, curl and sendmail" + else + _fail "notify: lock fd inherited by a child process" + fi _notify_env n4 tn_rc=0 @@ -779,6 +831,116 @@ _chk "$sd" _chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-vzdump-200.json" _expect_chk "--check: stop during a running vzdump (stop mode) is OK" 0 "PMAN OK" +# Review regressions ------------------------------------------------------- +# Metric alerts of a guest that stops resolve instead of sticking forever. +sd="$(_new_state_dir)" +_chk "$sd" "$HOT" +_chk "$sd" "PMAN_MOCK_RESOURCES=$FIXTURES/resources-100-stopped.json" "PMAN_MOCK_TASKS=$FIXTURES/tasks-vzshutdown-100.json" +_expect_chk "--check: metric alerts resolve when the guest stops" 0 "[RESOLVED] CT 100 (ct-one): mem alert cleared (guest stopped)" +_chk "$sd" "PMAN_MOCK_RESOURCES=$FIXTURES/resources-100-stopped.json" "PMAN_MOCK_TASKS=$FIXTURES/tasks-vzshutdown-100.json" +if [[ "$chk_rc" == "0" && "$(wc -l <<<"$chk_out")" == "1" ]]; then + _pass "--check: stopped guest stays quiet afterwards" +else + _fail "--check: stopped guest still reported (exit $chk_rc)" +fi +# A metric that is n/a on a running guest keeps its state but is not reported or counted. +sd="$(_new_state_dir)" +printf '%s\n' '# pman-health-state v1' "W 1 1700000005" "C disk:200 2 1 0" >"$sd/health.state" +chmod 600 "$sd/health.state" +_chk "$sd" +if [[ "$chk_rc" == "0" ]] && ! grep -q 'disk' <<<"$chk_out" && grep -q $'^C\tdisk:200\t2' "$sd/health.state"; then + _pass "--check: n/a metric on a running guest is kept silently" +else + _fail "--check: n/a metric on a running guest reported or dropped (exit $chk_rc)" +fi + +# Shutdown task older than the last run (pvestatd lag) still explains the stop. +sd="$(_new_state_dir)" +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-shutdown-old-200.json" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-shutdown-old-200.json" +_expect_chk "--check: shutdown seen before the stop (status lag) is OK" 0 "PMAN OK" +sd="$(_new_state_dir)" +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-suspend-200.json" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-suspend-200.json" +_expect_chk "--check: hibernate (qmsuspend, WARNINGS) is not an unexpected stop" 0 "PMAN OK" +sd="$(_new_state_dir)" +_chk "$sd" "PMAN_MOCK_TASKS=$FIXTURES/tasks-restart-after-stop-200.json" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-restart-after-stop-200.json" +_expect_chk "--check: stop after a later start task is unexpected" 1 "stopped unexpectedly" +sd="$(_new_state_dir)" +_chk "$sd" +_chk "$sd" "$STOPPED_200" +_chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-shutdown-old-200.json" +_expect_chk "--check: a stop task found later clears the unexpected-stop latch" 0 "[RESOLVED] VM 200 (vm-one)" + +# Tasks are read with --source all (running tasks are not archived yet). +pv_log="$TEST_TMP/pvesh.args" +_chk "$(_new_state_dir)" "PMAN_MOCK_PVESH_LOG=$pv_log" +if grep -q -- '^get /nodes/mock-host/tasks --source all --limit 200 --output-format json$' "$pv_log"; then + _pass "--check: task list uses --source all" +else + _fail "--check: task list does not use --source all" +fi + +# onboot=1 guest stopped by an admin task is WARN, unexplained it stays CRIT. +sd="$(_new_state_dir)" +_chk "$sd" PMAN_MOCK_ONBOOT_IDS=201 "PMAN_MOCK_TASKS=$FIXTURES/tasks-shutdown-201.json" +_expect_chk "--check: onboot guest stopped by a task is WARN" 1 "[WARN] VM 201 (vm-fallback): stopped by a Proxmox task although onboot=1" + +# --force must not break machine-readable output. +force_json="$("$SCRIPT" --health --json --force 2>/dev/null)" +force_list_json="$("$SCRIPT" --json --force 2>/dev/null)" +if python3 -c 'import json,sys; json.loads(sys.argv[1]); json.loads(sys.argv[2])' "$force_json" "$force_list_json" 2>/dev/null; then + _pass "--force: --json and --health --json stay valid JSON" +else + _fail "--force: JSON output corrupted by the --force warning" +fi +_chk "$(_new_state_dir)" -- --force +if [[ "$(head -1 <<<"$chk_out")" == "PMAN OK -"* ]]; then + _pass "--force: --check keeps the Nagios summary as first line" +else + _fail "--force: --check first line is not the summary" +fi + +# Ignored guests are not counted; an empty node is not an error. +_chk "$(_new_state_dir)" HEALTH_IGNORE_IDS=100 +_expect_chk "--check: ignored guests are not counted" 0 "; 3 guests on mock-host" +empty_rc=0 +empty_out="$(PMAN_MOCK_RESOURCES="$FIXTURES/resources-empty.json" "$SCRIPT" --health 2>&1)" || empty_rc=$? +if [[ "$empty_rc" == "0" && "$empty_out" == *"No guests found on node mock-host."* ]]; then + _pass "--health: no guests exits 0 with a message" +else + _fail "--health: no guests exit $empty_rc" +fi + +# Narrow terminals: no line wider than COLUMNS, finding text never cut. +narrow_out="$(COLUMNS=50 LANG=C.UTF-8 PMAN_MOCK_RESOURCES="$FIXTURES/resources-hot.json" PMAN_MOCK_ONBOOT_IDS=101 "$SCRIPT" --health)" +if python3 -c 'import sys; assert max(len(l) for l in sys.stdin.read().splitlines()) <= 50' <<<"$narrow_out" 2>/dev/null && + grep -qF 'stopped although onboot=1' <<<"$narrow_out" && grep -qF 'memory 96% (>= 95%)' <<<"$narrow_out"; then + _pass "--health: fits 50 columns without cutting findings" +else + _fail "--health: narrow layout overflows or cuts findings" +fi + +# onboot lookup reads the current section of /etc/pve configs; node name from uname -n. +pve_etc="$TEST_TMP/pve-etc" +mkdir -p "$pve_etc/qemu-server" "$pve_etc/lxc" +printf '%s\n' 'name: a' 'onboot: 1' '[snap1]' 'onboot: 0' >"$pve_etc/qemu-server/555.conf" +printf '%s\n' 'hostname: b' '[snap1]' 'onboot: 1' >"$pve_etc/lxc/556.conf" +saved_etc="$PVE_ETC_DIR" +PVE_ETC_DIR="$pve_etc" +if _guest_onboot 555 VM && ! _guest_onboot 556 CT && PMAN_MOCK_ONBOOT_IDS=557 _guest_onboot 557 VM; then + _pass "_guest_onboot: reads the current config section, falls back to qm/pct" +else + _fail "_guest_onboot: wrong onboot detection" +fi +PVE_ETC_DIR="$saved_etc" +if [[ "$(_local_node)" == "mock-host" ]]; then + _pass "_local_node: uname -n up to the first dot" +else + _fail "_local_node: unexpected node name" +fi + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From 24d5812a2d400cb5e69b6a9f0fb69ea8e5ddaf7c Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 19:07:05 +0200 Subject: [PATCH 7/8] fix(health): harden notification and state handling - http:// NTFY_URL together with NTFY_TOKEN_FILE is a config error (--check exits 3, --test-notify exits 1). - Token file: directory must be owned by root/current user and not group/world-writable; the token is read through one opened descriptor that is verified via /proc/self/fd (regular file, owner, mode 0600). - Lock file opened for append (never truncated); the parent of HEALTH_STATE_DIR must not be group/world-writable. - qm/pct config fallback runs with timeout 10. - curl: -q first, --proto =https when a token is used, no --retry. - json_escape escapes remaining control characters as \u00XX. - --test-notify removes its temp body file on exit and on INT/TERM. - _truncate keeps UTF-8 characters intact. - Tests for each item where practical. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 12 +++-- README.md | 5 +- proxmox-manager.sh | 110 ++++++++++++++++++++++++++++++++++++++------ tests/mock-bin/curl | 4 ++ tests/run.sh | 91 +++++++++++++++++++++++++++++++++++- 5 files changed, 202 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b78db..ae9910a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,11 +35,17 @@ Versions follow [Semantic Versioning](https://semver.org/). - Outbound traffic is now possible, but only when notifications are configured (opt-in). ### Security -- The ntfy access token is read from a `0600` file owned by the caller; an inline `NTFY_TOKEN` in a - config file is rejected. The token, URL and headers reach `curl` only via its stdin config, never - argv; redirects are not followed, and the token is never sent over plain `http://`. +- The ntfy access token is read from a `0600` file owned by the caller in a directory nobody else can + write; the open descriptor is verified before reading (no swap between check and use). An inline + `NTFY_TOKEN` in a config file and a token with an `http://` URL are configuration errors. +- The token, URL and headers reach `curl` only via its stdin config, never argv; `curl -q` ignores + `~/.curlrc`, redirects are not followed, a token forces `https` only, and requests are not retried. - Logs name only the ntfy host, never the topic path or the token. - Mail headers are built from validated single-line values (no header injection). +- JSON output escapes all control characters; `_truncate` never cuts inside a UTF-8 character. +- `qm`/`pct` config fallbacks run with a timeout; the `--check` lock descriptor is not inherited by + child processes and the lock file is never truncated; the state directory's parent must not be + group/world-writable. - The `--check` state lives in a `0700` directory, is written atomically with mode `0600`, is parsed as validated data (never sourced), and parallel runs are prevented with `flock`. diff --git a/README.md b/README.md index 9c30bfd..91f241d 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,8 @@ install -d -m 700 /etc/pman install -m 600 /dev/null /etc/pman/ntfy.token && nano /etc/pman/ntfy.token ``` -The token is only sent over `https://`. **E-mail** uses the local `sendmail` (postfix, which PVE +The token file and its directory must not be writable by others; a token requires an `https://` +URL (`http://` plus a token is a configuration error). **E-mail** uses the local `sendmail` (postfix, which PVE installs for its own notifications, works out of the box when it can relay): ```bash @@ -362,7 +363,7 @@ shellcheck proxmox-manager.sh tests/run.sh ``` -100 tests covering `validate_vmid`, `validate_snapshot_name`, `ip_info`, `--filter`, CLI flags, config +131 tests covering `validate_vmid`, `validate_snapshot_name`, `ip_info`, `--filter`, CLI flags, config parsing, the health view, the `--check` engine and notifications (mocked `pvesh`, `curl`, `sendmail`; no network, no root). diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 3b8ba95..3cd8338 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -283,6 +283,22 @@ _owner_mode_ok() { return 0 } +# _trusted_dir DIR — true when DIR is a directory (no symlink) owned by root or the +# current user and not writable by group or others, so nobody else can swap its entries. +_trusted_dir() { + local dir="$1" owner mode + [[ -d "$dir" && ! -L "$dir" ]] || return 1 + owner="$(stat -c '%u' -- "$dir" 2>/dev/null || printf 'invalid')" + mode="$(stat -c '%a' -- "$dir" 2>/dev/null || printf 'invalid')" + if [[ "$owner" != "0" && "$owner" != "$EUID" ]] || [[ ! "$mode" =~ ^[0-7]{3,4}$ ]]; then + return 1 + fi + if ((8#$mode & 8#022)); then + return 1 + fi + return 0 +} + # _prepare_log_file — require a private regular file in a trusted directory. _prepare_log_file() { [[ -z "$LOG_FILE" ]] && return 0 @@ -397,9 +413,13 @@ _validate_health_config() { elif ! _owner_mode_ok "$NTFY_TOKEN_FILE" 077; then err "NTFY_TOKEN_FILE must be owned by the current user with mode 0600." rc=1 + elif ! _trusted_dir "$(dirname -- "$NTFY_TOKEN_FILE")"; then + err "The directory of NTFY_TOKEN_FILE must be owned by root or the current user and not group/world-writable." + rc=1 fi if [[ "$NTFY_URL" == http://* ]]; then - _config_warn "NTFY_URL uses plain http; the access token is sent unencrypted." + err "NTFY_TOKEN_FILE requires an https:// NTFY_URL; the token is never sent over plain http." + rc=1 fi fi if [[ -n "$HEALTH_MAIL_TO" ]]; then @@ -679,6 +699,18 @@ json_escape() { s="${s//$'\n'/\\n}" s="${s//$'\r'/\\r}" s="${s//$'\t'/\\t}" + # Any other control character becomes \u00XX. + if [[ "$s" == *[$'\x01'-$'\x1f']* ]]; then + local out='' c i + for ((i = 0; i < ${#s}; i++)); do + c="${s:i:1}" + if [[ "$c" == [$'\x01'-$'\x1f'] ]]; then + printf -v c '\\u%04x' "'$c" + fi + out+="$c" + done + s="$out" + fi printf '%s' "$s" } @@ -827,9 +859,20 @@ _truncate() { local text="$1" max="$2" if (($(_vis_width "$text") <= max)); then printf '%s' "$text" - else - printf '%s%s' "${text:0:max-1}" "$TRUNC_MARK" + return 0 fi + # Copy whole UTF-8 characters (lead byte plus continuation bytes) up to max-1 columns. + local out='' i=0 n=0 len=${#text} + while ((n < max - 1 && i < len)); do + out+="${text:i:1}" + i=$((i + 1)) + while ((i < len)) && [[ "${text:i:1}" == [$'\x80'-$'\xbf'] ]]; do + out+="${text:i:1}" + i=$((i + 1)) + done + n=$((n + 1)) + done + printf '%s%s' "$out" "$TRUNC_MARK" } # _term_cols — terminal width (fallback 80). @@ -1386,8 +1429,8 @@ _guest_onboot() { return 1 fi case "$ty" in - CT) cfg="$(pct config "$id" 2>/dev/null 9>&- || true)" ;; - VM) cfg="$(qm config "$id" 2>/dev/null 9>&- || true)" ;; + CT) cfg="$(timeout 10 pct config "$id" 2>/dev/null 9>&- || true)" ;; + VM) cfg="$(timeout 10 qm config "$id" 2>/dev/null 9>&- || true)" ;; esac grep -qE '^onboot:[[:space:]]*1[[:space:]]*$' <<<"$cfg" } @@ -1810,11 +1853,16 @@ _check_on_exit() { # _check_state_dir DIR — create DIR (0700) if needed; require a private, owned directory. _check_state_dir() { - local dir="$1" + local dir="$1" parent if [[ -L "$dir" ]]; then err "HEALTH_STATE_DIR must not be a symlink." return 1 fi + parent="$(dirname -- "$dir")" + if [[ -d "$parent" ]] && ! _trusted_dir "$parent"; then + err "The parent of HEALTH_STATE_DIR (${parent}) must be owned by root or the current user and not group/world-writable." + return 1 + fi if [[ ! -d "$dir" ]]; then if ! (umask 077 && mkdir -p -- "$dir") 2>/dev/null; then err "Could not create HEALTH_STATE_DIR ${dir}." @@ -2206,7 +2254,7 @@ run_check() { umask 077 # The lock lives on fd 9; child processes get "9>&-" so e.g. a forking MTA # cannot keep holding it after this run has finished. - if ! { exec 9>"${dir%/}/check.lock"; } 2>/dev/null; then + if ! { exec 9>>"${dir%/}/check.lock"; } 2>/dev/null; then _check_unknown "Could not open the lock file in ${dir}." fi umask "$old_umask" @@ -2360,11 +2408,34 @@ _notify_warn() { warn "$*" >&2 } +# _read_ntfy_token — print the token from NTFY_TOKEN_FILE. The file is opened once and +# the open descriptor itself is checked (regular file, owner, mode 0600), so swapping the +# path between validation and use cannot leak another file; its directory must be trusted. +_read_ntfy_token() { + local fd info line='' + if [[ -L "$NTFY_TOKEN_FILE" ]] || ! _trusted_dir "$(dirname -- "$NTFY_TOKEN_FILE")"; then + return 1 + fi + if ! { exec {fd}<"$NTFY_TOKEN_FILE"; } 2>/dev/null; then + return 1 + fi + info="$(stat -L -c '%F|%u|%a' -- "/proc/self/fd/${fd}" 2>/dev/null || true)" + if [[ "$info" =~ ^regular(\ empty)?\ file\|([0-9]+)\|([0-7]{3,4})$ ]] && + [[ "${BASH_REMATCH[2]}" == "$EUID" ]] && ((!(8#${BASH_REMATCH[3]} & 8#077))); then + IFS= read -r line <&"$fd" || true + else + line='' + fi + exec {fd}<&- + [[ "$line" =~ ^[A-Za-z0-9_.-]{1,256}$ ]] || return 1 + printf '%s' "$line" +} + # _notify_ntfy — send the message to NTFY_URL. URL, headers and the optional token are # passed to curl via its stdin config, never in argv; the body comes from a private temp # file. Only the host is logged, never the topic path or the token. _notify_ntfy() { - local host token='' tag title cfg rc=0 out + local host token='' tag title cfg rc=0 out proto host="${NTFY_URL#*://}" host="${host%%/*}" if ! have curl; then @@ -2376,11 +2447,8 @@ _notify_ntfy() { _notify_warn "ntfy: refusing to send the access token over plain http to ${host}; use https." return 1 fi - if ! IFS= read -r token <"$NTFY_TOKEN_FILE" && [[ -z "$token" ]]; then - token='' - fi - if [[ ! "$token" =~ ^[A-Za-z0-9_.-]{1,256}$ ]]; then - _notify_warn "ntfy: token in NTFY_TOKEN_FILE is empty or has unsupported characters." + if ! token="$(_read_ntfy_token)"; then + _notify_warn "ntfy: NTFY_TOKEN_FILE is unsafe, unreadable or has an invalid token." return 1 fi fi @@ -2404,7 +2472,11 @@ _notify_ntfy() { if [[ -n "$token" ]]; then cfg+="header = \"Authorization: Bearer ${token}\""$'\n' fi - out="$(curl -fsS --max-time 10 --retry 2 --proto '=http,https' -o /dev/null \ + # -q first: ignore ~/.curlrc. No --retry: a retried POST can duplicate a push, and + # unsent changes are resent by the next --check run anyway. + proto='=http,https' + [[ -n "$token" ]] && proto='=https' + out="$(curl -q -fsS --max-time 10 --proto "$proto" -o /dev/null \ --data-binary "@${_NOTIFY_TMP}" --config - <<<"$cfg" 2>&1 9>&-)" || rc=$? rm -f -- "$_NOTIFY_TMP" _NOTIFY_TMP='' @@ -2487,6 +2559,14 @@ _notify_all() { return 0 } +# _notify_cleanup — EXIT trap of --test-notify: remove a pending ntfy body file. +_notify_cleanup() { + if [[ -n "$_NOTIFY_TMP" ]]; then + rm -f -- "$_NOTIFY_TMP" + _NOTIFY_TMP='' + fi +} + # run_test_notify — send a test message through all configured channels. run_test_notify() { if ! _notify_channels_configured; then @@ -2494,6 +2574,8 @@ run_test_notify() { exit 1 fi local node + trap '_notify_cleanup' EXIT + trap 'exit 1' INT TERM node="$(_local_node || printf 'unknown')" MSG_TITLE="pman ${node}: test notification" MSG_BODY="Test message from proxmox-manager on ${node}. Alerts from --check will arrive here." diff --git a/tests/mock-bin/curl b/tests/mock-bin/curl index 785320e..1e970fe 100755 --- a/tests/mock-bin/curl +++ b/tests/mock-bin/curl @@ -2,6 +2,7 @@ # Mock curl: never touches the network. # Appends argv to $PMAN_MOCK_CURL_LOG, the stdin config to ${PMAN_MOCK_CURL_LOG}.cfg and # every @file body to ${PMAN_MOCK_CURL_LOG}.body; exits ${PMAN_MOCK_CURL_RC:-0}. +# PMAN_MOCK_CURL_SLEEP=N delays the call; PMAN_MOCK_FD_LOG records open fds. set -euo pipefail if [[ -n "${PMAN_MOCK_FD_LOG:-}" ]]; then for fd in /proc/"$$"/fd/*; do @@ -10,6 +11,9 @@ if [[ -n "${PMAN_MOCK_FD_LOG:-}" ]]; then fi log="${PMAN_MOCK_CURL_LOG:-}" cfg="$(cat)" +if [[ -n "${PMAN_MOCK_CURL_SLEEP:-}" ]]; then + sleep "$PMAN_MOCK_CURL_SLEEP" +fi if [[ -n "$log" ]]; then printf '%s\n' "$*" >>"$log" printf '%s\n---\n' "$cfg" >>"${log}.cfg" diff --git a/tests/run.sh b/tests/run.sh index 44d5ff0..6fa0643 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -674,6 +674,11 @@ else else _fail "notify: hot run did not send exactly one ntfy + one mail (exit $chk_rc)" fi + if grep -q '^-q -fsS --max-time 10 --proto =https -o /dev/null --data-binary @' "$nlog.curl" && ! grep -q -- '--retry' "$nlog.curl"; then + _pass "notify: curl ignores .curlrc (-q first), https only with a token, no retry" + else + _fail "notify: unexpected curl arguments: $(head -1 "$nlog.curl")" + fi if grep -q 'tk_secret123' "$nlog.curl.cfg" && ! grep -q 'tk_secret123' "$nlog.curl" && ! grep -q 'pman-topic' "$nlog.curl"; then _pass "notify: token and topic only in curl stdin config, never in argv" else @@ -802,10 +807,16 @@ else env "PMAN_MOCK_CURL_LOG=$nlog.curl" NTFY_URL=http://ntfy.example.invalid/pman-topic "NTFY_TOKEN_FILE=$ntfy_tok" \ "$SCRIPT" --test-notify >/dev/null 2>&1 || tn_rc=$? if [[ "$tn_rc" == "1" && ! -e "$nlog.curl" ]]; then - _pass "notify: token is never sent over plain http (channel fails)" + _pass "notify: token with plain http is rejected by --test-notify" else _fail "notify: token sent over http or wrong exit ($tn_rc)" fi + _chk "$(_new_state_dir)" "PMAN_MOCK_CURL_LOG=$nlog.curl" NTFY_URL=http://ntfy.example.invalid/pman-topic "NTFY_TOKEN_FILE=$ntfy_tok" + if [[ "$chk_rc" == "3" && ! -e "$nlog.curl" ]]; then + _pass "notify: token with plain http makes --check exit 3" + else + _fail "notify: http + token not rejected by --check (exit $chk_rc)" + fi # Header injection: CR/LF in the title must not create extra headers. _notify_env n6 @@ -831,6 +842,84 @@ _chk "$sd" _chk "$sd" "$STOPPED_200" "PMAN_MOCK_TASKS=$FIXTURES/tasks-vzdump-200.json" _expect_chk "--check: stop during a running vzdump (stop mode) is OK" 0 "PMAN OK" +# Security review regressions ---------------------------------------------- +# Token file: directory must be trusted; the open descriptor is verified. +tok_dir="$TEST_TMP/tokdir" +mkdir -p "$tok_dir" +chmod 777 "$tok_dir" +printf 'tk_other\n' >"$tok_dir/t" +chmod 600 "$tok_dir/t" +_chk "$(_new_state_dir)" NTFY_URL=https://ntfy.example.invalid/pman-topic "NTFY_TOKEN_FILE=$tok_dir/t" +_expect_chk "security: token file in a world-writable directory makes --check exit 3" 3 "PMAN UNKNOWN" +chmod 700 "$tok_dir" +saved_tf="$NTFY_TOKEN_FILE" +NTFY_TOKEN_FILE="$tok_dir/t" +tok_ok="$(_read_ntfy_token)" || tok_ok='' +printf 'tk_leak\n' >"$tok_dir/other" +chmod 644 "$tok_dir/other" +NTFY_TOKEN_FILE="$tok_dir/other" +tok_bad=0 +_read_ntfy_token >/dev/null 2>&1 || tok_bad=$? +ln -s "$tok_dir/t" "$tok_dir/link" +NTFY_TOKEN_FILE="$tok_dir/link" +tok_link=0 +_read_ntfy_token >/dev/null 2>&1 || tok_link=$? +NTFY_TOKEN_FILE="$saved_tf" +if [[ "$tok_ok" == "tk_other" && "$tok_bad" == "1" && "$tok_link" == "1" ]]; then + _pass "security: token read via a verified descriptor (mode and symlink checked)" +else + _fail "security: token reader accepted an unsafe file ($tok_ok/$tok_bad/$tok_link)" +fi + +# State dir: parent must not be group/world-writable; lock file is never truncated. +open_parent="$TEST_TMP/open-parent" +mkdir -p "$open_parent" +chmod 777 "$open_parent" +_chk "$open_parent/state" +if [[ "$chk_rc" == "3" && ! -e "$open_parent/state" ]]; then + _pass "security: HEALTH_STATE_DIR below a world-writable parent is refused" +else + _fail "security: world-writable parent accepted (exit $chk_rc)" +fi +sd="$(_new_state_dir)" +printf 'keep\n' >"$sd/check.lock" +chmod 600 "$sd/check.lock" +_chk "$sd" +if [[ "$chk_rc" == "0" && "$(cat "$sd/check.lock")" == "keep" ]]; then + _pass "security: lock file is opened for append, not truncated" +else + _fail "security: lock file truncated (exit $chk_rc)" +fi + +if [[ "$(json_escape $'a\x01b\x1fc\n\t"')" == 'a\u0001b\u001fc\n\t\"' ]]; then + _pass "json_escape: control characters escaped as \\u00XX" +else + _fail "json_escape: control characters not escaped" +fi +utf_cut="$(_truncate 'aüüüüüüüüü' 5)" +if python3 -c 'import sys; sys.stdin.buffer.read().decode("utf-8")' <<<"$utf_cut" 2>/dev/null && + [[ "$(_vis_width "$utf_cut")" == "5" ]]; then + _pass "_truncate: never cuts inside a UTF-8 character" +else + _fail "_truncate: produced invalid UTF-8 or wrong width" +fi + +# --test-notify removes the ntfy body temp file when interrupted. +tn_tmp="$TEST_TMP/tn-tmp" +mkdir -p "$tn_tmp" +NTFY_URL=https://ntfy.example.invalid/pman-topic PMAN_MOCK_CURL_SLEEP=1 TMPDIR="$tn_tmp" \ + "$SCRIPT" --test-notify >/dev/null 2>&1 & +tn_pid=$! +sleep 0.4 +kill -TERM "$tn_pid" 2>/dev/null || true +tn_rc=0 +wait "$tn_pid" || tn_rc=$? +if [[ "$tn_rc" != "0" && -z "$(ls -A "$tn_tmp")" ]]; then + _pass "--test-notify: interrupted run removes its temp file" +else + _fail "--test-notify: temp file left after SIGTERM (exit $tn_rc)" +fi + # Review regressions ------------------------------------------------------- # Metric alerts of a guest that stops resolve instead of sticking forever. sd="$(_new_state_dir)" From 73f60ae6015cad40e2f7701c97c624335662b0f7 Mon Sep 17 00:00:00 2001 From: Tim Baumann Date: Sat, 19 Sep 2026 19:22:42 +0200 Subject: [PATCH 8/8] fix(tui): detect terminal width inside command substitution _term_cols is always called as $(_term_cols), where stdout is a pipe, so tput was never consulted and the width was fixed at 80. Also check stderr for a terminal. Found while verifying --health on a real PVE 9.2 node at 60 columns. Co-Authored-By: Claude Opus 5 --- proxmox-manager.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxmox-manager.sh b/proxmox-manager.sh index 3cd8338..a3e0c13 100755 --- a/proxmox-manager.sh +++ b/proxmox-manager.sh @@ -878,7 +878,9 @@ _truncate() { # _term_cols — terminal width (fallback 80). _term_cols() { local c="${COLUMNS:-}" - if [[ ! "$c" =~ ^[0-9]+$ ]] && [[ -t 1 ]]; then + # Callers use $(_term_cols), so stdout is a pipe here; stderr still + # points at the terminal in interactive use. + if [[ ! "$c" =~ ^[0-9]+$ ]] && { [[ -t 1 ]] || [[ -t 2 ]]; }; then c="$(tput cols 2>/dev/null || true)" fi [[ "$c" =~ ^[0-9]+$ ]] && ((c >= 40)) || c=80