From 60d9333a118f028a8dfd6a04cb418f972dfb013e Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Wed, 16 Sep 2026 20:38:43 +0000 Subject: [PATCH] refactor(template): organize runtime assets into subdirectories --- .../{ => config}/supervisord.conf | 0 .../kubernetes/homelab-workspace/configmap.tf | 20 +- .../{ => scripts}/script-agent-startup.sh | 0 .../script-container-entrypoint.sh | 0 .../{ => scripts}/script-dotfiles.sh | 0 .../script-memory-watchdog-start.sh | 0 .../scripts/script-memory-watchdog.sh | 1887 +++++++++++++++++ .../{ => scripts}/script-prepare-workspace.sh | 0 .../{ => scripts}/script-service-command.sh | 0 .../{ => scripts}/script-start-services.sh | 0 .../{ => scripts}/script-vscode-server-gc.sh | 0 11 files changed, 1897 insertions(+), 10 deletions(-) rename templates/kubernetes/homelab-workspace/{ => config}/supervisord.conf (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-agent-startup.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-container-entrypoint.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-dotfiles.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-memory-watchdog-start.sh (100%) create mode 100644 templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog.sh rename templates/kubernetes/homelab-workspace/{ => scripts}/script-prepare-workspace.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-service-command.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-start-services.sh (100%) rename templates/kubernetes/homelab-workspace/{ => scripts}/script-vscode-server-gc.sh (100%) diff --git a/templates/kubernetes/homelab-workspace/supervisord.conf b/templates/kubernetes/homelab-workspace/config/supervisord.conf similarity index 100% rename from templates/kubernetes/homelab-workspace/supervisord.conf rename to templates/kubernetes/homelab-workspace/config/supervisord.conf diff --git a/templates/kubernetes/homelab-workspace/configmap.tf b/templates/kubernetes/homelab-workspace/configmap.tf index 890e880d..7ff01ef1 100644 --- a/templates/kubernetes/homelab-workspace/configmap.tf +++ b/templates/kubernetes/homelab-workspace/configmap.tf @@ -8,16 +8,16 @@ resource "kubernetes_config_map_v1" "workspace_scripts" { } data = { - "script-agent-startup.sh" = file("${path.cwd}/script-agent-startup.sh") - "script-container-entrypoint.sh" = file("${path.cwd}/script-container-entrypoint.sh") - "script-dotfiles.sh" = file("${path.cwd}/script-dotfiles.sh") - "script-memory-watchdog.sh" = file("${path.cwd}/script-memory-watchdog.sh") - "script-memory-watchdog-start.sh" = file("${path.cwd}/script-memory-watchdog-start.sh") - "script-prepare-workspace.sh" = file("${path.cwd}/script-prepare-workspace.sh") - "script-service-command.sh" = file("${path.cwd}/script-service-command.sh") - "script-start-services.sh" = file("${path.cwd}/script-start-services.sh") - "supervisord.conf" = file("${path.cwd}/supervisord.conf") - "script-vscode-server-gc.sh" = file("${path.cwd}/script-vscode-server-gc.sh") + "script-agent-startup.sh" = file("${path.cwd}/scripts/script-agent-startup.sh") + "script-container-entrypoint.sh" = file("${path.cwd}/scripts/script-container-entrypoint.sh") + "script-dotfiles.sh" = file("${path.cwd}/scripts/script-dotfiles.sh") + "script-memory-watchdog.sh" = file("${path.cwd}/scripts/script-memory-watchdog.sh") + "script-memory-watchdog-start.sh" = file("${path.cwd}/scripts/script-memory-watchdog-start.sh") + "script-prepare-workspace.sh" = file("${path.cwd}/scripts/script-prepare-workspace.sh") + "script-service-command.sh" = file("${path.cwd}/scripts/script-service-command.sh") + "script-start-services.sh" = file("${path.cwd}/scripts/script-start-services.sh") + "supervisord.conf" = file("${path.cwd}/config/supervisord.conf") + "script-vscode-server-gc.sh" = file("${path.cwd}/scripts/script-vscode-server-gc.sh") "workspace-init.sh" = coder_agent.main.init_script } } diff --git a/templates/kubernetes/homelab-workspace/script-agent-startup.sh b/templates/kubernetes/homelab-workspace/scripts/script-agent-startup.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-agent-startup.sh rename to templates/kubernetes/homelab-workspace/scripts/script-agent-startup.sh diff --git a/templates/kubernetes/homelab-workspace/script-container-entrypoint.sh b/templates/kubernetes/homelab-workspace/scripts/script-container-entrypoint.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-container-entrypoint.sh rename to templates/kubernetes/homelab-workspace/scripts/script-container-entrypoint.sh diff --git a/templates/kubernetes/homelab-workspace/script-dotfiles.sh b/templates/kubernetes/homelab-workspace/scripts/script-dotfiles.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-dotfiles.sh rename to templates/kubernetes/homelab-workspace/scripts/script-dotfiles.sh diff --git a/templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh b/templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog-start.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-memory-watchdog-start.sh rename to templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog-start.sh diff --git a/templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog.sh b/templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog.sh new file mode 100644 index 00000000..44b054cf --- /dev/null +++ b/templates/kubernetes/homelab-workspace/scripts/script-memory-watchdog.sh @@ -0,0 +1,1887 @@ +#!/bin/bash +# +# Memory watchdog for the workspace pod: it bounds the *standing population* of +# restartable helper processes. +# +# In one sentence: the VS Code tree is given a fixed 2048 MiB envelope, each role +# holds a named share of it that does not vary with pod size, and a process is +# killed when its PSS has been above its role's share for ten continuous minutes +# *and* it has been observed inside that share at some point since it started. +# The long form is in the "the envelope" section below. +# +# What it is for, and what it deliberately is not for. +# +# It is not an OOM preventer. The earlier version of this script was, and the +# premise did not survive measurement. Every memcg OOM recorded for this +# workspace was a spike - 70 to 220 MB/s, idle to dead inside a minute - and the +# victims named in the kernel log were agent sessions and node, never a VS Code +# process. A poll loop cannot win that race: a generic biggest-RSS killer only +# beats the kernel at a 0.3s interval, loses at 0.5s, and under the +# memory.oom.group=1 that applied at the time was killed by the very event it +# lost to. The kubelet now sets singleProcessOOMKill, so a container created +# since reads memory.oom.group=0 and an OOM takes the offending process only - +# which is the acute job done properly, by the layer that can actually do it. The graded +# shedding ladder that used to live here climbed correctly during a live spike +# and then logged `no-candidates`, because the runaway was not in the tree it +# managed. That ladder has been removed rather than tuned. +# +# What a poll loop is genuinely good at is growth measured in MB per *minute*: +# the slow drift of long-lived helpers - the extension host, language servers, +# the file watcher, and the MCP servers and other helpers an agent session +# spawns. That drift is real and was being policed by hand (repeatedly killing +# VS Code to save agent sessions); a python MCP server holding 1.66 GB was +# observed during one of the kills. Every process this script may signal is +# restarted by its own supervisor - VS Code respawns its forks and language +# servers, an agent session respawns its MCP servers - so a wrong kill costs a +# reload, not a session. That asymmetry is what licenses being aggressive. +# +# The goal is therefore runway, not rescue: keep the resting population small so +# that when a spike does arrive it starts from as much free memory as possible, +# and stop the operator having to do it by hand. +# +# Dependencies are deliberately tiny: bash 4.4+, /proc, /sys/fs/cgroup, +# /usr/bin/sleep, coreutils mv/rm/mkdir. Measurement and process enumeration use +# bash builtins, so a sweep forks nothing at all. The operator's PATH is shadowed +# by brew, so anything invoked here is called by absolute /usr/bin/... path and +# never by name. +# +# Modes (the memory_watchdog_mode template parameter; the script itself falls +# back to observe when the value is anything else): +# observe - measure, publish, and log the action it *would* have taken. +# Signals nothing. +# enforce - additionally kill drifted processes, in *every* role. This is the +# template default. +# +# There used to be a third mode, enforce-all, which armed the editor roles that +# enforce left alone. Under a fixed envelope there is nothing left for it to add: +# a role outside the enforced set is a role whose share of the envelope is not +# actually enforced, which makes the envelope a description rather than a bound. +# The value is still accepted, as a synonym for enforce, so that a workspace +# carrying it does not silently fall back to observing. +# +# Test seams, exercised by script-memory-watchdog-test.sh: +# WATCHDOG_CGROUP_DIR WATCHDOG_PROC_DIR WATCHDOG_STATE_DIR +# WATCHDOG_MODE WATCHDOG_ONESHOT WATCHDOG_NOW WATCHDOG_SOURCE_ONLY +# +# Deliberately NOT using `set -e`: this is a supervisor with no supervisor of its +# own. A read that fails because a /proc entry vanished mid-sweep must skip that +# entry, not take the watchdog down and leave the pod unpoliced. +set -uo pipefail + +# --------------------------------------------------------------------------- # +# configuration +# --------------------------------------------------------------------------- # + +CGROUP_DIR="${WATCHDOG_CGROUP_DIR:-/sys/fs/cgroup}" +PROC_DIR="${WATCHDOG_PROC_DIR:-/proc}" +STATE_DIR="${WATCHDOG_STATE_DIR:-${HOME:-/home/coder}/.local/state/vscode-memory-watchdog}" +MODE="${WATCHDOG_MODE:-observe}" +ONESHOT="${WATCHDOG_ONESHOT:-0}" + +GIB=1073741824 +MIB=1048576 +PAGE_SIZE=4096 +USER_HZ=100 + +# Two cadences, one loop. The cgroup sample is a handful of file reads and is +# what keeps the telemetry legible across a fast event; the sweep walks every +# process and reads smaps_rollup, and drives every decision. Drift is measured in +# MB per minute, so a decision cadence of a minute is not a compromise - it is +# the correct resolution for the thing being policed. +SAMPLE_INTERVAL="${WATCHDOG_SAMPLE_INTERVAL:-10}" +SWEEP_EVERY="${WATCHDOG_SWEEP_EVERY:-6}" # samples per sweep => 60s + +# A process must be over its budget continuously for this long before anything +# happens to it. This is the single most important number in the file, and it is +# what separates drift policing from the spike chasing that did not work: a +# language server that balloons while indexing and then hands the memory back is +# load, not drift, and must survive. Expressed in seconds so that changing the +# sweep cadence cannot silently change the policy. +DWELL_SECONDS="${WATCHDOG_DWELL_SECONDS:-600}" + +# Never signal something that has not been alive long enough to have finished +# starting up. A process that is over budget within seconds of its own start is +# a spike, which is the kernel's problem, not this one's. +MIN_AGE="${WATCHDOG_MIN_AGE:-300}" + +# SIGTERM, then SIGKILL on a later sweep if it is still there and still over. +# There is no in-loop sleep: the grace period is measured across sweeps, so a +# process that is shutting down politely is never hurried. +KILL_GRACE="${WATCHDOG_KILL_GRACE:-30}" + +# The kill-rate report. This was a circuit breaker: a role killed LOOP_KILLS +# times inside LOOP_WINDOW disarmed itself permanently. It no longer disarms +# anything, and the reason is the envelope below. +# +# The breaker assumed that repeated kills meant a *wrong budget*, which was true +# while budgets were per-role numbers reasoned about one at a time. Under a total +# envelope it is no longer a safe inference: repeated kills can equally mean the +# process does not fit in the share the operator decided to give it, and that is +# a decision rather than a defect. A mechanism that switches itself off after +# three of those is inert by mid-morning - and it arrives looking exactly like +# the watchdog working, which is precisely the failure the breaker was built to +# prevent, now caused by the breaker. +# +# The kill *loop* it protected against is prevented structurally instead, by the +# oversize rule in sweep_once: a process that has never been observed inside its +# budget is never signalled, so the loop "kill it -> the replacement is over +# budget too -> kill it" cannot form. That rule is per process and evidence-based +# rather than per role and count-based, it re-arms by itself the moment a +# replacement does fit, and it can never widen a budget. +# +# So these two numbers now only decide when a kill *rate* is worth a line in the +# container log. Nothing is disarmed and nothing changes behaviour: the line is a +# sizing signal for a human, and the watchdog goes on enforcing. +LOOP_WINDOW="${WATCHDOG_LOOP_WINDOW:-3600}" +LOOP_KILLS="${WATCHDOG_LOOP_KILLS:-3}" +# The same idea across all roles at once. +GLOBAL_LOOP_KILLS="${WATCHDOG_GLOBAL_LOOP_KILLS:-8}" + +# --------------------------------------------------------------------------- # +# the envelope +# --------------------------------------------------------------------------- # +# +# One sentence: the VS Code tree is given a fixed 2048 MiB envelope, each role +# holds a named share of it that does not vary with pod size, and a process is +# killed when its PSS has been above its role's share for ten continuous minutes +# *and* it has been observed inside that share at some point since it started. +# +# Everything below is that sentence written out. What it replaced was three +# interacting terms - a hand-picked per-role number, a memory.max/8 pod share, +# and a resting x 1.5 floor - each patching the previous one's failure, with no +# term anywhere that bounded the total. +# +# Why the floor is gone, and it is *gone* rather than relaxed. It existed so a +# budget could never land below what a process already held at rest, which is a +# real scar: this design twice shipped a limit below a running process's size, +# most memorably an RLIMIT_DATA ceiling below what an idle file watcher held. But +# a floor and an envelope are incompatible instruments. Under a total, a process +# whose resting size does not fit means the process must be made smaller; a floor +# says the budget must grow to meet it, and enough of those push the sum past the +# envelope, which is the envelope defeated by arithmetic. What the floor was +# protecting against is now protected against by the oversize rule (see +# sweep_once): a process too big for its share is *reported and left alone* +# rather than killed on a loop. That is a better guard than the floor was, +# because it is measured on the process actually running rather than read from a +# hand-maintained constant that goes stale, and because it cannot inflate a +# budget. +# +# Why memory.max/8 is gone. It scaled each role with the pod, on the reasoning +# that a budget reasoned about for an 8 GiB pod would let one helper own a +# quarter of a 4 GiB one. It was never a bound on the total, and it is meaningless +# against a fixed envelope: 2048 MiB is 2048 MiB on every pod size. Its protection +# survives by construction rather than by arithmetic - the largest single share +# below is 512 MiB, which is exactly memory.max/8 at the smallest pod this +# template offers, and script-memory-watchdog-test.sh asserts that property at +# every pod size rather than trusting this sentence. +# +# Why PSS and not VmData or RSS. RLIMIT_DATA accounts VmData, which is why the +# long-removed ceiling measured it; for "is this process holding too much memory" +# it is wrong by roughly an order of magnitude on a V8 process (measured: file +# watcher 66 MB resident against 622 MB of data). PSS is what the pod actually +# pays - shared pages counted once and divided among the sharers - which matters +# when a dozen node processes share one binary's file-backed pages. RSS would +# charge each of them the full share. +ENVELOPE="${WATCHDOG_ENVELOPE:-2147483648}" # 2048 MiB + +# The roles inside the envelope. claudeHelper is deliberately not one of them: it +# is not VS Code, it is not spawned by the editor, and it was sized against a +# measured adversary rather than against a share of anything. +ENVELOPE_ROLES=" serverMain extensionHost tsserver treeHelper fileWatcher languageServer extensionHelper " + +# The apportionment. Every figure below is a decision about how to divide a fixed +# 2048 MiB, not an estimate of how large a role "should" be, and the two are not +# the same kind of number: raising one of these lowers another. +# +# The measurements each share is argued against were taken over ~38 hours on the +# live 8 GiB workspace - one operator, one Terraform/YAML/shell repository, and +# only *eight distinct processes* behind the whole table. A role sampled 1700 +# times is one process observed 1700 times, not 1700 observations, so none of +# these is a distribution and none should be read as one. What makes conservative +# rounding unnecessary here is the oversize rule: under it, a share that turns +# out to be too small for a legitimate process produces a repeating log line and +# no kill, so being wrong costs evidence rather than a restart loop. +declare -gA BUDGET_ROLE=( + # The largest single share, and it is allocated by *consequence* rather than by + # size: serverMain holds the remote connection, so a wrong kill costs the + # operator a disconnect rather than a reload, and it should be the share least + # likely to be reached by accident. Observed maximum over the window: 233 MB. + # That the biggest share goes to the role with the smallest appetite is a real + # cost, roughly 280 MiB, and it is paid on purpose. + [serverMain]=536870912 # 512 MiB + # The residual, and deliberately the second-largest, because this is the only + # role that measurably drifts: its hourly *minimum* climbed 528 -> 686 MiB over + # 29 hours and it peaked at 738 MiB. + # + # It does not fit today, and that is stated rather than papered over: the live + # extension host arrived at 610 MB PSS at 11 seconds old and settled at 577 MB + # before doing any work, so with today's extension set it is over this share + # from birth, is classified oversize, and is reported rather than killed. No + # apportionment of 2048 MiB can both keep serverMain the largest share and put + # this one above 610 MB - that is arithmetic, not a preference - so the + # envelope presupposes a tree that has been trimmed. The only lever that trims + # it is loading less - remote.extensionKind in the operator's dotfiles, moving + # extensions to the client side. There is no heap ceiling for this role at any + # value: the server registers no such setting, exposes no CLI flag, and deletes + # NODE_OPTIONS from the child environment before every fork, so a variable set + # in env.tf would reach every mise-node process in the pod and not this one. + # The evidence is in DESIGN.md, "Ordering the heap ceiling against the share". + # This template cannot depend on the trim having been applied either. If it + # has not, this role degrades to reporting and the other six go on being + # enforced; the repeating event=oversize line is exactly the evidence that the + # trim has not landed. + [extensionHost]=503316480 # 480 MiB + # 1.5x the 208 MB maximum observed, on a repository containing no TypeScript at + # all - not one .ts file and not one tsconfig.json anywhere under ~/code - so + # that figure is a floor of evidence rather than a ceiling. The number worth + # quoting is the other one: tsserver is launched with --max-old-space-size=3072, + # VS Code's shipped default, observed live on both instances in this pod. + # + # This is the one role where the operator's dotfiles hold a heap ceiling for the + # same process this share bounds, and the two are *deliberately* not ordered so + # that the ceiling binds first. A heap ceiling bounds V8's old space; PSS also + # carries the binary, native allocations, external buffers and V8's other + # spaces, and that remainder is near-constant per role rather than proportional, + # so the two relate by addition: peak PSS is about resting PSS plus the ceiling. + # For V8 to fail first the ceiling would have to be at or under share minus + # resting, which here is 320 - 141 = 179 MiB. It is not set there, on purpose: + # the setting is window-scoped, so it lives in the operator's client-side User + # settings and applies to every window they open including local ones, and the + # failure it would buy is not more recoverable than this one - a V8 fatal heap + # error and a SIGKILL reach the same handler in the TypeScript extension, which + # stops restarting the service after five crashes in five minutes. A ten-minute + # dwell cannot reach that rate; a binding ceiling on a large project reaches it + # immediately. See DESIGN.md, "Ordering the heap ceiling against the share". + # + # What that costs is stated rather than hidden: unlike the extension host this + # role *does* fit at rest, so the oversize rule never covers it, and on a + # project large enough to hold it above this share for ten minutes the share + # means "kill a healthy tsserver". That kill is recoverable - the extension + # restarts the service - and it is rate-limited by the dwell to well under the + # five-in-five-minutes that would make the extension give up. + [tsserver]=335544320 # 320 MiB + # Above the 284 MB maximum observed for the largest member of the role - a + # tamasfe.even-better-toml server - with 13% of headroom. The role's other two + # measured members hold 61 MB (tsserver's typingsInstaller) and 54 MB (the + # built-in markdown server). 256 MiB was considered and rejected: it is below a + # member measured on this very workspace, so it would have shipped a role that + # was oversize on day one. + [treeHelper]=335544320 # 320 MiB + # 1.5x the 107 MB maximum observed, which was flat to the megabyte for 28 + # straight hours. The headroom is for repositories with more files than this + # one, since this is the one role whose size is a direct function of what is + # being watched. + [fileWatcher]=167772160 # 160 MiB + # 4x the 31 MB maximum observed (yaml/json/terraform servers). The class + # contains much larger members on other ecosystems, and this share does not fit + # them; that is the envelope being honest about what 2048 MiB divides into + # rather than a claim that a rust-analyzer is 128 MiB. + [languageServer]=134217728 # 128 MiB + # 5x the 24 MB maximum observed (terraform-ls), with the same caveat: gopls and + # rust-analyzer are in this class and are much larger. + [extensionHelper]=134217728 # 128 MiB + # Outside the envelope. The only budget here anchored on a measured *offender* + # rather than on a measured healthy size: the python MCP server holding 1.66 GB + # that this whole mechanism was built for. The largest healthy MCP server + # measured on this workspace is 142 MB, so this is ~3.6x the honest case and + # ~3.2x smaller than the adversary. It sits outside the envelope because an MCP + # server is not VS Code and tightening it would buy the editor nothing. + [claudeHelper]=536870912 # 512 MiB +) + +# The order every budget is reported in. A literal role list at the reporting +# call site is a place for a fully-enforced budget to go unreported, so the order +# is declared once here and the test suite asserts it names every declared role +# and nothing else. +BUDGET_ORDER="serverMain extensionHost tsserver treeHelper fileWatcher languageServer extensionHelper claudeHelper" + +# Filled by derive_budgets(). +declare -gA BUDGET=() + +# Only processes at or above this are written to the sweep log, plus every +# process the watchdog is policing regardless of size. The floor keeps a sweep +# from being a hundred rows of 2 MB shells while still recording anything that +# could plausibly matter later. +SWEEP_LOG_FLOOR="${WATCHDOG_SWEEP_LOG_FLOOR:-33554432}" # 32 MiB + +# Where the action lines also go, so that they survive the pod. +# +# The cluster's log agent tails container stdout and nothing else, so no file in +# this pod can reach Loki - but PID 1 in this container is the coder agent, and +# /proc/1/fd/1 *is* the container's stdout. Writing there needs no new +# infrastructure, no configuration anywhere, and the lines arrive already +# labelled by namespace, pod and container. +# +# Only *actions* go there: kills, refusals, disarms, the budgets in force, and a +# low-rate census line. The per-process sweep stays in the local file, because a +# few dozen rows a minute is not something to put through a log pipeline, and +# the local file remains the detailed record either way. +# +# Every write is best-effort. The fd may not be writable in every context - a +# harness, a different container layout, a future PID 1 that closes it - and a +# watchdog that died because logging failed would be a worse bug than any it +# prevents. The first failure disables the path and says so once, locally. +STDOUT_PATH="${WATCHDOG_STDOUT_PATH:-/proc/1/fd/1}" +CENSUS_EVERY="${WATCHDOG_CENSUS_EVERY:-3600}" # seconds between census lines + +MAX_LOG_LINES="${WATCHDOG_MAX_LOG_LINES:-20000}" +MAX_CSV_LINES="${WATCHDOG_MAX_CSV_LINES:-50000}" +MAX_SWEEP_LINES="${WATCHDOG_MAX_SWEEP_LINES:-200000}" + +# Pressure labels. These drive nothing at all - no action is keyed on them - and +# exist so the telemetry says whether the pod was comfortable at the time. The +# ladder that used to act on these numbers is gone; the numbers were the half of +# it worth keeping. +PRESSURE_LOW_DEN="${WATCHDOG_PRESSURE_LOW_DEN:-4}" # H < max/4 => low +PRESSURE_CRIT_DEN="${WATCHDOG_PRESSURE_CRIT_DEN:-8}" # H < max/8 => critical + +# Sweep state. Declared at file scope, not inside main(), so that sourcing with +# WATCHDOG_SOURCE_ONLY=1 gives the test harness correctly-typed globals. +declare -gA P_COMM=() P_CMD=() P_ARGV0=() P_PPID=() P_RSS=() P_PSS=() P_START=() P_AGE=() P_SID=() +declare -gA CHILDREN=() SERVER_TREE=() PROTECTED=() NOT_EDITOR=() PROTECT_REASON=() WATCHDOG_KIN=() +declare -gA POLICED=() CLAUDE_ROOTS=() +declare -ga PIDS=() SERVER_ROOTS=() +declare -gA OVER_SINCE=() KILLED_AT=() KILL_TIMES=() KILLS=() +# FIT_SEEN[pid:starttime] is set the first time a process is observed at or +# below its budget while old enough to have finished starting up. It is the +# whole of the oversize rule: no entry means this process has never fitted, and +# nothing that has never fitted is signalled. Keyed on pid:starttime like every +# other piece of carried state, so a recycled pid cannot inherit the evidence. +declare -gA FIT_SEEN=() +# Per-role aggregates, recomputed each sweep (SUM) and kept for the life of the +# watchdog (PEAK). Neither drives any decision - see the note on aggregation in +# sweep_once - and PEAK is how a share that no longer matches the workload +# becomes visible to a human without the loop reacting to it. +declare -gA ROLE_SUM_PSS=() ROLE_PEAK_PSS=() +TREE_PSS=0 +OVERSIZE_N=0 +SERVER_PID="" +ROLE=other +GUARD="" +PRESSURE=ok +PREV_AT=0 +PREV_REFAULT=0 +PREV_PGSCAN=0 +PREV_U=0 +CYCLE=0 +SWEEPS=0 +UPTIME=0 +H_MAX_SEEN=0 +H_MIN_SEEN=0 +M_MAX=0 +KILLS_TOTAL=0 +KILL_TIMES_ALL="" +LAST_KILL_AT=0 +PSS_UNAVAILABLE=0 +VISIBILITY_WARNED=0 +BLIND_SINCE=-1 +STDOUT_OK=1 +LAST_CENSUS_AT=0 +STARTED_AT=${WATCHDOG_NOW:-$EPOCHSECONDS} + +# --------------------------------------------------------------------------- # +# derivation - now a copy, and that is the point +# --------------------------------------------------------------------------- # + +# Sets BUDGET. There is nothing to derive: a share of a fixed envelope is a +# constant, so this copies the declared table and honours an explicit +# WATCHDOG_BUDGET_, which is how a number is tried on a live workspace +# without editing the script. +# +# It deliberately takes no arguments. It used to take memory.max, and taking it +# again would invite a term that scales a share with the pod - which is exactly +# what the envelope replaced. +derive_budgets() { + local role var + BUDGET=() + for role in "${!BUDGET_ROLE[@]}"; do + var="WATCHDOG_BUDGET_${role}" + if [[ -n ${!var:-} ]]; then + BUDGET[$role]=${!var} + continue + fi + BUDGET[$role]=${BUDGET_ROLE[$role]} + done + return 0 +} + +# The sum of the shares actually in force for the roles inside the envelope. This +# is what makes an override visible as a change to the total rather than as a +# change to one line, and it is what the census reports against. +envelope_in_force() { + local role total=0 + for role in ${ENVELOPE_ROLES}; do + total=$((total + ${BUDGET[$role]:-0})) + done + printf '%s' "$total" +} + +# enforce-all is the retired third mode. It asked for a superset of enforce that +# no longer exists, so it is honoured as enforce rather than dropped to observe: +# a workspace that had asked for more enforcement must not silently get none. +# Anything unrecognised still falls back to the inert mode - this is the last +# line of defence for a value that has already been through Terraform's +# allowlist, and it is a function rather than a case inside main() so that the +# test suite can reach it. +normalise_mode() { + case "$MODE" in + observe | enforce) ;; + enforce-all) MODE=enforce ;; + *) MODE=observe ;; + esac + return 0 +} + +# True when the mode may signal this role at all. +# +# There is no per-role arming any more. Every role with a budget is armed under +# enforce, because a role inside the envelope that cannot be signalled is a share +# that is not enforced, and an envelope with unenforced shares is a description +# of the tree rather than a bound on it. What separates a role that should be +# killed from one that should only be reported is not the role - it is whether +# the individual process ever fitted, which sweep_once decides per process. +role_is_armed() { + local role=$1 + [[ -n ${BUDGET[$role]:-} ]] || return 1 + case "$MODE" in + enforce) return 0 ;; + *) return 1 ;; + esac +} + +# --------------------------------------------------------------------------- # +# measurement - reads only, sets M_*/P_* globals, decides nothing +# --------------------------------------------------------------------------- # + +# Unreclaimable memory U. With memory.swap.max=0, all anon is unreclaimable. +# +# The `kernel` roll-up in memory.stat must NOT be used: it is dominated by +# slab_reclaimable (dentry/inode cache - 1.9 GiB on the real pod), which the +# kernel hands back under pressure. Counting it makes an idle container look like +# it is about to die. memory.current has the same defect plus the page cache, +# which is why it reads 96% here while U is 28%. +# +# Nothing acts on this any more; it is the pod-level context every per-process +# row is read against, and the only record of what the container was doing at the +# time. Returns 0 on success, 1 on read failure, 2 if the cgroup has no limit. +read_cgroup_memory() { + local key val + local anon=0 shmem=0 unevictable=0 slab_unreclaimable=0 kernel_stack=0 + local pagetables=0 sec_pagetables=0 percpu=0 sock=0 + + M_FILE=0 + M_SLAB_RECLAIMABLE=0 + M_REFAULT_FILE=0 + M_PGSCAN_DIRECT=0 + + while read -r key val; do + case "$key" in + anon) anon=$val ;; + shmem) shmem=$val ;; + unevictable) unevictable=$val ;; + slab_unreclaimable) slab_unreclaimable=$val ;; + kernel_stack) kernel_stack=$val ;; + pagetables) pagetables=$val ;; + sec_pagetables) sec_pagetables=$val ;; + percpu) percpu=$val ;; + sock) sock=$val ;; + file) M_FILE=$val ;; + slab_reclaimable) M_SLAB_RECLAIMABLE=$val ;; + workingset_refault_file) M_REFAULT_FILE=$val ;; + pgscan_direct) M_PGSCAN_DIRECT=$val ;; + esac + done <"${CGROUP_DIR}/memory.stat" || return 1 + + M_ANON=$anon + M_SHMEM=$shmem + M_UNEVICTABLE=$unevictable + M_SLAB_UNRECLAIMABLE=$slab_unreclaimable + M_KERNEL_STACK=$kernel_stack + M_PAGETABLES=$pagetables + M_SEC_PAGETABLES=$sec_pagetables + M_PERCPU=$percpu + M_SOCK=$sock + + # sec_pagetables is zero outside nested virtualisation but is genuinely + # unreclaimable when present, so it is counted rather than assumed away. + M_U=$((anon + shmem + unevictable + slab_unreclaimable + + kernel_stack + pagetables + sec_pagetables + percpu + sock)) + + local raw="" + read -r raw <"${CGROUP_DIR}/memory.max" || return 1 + [[ $raw == "max" ]] && return 2 + [[ $raw =~ ^[0-9]+$ ]] || return 1 + M_MAX=$raw + + M_CURRENT=0 + read -r M_CURRENT 2>/dev/null <"${CGROUP_DIR}/memory.current" + + M_H=$((M_MAX - M_U)) + + PRESSURE=ok + ((M_H < M_MAX / PRESSURE_LOW_DEN)) && PRESSURE=low + ((M_H < M_MAX / PRESSURE_CRIT_DEN)) && PRESSURE=critical + return 0 +} + +# memory.pressure "full avg10", scaled by 100 so it compares as an integer. +read_cgroup_pressure() { + local kind field + M_PSI_CENTI=0 + while read -r kind field _; do + [[ $kind == "full" ]] || continue + field=${field#avg10=} + [[ $field == *.* ]] || field="${field}.00" + [[ ${field%%.*} =~ ^[0-9]+$ && ${field##*.} =~ ^[0-9]+$ ]] || continue + M_PSI_CENTI=$((10#${field%%.*} * 100 + 10#${field##*.})) + done 2>/dev/null <"${CGROUP_DIR}/memory.pressure" + return 0 +} + +read_uptime() { + local raw="" + read -r raw _ 2>/dev/null <"${PROC_DIR}/uptime" + UPTIME=${raw%%.*} + [[ $UPTIME =~ ^[0-9]+$ ]] || UPTIME=0 + return 0 +} + +# Fills PIDS / P_COMM / P_CMD / P_ARGV0 / P_PPID / P_START / P_AGE / CHILDREN. +# +# starttime (stat field 22) is read for a reason that is not cosmetic: pids are +# recycled, and every piece of state this watchdog carries between sweeps - how +# long something has been over budget, whether it has already been sent a +# SIGTERM - is keyed on pid *and* starttime. Without that, a recycled pid +# inherits another process's history and can be killed for it. +read_process_table() { + PIDS=() + P_COMM=() + P_CMD=() + P_ARGV0=() + P_PPID=() + P_START=() + P_AGE=() + P_SID=() + P_RSS=() + P_PSS=() + CHILDREN=() + + local entry pid line rest comm ppid start sid + local -a argv f + + for entry in "${PROC_DIR}"/[0-9]*; do + pid=${entry##*/} + read -r line 2>/dev/null <"$entry/stat" || continue + + # /proc//stat is "pid (comm) state ppid ...", and comm may contain + # spaces and parentheses, so split on the *last* ") " rather than tokenise. + rest=${line##*') '} + [[ $rest == "$line" ]] && continue + comm=${line#*'('} + comm=${comm%%') '*} + read -r -a f <<<"$rest" + ppid=${f[1]:-} + [[ $ppid =~ ^[0-9]+$ ]] || continue + # f[0] is state, so f[19] is stat field 22, starttime, in USER_HZ ticks, and + # f[3] is field 6, the session id. + start=${f[19]:-0} + [[ $start =~ ^[0-9]+$ ]] || start=0 + sid=${f[3]:-0} + [[ $sid =~ ^[0-9]+$ ]] || sid=0 + + argv=() + mapfile -d '' -t argv 2>/dev/null <"$entry/cmdline" + PIDS+=("$pid") + P_COMM[$pid]=$comm + P_CMD[$pid]="${argv[*]}" + # argv[0] is kept as its own field rather than recovered later by cutting + # P_CMD at the first space, which is wrong for any executable path + # containing one, and which is precisely the kind of "close enough" string + # handling this script has already been bitten by. + P_ARGV0[$pid]="${argv[0]:-}" + P_PPID[$pid]=$ppid + P_START[$pid]=$start + P_SID[$pid]=$sid + P_AGE[$pid]=$((UPTIME - start / USER_HZ)) + # Written with an explicit $: inside (( )) an associative-array subscript is + # a string, so `P_AGE[pid]` would look up the key "pid" and quietly read 0. + ((${P_AGE[$pid]} < 0)) && P_AGE[$pid]=0 + CHILDREN[$ppid]+=" $pid" + done + return 0 +} + +# PSS from smaps_rollup, falling back to RSS from statm. +# +# smaps_rollup walks the page tables, so it is the expensive read in this script; +# it is done once a minute for a few dozen processes and never in the cgroup +# sample. The fallback exists because smaps_rollup is absent on some kernels and +# unreadable for a process that exits mid-sweep - and when it is used, the row +# says so, because silently substituting a number that is 30% larger would make +# every budget look tighter than it is. +read_usage() { + local pid res key val got + for pid in "$@"; do + read -r _ res _ 2>/dev/null <"${PROC_DIR}/${pid}/statm" || continue + P_RSS[$pid]=$((res * PAGE_SIZE)) + got="" + if [[ -r "${PROC_DIR}/${pid}/smaps_rollup" ]]; then + while read -r key val _; do + if [[ $key == "Pss:" ]]; then + got=$val + break + fi + done 2>/dev/null <"${PROC_DIR}/${pid}/smaps_rollup" + fi + if [[ $got =~ ^[0-9]+$ ]]; then + P_PSS[$pid]=$((got * 1024)) + else + P_PSS[$pid]=${P_RSS[$pid]} + PSS_UNAVAILABLE=1 + fi + done + return 0 +} + +# --------------------------------------------------------------------------- # +# selection - pure over the tables above; produces sets, takes no action +# --------------------------------------------------------------------------- # + +# Fills SERVER_ROOTS with every remote-server entrypoint, and sets SERVER_PID to +# the first of them (used only for logging). +# +# Three things about a real server tree that a plausible-looking implementation +# gets wrong, all three confirmed against a live workspace: +# +# - comm is NOT "node". Every node process in the server tree - server-main.js, +# ptyHost, extensionHost, fileWatcher, node-based language servers - reports +# comm=MainThread, because V8 renames its main thread with prctl(PR_SET_NAME). +# Anything that keys off comm=="node" is keying off a state that never occurs. +# +# - argv[0] is the signal that does hold. VS Code launches the server as +# `/node /out/server-main.js ...`, and argv[0] keeps +# the interpreter's real path. Requiring it to live under /.vscode-server/ (or, +# for a future launcher outside that tree, to be named node) is what keeps a +# `cat`, `tail -f` or `grep` over the same path from being elected as the root +# of everything the watchdog then decides. +# +# - there can be more than one. `--reconnection-grace-time` keeps a disconnected +# server alive for hours, and a window on a different commit gets its own +# server. Electing one and scoping to it would leave the other tree not merely +# unmanaged but unprotected, because the ptyHost excision only runs inside the +# tree that was discovered. So every root counts and the managed tree is the +# union of their subtrees. +# +# Matching is always on a whole argv element, never on a substring of the joined +# command line. +find_server_roots() { + local pid arg exe + local -a argv + SERVER_ROOTS=() + SERVER_PID="" + + for pid in "${PIDS[@]}"; do + [[ ${P_CMD[$pid]} == *"/.vscode-server/"*"out/server-main.js"* ]] || continue + argv=() + mapfile -d '' -t argv 2>/dev/null <"${PROC_DIR}/${pid}/cmdline" + ((${#argv[@]} > 1)) || continue + exe=${argv[0]##*/} + [[ ${argv[0]} == *"/.vscode-server/"* || $exe == node || $exe == node[0-9]* ]] || continue + # From argv[1]: the server path appearing as argv[0] would mean the .js file + # is itself being executed as the program, which is not how it is launched. + for arg in "${argv[@]:1}"; do + [[ $arg == *"/.vscode-server/"*"out/server-main.js" ]] || continue + SERVER_ROOTS+=("$pid") + break + done + done + + ((${#SERVER_ROOTS[@]})) || return 1 + # shellcheck disable=SC2034 + # Read by script-memory-watchdog-test.sh, which asserts which process is + # elected as the root - the one selection defect a fixture can catch early. + SERVER_PID=${SERVER_ROOTS[0]} + return 0 +} + +# Sets SERVER_TREE to the union of every root's descendants. Returns 1 when no +# server is running, leaving SERVER_TREE empty. +build_server_tree() { + local root pid + local -A one=() + SERVER_TREE=() + find_server_roots || return 1 + for root in "${SERVER_ROOTS[@]}"; do + subtree_of "$root" one + for pid in "${!one[@]}"; do + SERVER_TREE[$pid]=1 + done + done + return 0 +} + +# subtree_of +subtree_of() { + local root=$1 + local -n out=$2 + local queue=("$root") cur kids k + out=() + # shellcheck disable=SC2004 + # `out` is a nameref to an associative array, so the subscript is a string + # key. Dropping the $ would make bash key it on the literal name. + out[$root]=1 + while ((${#queue[@]})); do + cur=${queue[0]} + queue=("${queue[@]:1}") + kids=${CHILDREN[$cur]:-} + # shellcheck disable=SC2086 + # kids is a space-joined list of integers this script built itself. + for k in $kids; do + [[ -n ${out[$k]:-} ]] && continue + # shellcheck disable=SC2004 + out[$k]=1 + queue+=("$k") + done + done + return 0 +} + +# True when the process is running a binary that VS Code itself shipped, i.e. one +# under ~/.vscode-server. This is the structural boundary between "a process VS +# Code started with its own runtime" and "a process that merely happens to sit +# inside the tree", and it is one of the two primary safety rules here. +# +# A fully provisioned workspace has two unrelated node installations: VS Code's +# bundled one at ~/.vscode-server/cli/servers/Stable-/server/node, which +# arrives with the server download, and mise's on PATH, which is what the +# operator's repo tooling and agent sessions run on. There is no /usr/bin/node +# and no node on PATH at all without dotfiles. Every process VS Code spawns runs +# a binary under ~/.vscode-server; nothing the operator runs does. +# +# So keying detection on "is this node" by comm, by basename, or by a loose +# cmdline match would make an agent session indistinguishable from an editor +# helper. Path, and only path. +is_vscode_binary() { + [[ ${P_ARGV0[$1]:-} == *"/.vscode-server/"* ]] +} + +# True when the process was forked directly by a remote-server entrypoint, i.e. +# it is one of VS Code's own core forks rather than something an extension +# started. Measured on the live tree: server-main.js's direct children are +# exactly fileWatcher, extensionHost and ptyHost, while every language server, +# typings installer and extension binary hangs off the extension host or off +# another helper. +# +# It exists for one purpose, and it is a Munger-style "what would make this +# actively harmful" guard rather than a classification aid. The treeHelper +# fallback below governs anything in the tree it cannot name; if VS Code ever +# renames `--type=extensionHost`, that fallback would hand the largest and most +# user-visible process in the pod a 512 MiB helper budget and, under +# enforce-all, a standing kill order. A core fork that stops being recognised +# must become *unmanaged and loud*, never *managed on a guessed budget*. +is_server_fork() { + local ppid=${P_PPID[$1]:-0} r + ((${#SERVER_ROOTS[@]})) || return 1 + for r in "${SERVER_ROOTS[@]}"; do + [[ $ppid == "$r" ]] && return 0 + done + return 1 +} + +# True for the root process of an agent session. Structural: the program being +# executed is named claude, by comm (a compiled launcher) or by argv[0]'s +# basename (a shebang script, where comm is the interpreter). Never a substring +# of the joined command line - `*/claude*` once protected an unrelated process +# whose scratchpad path contained /claude-10001. +is_claude_root() { + local pid=$1 + [[ ${P_COMM[$pid]:-} == "claude" ]] && return 0 + local exe=${P_ARGV0[$pid]:-} + [[ ${exe##*/} == "claude" ]] +} + +# True for an interactive shell or a terminal multiplexer, i.e. a boundary the +# helper walk must not cross. Everything below one of these inside an agent +# session is the agent's *tool calls* - builds, test runs, `gh run watch` - which +# are in-flight work with no supervisor to restart them. They are excluded from +# policing on purpose, and that exclusion is the difference between "anything a +# session invokes is fair game" as a principle and as a foot-gun. +is_shell_like() { + local pid=$1 + local exe=${P_ARGV0[$pid]:-} + case "${P_COMM[$pid]:-}" in + bash | sh | dash | zsh | fish | ksh | tmux* | screen | sshd | ssh) return 0 ;; + esac + case "${exe##*/}" in + bash | sh | dash | zsh | fish | ksh | tmux | screen | sshd | ssh) return 0 ;; + esac + return 1 +} + +# The watchdog's own process, everything that started it, and everything it +# started. Structural, by pid: this replaces a `*memory-watchdog*` cmdline match +# that once protected every process in a test harness because the harness lived +# in a directory whose path contained that string. Two full runs of that harness +# looked clean while exercising nothing. Identity is a thing you know about +# yourself, not a thing you pattern-match out of other processes' arguments. +compute_watchdog_kin() { + WATCHDOG_KIN=() + local cur=$$ hops=0 p + local -A mine=() + subtree_of "$$" mine + for p in "${!mine[@]}"; do + WATCHDOG_KIN[$p]=self + done + # Ancestors. Bounded, because a corrupt or racing ppid chain must not spin. + while ((cur > 1 && hops < 64)); do + cur=${P_PPID[$cur]:-0} + ((cur > 1)) || break + WATCHDOG_KIN[$cur]=ancestor + ((hops += 1)) + done + return 0 +} + +# True when an argv element names one of the operator's own programs. This is +# the one guard that has to look at arguments rather than at argv[0], because the +# case it exists for is a payload run by somebody else's interpreter: an agent +# session that an extension spawned as `/node .../claude-code/cli.js` +# has VS Code's binary at argv[0] and is not under ptyHost, so neither of the +# structural rules reaches it. +# +# Matching is on whole path segments, never on a raw substring. Substrings are +# what made `*/claude*` protect an unrelated process whose scratchpad path +# happened to contain /claude: the segment there was `claude-10001`, which is not +# the program and does not match. +is_operator_payload() { + local tok seg + for tok in ${P_CMD[$1]:-}; do + tok=${tok%/} + seg=${tok##*/} + case "$seg" in + claude | tmux | chezmoi) return 0 ;; + esac + case "/${tok}/" in + */claude-code/* | */claude-code-*/*) return 0 ;; + esac + done + return 1 +} + +# The never-signal list. Every action consults this directly, so a defect in +# tree-walking still cannot route around it. Sets GUARD to the rule that fired, +# so that a protection can be reported rather than merely happening. +is_never_signal() { + local pid=$1 + GUARD="" + ((pid <= 1)) && GUARD=pid1 && return 0 + ((pid == $$)) && GUARD=self && return 0 + ((pid == BASHPID)) && GUARD=self && return 0 + ((pid == PPID)) && GUARD=self && return 0 + if [[ -n ${WATCHDOG_KIN[$pid]:-} ]]; then + GUARD=watchdog-${WATCHDOG_KIN[$pid]} + return 0 + fi + case "${P_COMM[$pid]:-}" in + coder | claude | chezmoi | sshd | screen | init | systemd) + GUARD="comm" + return 0 + ;; + tmux*) + GUARD="comm" + return 0 + ;; + esac + # argv[0]'s basename, i.e. the program actually being executed - the same + # question is_vscode_binary asks, asked of the other side. `/home/coder/...` + # appears in nearly every command line in this pod, so a rule that looked + # anywhere but argv[0] for the name `coder` would protect the entire pod. + case "${P_ARGV0[$pid]:-}" in + */coder | coder | */chezmoi | chezmoi | */tmux | tmux | */claude | claude) + GUARD=argv0 + return 0 + ;; + esac + if is_operator_payload "$pid"; then + GUARD=payload + return 0 + fi + return 1 +} + +# Everything the watchdog must never touch: the never-signal rules anywhere in +# the pod, plus every ptyHost fork inside the server tree and all its +# descendants, plus anything in the tree not running a VS Code binary. +# PROTECT_REASON records which rule claimed each pid, which is what makes an +# over-broad guard visible instead of silently inert. +compute_protected() { + PROTECTED=() + NOT_EDITOR=() + PROTECT_REASON=() + local pid p + local -A pty=() + + compute_watchdog_kin + + for pid in "${PIDS[@]}"; do + if is_never_signal "$pid"; then + PROTECTED[$pid]=1 + PROTECT_REASON[$pid]=$GUARD + fi + done + + # Two rules about *position* rather than identity, kept in their own set. The + # distinction is load-bearing and was learned the hard way: everything in + # PROTECTED is protected by what it is and may never be signalled by anything, + # whereas these two say only "this is not one of the editor's own helpers" - + # which is the correct rule for the editor walk and the wrong one for the + # second population. An MCP server belonging to a session that happens to be + # running in a VS Code terminal is inside the tree and does not run VS Code's + # binary, and it is exactly what this watchdog is meant to bound. + # + # First: anything in the tree not running a binary VS Code shipped. An agent + # session spawned by an extension is a child of the extension host and not of + # ptyHost, so tree position would not save it; its executable path does - and + # it is *also* claimed by the payload rule above, which is identity and is + # absolute. That overlap is deliberate belt-and-braces, not redundancy. + for pid in "${!SERVER_TREE[@]}"; do + if ! is_vscode_binary "$pid"; then + NOT_EDITOR[$pid]=1 + PROTECT_REASON[$pid]=${PROTECT_REASON[$pid]:-foreign-binary} + fi + done + + # Terminals, and everything started in one. Tree membership is NOT a safe kill + # criterion: tmux sessions and long-running agent runs started from a VS Code + # integrated terminal are descendants of the server tree via ptyHost. + # + # Second: the ptyHost subtree. Shells, multiplexers and everything started in + # one are absolutely protected anyway - by comm, by argv[0], by the payload + # rule, and by the walk in compute_policed stopping at shells - so what this + # adds is the position, not the only line of defence. + for pid in "${!SERVER_TREE[@]}"; do + if [[ ${P_CMD[$pid]:-} == *"--type=ptyHost"* ]]; then + subtree_of "$pid" pty + for p in "${!pty[@]}"; do + NOT_EDITOR[$p]=1 + PROTECT_REASON[$p]=${PROTECT_REASON[$p]:-ptyhost} + done + fi + done + return 0 +} + +# Sets ROLE for a process inside the VS Code server tree. +# +# `--type=` flags are matched as whole argv elements (P_CMD is argv joined with +# spaces, so the surrounding spaces make the match exact). ptyHost is tested +# first, and additionally as a loose substring: over-matching ptyHost only ever +# means protecting something that could have been touched, which is the safe +# direction, whereas over-matching any other role means mis-classifying it. +role_of() { + local cmd=" ${P_CMD[$1]:-} " + local argv0=${P_ARGV0[$1]:-} + case "$cmd" in + *" --type=ptyHost "* | *"--type=ptyHost"*) ROLE=ptyHost ;; + *" --type=extensionHost "*) ROLE=extensionHost ;; + *" --type=fileWatcher "*) ROLE=fileWatcher ;; + *"tsserver.js "* | *"/typescript/lib/tsserver"*) ROLE=tsserver ;; + # Looser than the rest: language servers have no common launch convention, so + # these are prefix/suffix guesses. They are only ever consulted for processes + # already inside the server tree and outside the ptyHost subtree, which is what + # keeps the blast radius of a wrong guess to one restartable helper. + *yaml-language-server* | *jsonServerMain* | *-language-server* | *languageserver*) ROLE=languageServer ;; + *"out/server-main.js "*) ROLE=serverMain ;; + *) + # Native helpers shipped inside an extension - terraform-ls on the live tree, + # and gopls / rust-analyzer / clangd on the same pattern. Keyed on argv[0], + # never on the joined command line: an agent session launched by an extension + # has the extension's directory all over its arguments while its executable + # is mise's node, and matching the arguments would classify it as a sheddable + # helper. Checked last, so a node language server - which also lives under + # extensions/ but runs VS Code's own node - keeps its role above. + if [[ $argv0 == *"/.vscode-server/extensions/"* ]]; then + ROLE=extensionHelper + elif is_vscode_binary "$1" && ! is_server_fork "$1"; then + # The default is inverted here, and this is the substantive change in this + # version: inside the server tree, running VS Code's own runtime, not one + # of the core forks, and not recognised - "I do not know what this is" is + # not the same statement as "this is not my business", and conflating them + # is what made three real helpers invisible. + # + # Measured on the live tree, `other` was returned for a + # tamasfe.even-better-toml server.js holding 280 MB PSS - already above the + # 256 MiB languageServer budget it would have been given had a pattern + # matched it - plus tsserver's typingsInstaller.js at 61 MB and the + # built-in markdown-language-features server at 54 MB. None of the + # patterns above reach them, and none of them ever could reliably: the set + # of launch shapes is the union of every extension's own choices, so a + # pattern list is a permanent race against a vendor-controlled vocabulary, + # and each new extension is silently ungoverned until somebody notices by + # hand. That is the failure this watchdog exists to end. + # + # The alternative to inverting is to keep adding a regex per extension. + # It is easier and it rots; worse, it fails *silently*, which is the one + # property this mechanism cannot afford. Inverting fails the other way - + # loudly. The risk it buys - a share applied to a process whose legitimate + # size is larger than the share - used to be paid for by arming this role + # separately; it is now paid for by the oversize rule, which spares any + # process that has never fitted, in every role alike. + ROLE=treeHelper + else + ROLE=other + fi + ;; + esac + return 0 +} + +# Fills POLICED with pid -> role: every process this watchdog is willing to have +# an opinion about. Two disjoint populations, found two different ways. +# +# 1. The VS Code server tree, minus the guards. Unchanged from the version that +# only knew about VS Code. +# +# 2. Helpers spawned by an agent session - the MCP servers and other long-lived +# children that the tree-scoped selection could not see at all, and where the +# largest single offender ever measured (1.66 GB of python) lived. The walk +# starts at each session root and descends, but never through a shell or a +# multiplexer: below one of those is the session's *tool calls*, which are +# in-flight work that nothing will restart. The session root itself is +# protected by name; this is only about what it spawned. +# +# The positional exclusions - the ptyHost subtree, and "does not run a binary VS +# Code shipped" - are permeable to the second walk, and that is the one +# deliberate hole in the rules; it is worth stating plainly. A session started +# with `coder ssh` sits under the agent and a session started in a VS Code +# terminal sits under ptyHost, running the operator's python or node rather than +# VS Code's; they are the same thing to the operator, and sparing one set of MCP +# servers because of which terminal its session came from would make the +# mechanism miss half its cases silently. What those rules are actually for - +# shells, multiplexers, the session itself, agent payloads, and every command run +# in a terminal - is still absolutely protected, by identity, because the walk +# stops at shells and every guard in is_never_signal still applies. +compute_policed() { + POLICED=() + CLAUDE_ROOTS=() + local pid root cur kids k + + for pid in "${!SERVER_TREE[@]}"; do + [[ -n ${PROTECTED[$pid]:-} || -n ${NOT_EDITOR[$pid]:-} ]] && continue + role_of "$pid" + [[ $ROLE == "ptyHost" || $ROLE == "other" ]] && continue + POLICED[$pid]=$ROLE + done + + for pid in "${PIDS[@]}"; do + is_claude_root "$pid" || continue + CLAUDE_ROOTS[$pid]=1 + done + + local -a queue=() + local sid + for root in "${!CLAUDE_ROOTS[@]}"; do + kids=${CHILDREN[$root]:-} + # shellcheck disable=SC2086 + for k in $kids; do queue+=("${k}:${P_SID[$root]:-0}"); done + done + while ((${#queue[@]})); do + cur=${queue[0]%%:*} + sid=${queue[0]#*:} + queue=("${queue[@]:1}") + # Two independent boundaries, either of which ends the walk. + # + # The session id is the stronger of the two and it is measured, not assumed: + # on the live workspace an agent session root has pgid == its own pid and + # sid == the login shell's session, while every tool call it runs has pgid + # == sid == its own pid - i.e. Claude Code detaches each Bash tool call into + # a new session so that it can kill the whole tree later. A stdio MCP server + # is spawned to talk over pipes and stays in the session it was started + # from. So "same session as the root" separates the servers from the work + # even when the tool call's shell has exec'd itself away, which the shell + # test alone would miss. + # + # If that ever stops being true, this walk polices nothing rather than + # policing the wrong thing, and check_visibility says so in the log. + [[ ${P_SID[$cur]:-0} == "$sid" ]] || continue + # And a shell is a boundary in its own right: neither it nor anything under + # it is policed, whatever session it is in. + is_shell_like "$cur" && continue + kids=${CHILDREN[$cur]:-} + # shellcheck disable=SC2086 + for k in $kids; do queue+=("${k}:${sid}"); done + # Protected by identity: not a candidate. Excluded only positionally - in + # the ptyHost subtree, or not running VS Code's own binary: still a + # candidate, for the reason given above. + [[ -n ${PROTECTED[$cur]:-} ]] && continue + is_claude_root "$cur" && continue + [[ -n ${POLICED[$cur]:-} ]] && continue + POLICED[$cur]=claudeHelper + done + return 0 +} + +# A short, stable identity for a policed process - what it *is*, as opposed to +# which pid it happens to have this time. This is the breadcrumb that makes the +# sweep log answerable after the fact ("which MCP server was that?"), and the +# label a metrics exporter would bucket on. +# +# Secrets are redacted here rather than at read time, because this is the only +# place a command line is written to a file that might later be shipped +# somewhere: VS Code puts --connection-token= on its own argv, and an MCP server +# can be launched with an API key in an argument. +identity_of() { + local pid=$1 role=$2 tok seg fallback rest ext + # For a helper, the interpreter is never the interesting part of the name: a + # dozen unrelated MCP servers are all `node` or `python3`, and what + # distinguishes them is the first argument that is not the interpreter and not + # a flag. For everything else argv[0] is the name, because that is the thing + # the role was decided from. + if [[ $role == "claudeHelper" || $role == "extensionHelper" || $role == "treeHelper" ]]; then + for tok in ${P_CMD[$pid]:-}; do + seg=${tok##*/} + [[ -z $seg ]] && continue + case "$seg" in + -* | node | node[0-9]* | python | python[0-9] | python[0-9].[0-9]* | uv | uvx | npx | bun | deno) continue ;; + esac + # A tree helper is named for the extension it came from as well as for its + # script, because the script name alone re-hides precisely what this role + # exists to reveal: on the live tree the toml server is `dist/server.js`, + # and so is roughly every other extension's language server. `server` as a + # breadcrumb answers nothing in a post-mortem. + if [[ $role == "treeHelper" && $tok == *"/extensions/"* ]]; then + rest=${tok##*"/extensions/"} + ext=${rest%%/*} + # Built-in extensions ship their servers under extensions/node_modules/, + # where the name worth having is one segment further in (typescript, + # not node_modules). + if [[ $ext == "node_modules" ]]; then + rest=${rest#node_modules/} + ext=${rest%%/*} + fi + if [[ -n $ext && $ext != "$seg" ]]; then + printf '%s/%s' "$ext" "${seg%.js}" + return 0 + fi + fi + printf '%s' "${seg%.js}" + return 0 + done + fi + fallback=${P_ARGV0[$pid]:-} + [[ -n $fallback ]] || fallback=${P_COMM[$pid]:-unknown} + printf '%s' "${fallback##*/}" + return 0 +} + +redact() { + local acc="" tok + for tok in $1; do + case "$tok" in + *token=* | *TOKEN=* | *key=* | *KEY=* | *secret=* | *SECRET=* | *password=* | *PASSWORD=*) + acc+=" ${tok%%=*}=" + ;; + *) acc+=" ${tok}" ;; + esac + done + printf '%s' "${acc# }" +} + +# --------------------------------------------------------------------------- # +# action - the only place that writes state or signals anything +# --------------------------------------------------------------------------- # + +LOG_LINES=0 +CSV_LINES=0 +SWEEP_LINES=0 + +# Appends one line to the local action log and nothing else. Used directly only +# by emit_stdout, so that a failure to reach stdout cannot recurse into itself. +append_action_log() { + printf '%s\n' "$1" >>"${STATE_DIR}/actions.log" + ((LOG_LINES += 1)) + if ((LOG_LINES > MAX_LOG_LINES)); then + mv -f "${STATE_DIR}/actions.log" "${STATE_DIR}/actions.log.1" 2>/dev/null + LOG_LINES=0 + fi + return 0 +} + +emit_stdout() { + ((STDOUT_OK)) || return 0 + [[ -n $STDOUT_PATH ]] || return 0 + if ! printf '%s\n' "$1" >>"$STDOUT_PATH" 2>/dev/null; then + STDOUT_OK=0 + append_action_log "component=memory-watchdog event=warning reason=stdout-unavailable path=${STDOUT_PATH} detail=@action lines are local-only from here@" + fi + return 0 +} + +# The one way an action is recorded. Arguments are logfmt pairs, and the whole +# line is logfmt - `component` first, so that a LogQL line filter +# (|= "component=memory-watchdog") and the logfmt parser both work on it without +# anyone adding a stream label. Free text goes in a detail= field and is quoted +# here rather than at each call site, because a call site that forgot would +# produce a line that parses into the wrong fields rather than an obvious error. +record_action() { + local stamp line="$*" + printf -v stamp '%(%Y-%m-%dT%H:%M:%S%z)T' -1 + line=${line//@/\"} + line="component=memory-watchdog time=${stamp} mode=${MODE} ${line}" + append_action_log "$line" + emit_stdout "$line" + return 0 +} + +fmt_gib() { + local v=$1 sign="" + if ((v < 0)); then + sign="-" + v=$((-v)) + fi + printf '%s%d.%02d GiB' "$sign" "$((v / GIB))" "$((v % GIB * 100 / GIB))" +} + +fmt_mib() { + printf '%d MiB' "$(($1 / MIB))" +} + +publish_headroom() { + printf '%s free (%s)\n' "$(fmt_gib "$1")" "$2" >"${STATE_DIR}/headroom.tmp" && + mv -f "${STATE_DIR}/headroom.tmp" "${STATE_DIR}/headroom" + return 0 +} + +# The workspace UI tile: the largest policed process as a share of its budget. +# This is the number the operator used to obtain by running ps himself, which is +# the habit this whole change exists to end. +publish_top() { + local best="none" line + if [[ -n ${TOP_ROLE:-} ]]; then + printf -v line '%s %s (%d%% of %s)' \ + "$TOP_ID" "$(fmt_mib "$TOP_PSS")" "$TOP_PCT" "$(fmt_mib "$TOP_BUDGET")" + best=$line + fi + printf '%s\n' "$best" >"${STATE_DIR}/top.tmp" && + mv -f "${STATE_DIR}/top.tmp" "${STATE_DIR}/top" + return 0 +} + +# du_bytes_per_s is (M_U - PREV_U) / dt, dt being one SAMPLE_INTERVAL (10s by +# default) - an instantaneous rate, not a drift rate, and naively averaging the +# column conflates the two. On the live pod that naive mean reads ~19.8 kB/s +# (~68 MB/hour); the same quantity taken from the hourly census's u_mb over a +# 26.1h window reads ~18.4 MB/hour, a 3.7x difference. The naive mean is +# dominated by a handful of 10s allocation/GC spikes (+102 MB/s to -58 MB/s) +# that wash out over an hour but dominate a per-row average. Read this column +# as "what changed in the last sample", not as "the drift rate" - for the +# latter, difference u_mb across two event=census rows far apart in time. +CSV_HEADER="ts,mem_max,mem_current,u,h,anon,shmem,unevictable,slab_unreclaimable,slab_reclaimable,kernel_stack,pagetables,sec_pagetables,percpu,sock,file,psi_full_avg10_centi,refault_file_per_s,pgscan_direct_per_s,pressure,du_bytes_per_s" + +append_calibration() { + local csv="${STATE_DIR}/calibration.csv" first="" + if [[ -s $csv ]]; then + # A file written by an earlier version has different columns, and appending + # to it would produce one CSV that is silently two schemas. Rotate it. + read -r first 2>/dev/null <"$csv" + if [[ $first != "$CSV_HEADER" ]]; then + mv -f "$csv" "${csv}.1" 2>/dev/null + fi + fi + if [[ ! -s $csv ]]; then + printf '%s\n' "$CSV_HEADER" >"$csv" + fi + printf '%s\n' "$*" >>"$csv" + ((CSV_LINES += 1)) + if ((CSV_LINES > MAX_CSV_LINES)); then + mv -f "$csv" "${csv}.1" 2>/dev/null + CSV_LINES=0 + fi + return 0 +} + +SWEEP_HEADER=$'#ts\tsweep\tpid\tstart\trole\tstate\tpss_kb\trss_kb\tage_s\tbudget_kb\tover_s\tguard\tidentity\tcmd' + +# The per-process record. It is the point of this version of the script as much +# as the killing is: every post-mortem in this investigation was unanswerable +# because the watchdog computed exactly this table on every cycle and then threw +# it away, so "what was holding the memory ten minutes before the kill" had no +# source. It is a local file first - Prometheus and Loki can be added later and +# cannot be backfilled - and it is written whatever the mode, because observe +# mode is where the budgets get their evidence. +append_sweep() { + local dst="${STATE_DIR}/sweep.log" first="" + if [[ -s $dst ]]; then + read -r first 2>/dev/null <"$dst" + [[ $first != "$SWEEP_HEADER" ]] && mv -f "$dst" "${dst}.1" 2>/dev/null + fi + [[ -s $dst ]] || printf '%s\n' "$SWEEP_HEADER" >"$dst" + printf '%s\n' "$@" >>"$dst" + SWEEP_LINES=$((SWEEP_LINES + $#)) + if ((SWEEP_LINES > MAX_SWEEP_LINES)); then + mv -f "$dst" "${dst}.1" 2>/dev/null + SWEEP_LINES=0 + fi + return 0 +} + +signal_pid() { + local sig=$1 pid=$2 role=$3 why=$4 + # Second, independent guard. Selection already excluded these; this exists so + # that a defect in tree-walking still cannot reach a protected process. + if [[ -n ${PROTECTED[$pid]:-} ]] || is_never_signal "$pid"; then + record_action "event=refused sig=${sig} pid=${pid} role=${role} reason=protected detail=@${why}@" + return 1 + fi + # And the positional rules, restated independently of selection: inside the + # server tree, the only thing that may be signalled without being one of the + # editor's own helpers is a helper a session spawned directly. + if [[ -n ${NOT_EDITOR[$pid]:-} && $role != "claudeHelper" ]]; then + record_action "event=refused sig=${sig} pid=${pid} role=${role} reason=${PROTECT_REASON[$pid]:-not-editor} detail=@${why}@" + return 1 + fi + [[ $MODE == "observe" ]] || kill "-${sig}" "$pid" 2>/dev/null + return 0 +} + +# Called after a kill. It counts, and it reports a rate; it disarms nothing. +# +# What it used to do - permanently disarm a role after LOOP_KILLS kills in +# LOOP_WINDOW - is described at LOOP_WINDOW above and was removed with the +# envelope, not lost. The loop it guarded against is now impossible by +# construction (see the oversize rule in sweep_once), and under a deliberate cap +# repeated kills are a fact about the workload rather than proof of a wrong +# number. What is left is worth saying out loud, because a role being recycled +# every twenty minutes is a sizing decision somebody should look at. +record_kill() { + local role=$1 now=$2 + local t keep="" n=0 all="" na=0 + + KILLS[$role]=$((${KILLS[$role]:-0} + 1)) + KILLS_TOTAL=$((KILLS_TOTAL + 1)) + LAST_KILL_AT=$now + + for t in ${KILL_TIMES[$role]:-}; do + ((now - t < LOOP_WINDOW)) || continue + keep+="${t} " + ((n += 1)) + done + keep+="${now}" + ((n += 1)) + KILL_TIMES[$role]=$keep + + for t in ${KILL_TIMES_ALL:-}; do + ((now - t < LOOP_WINDOW)) || continue + all+="${t} " + ((na += 1)) + done + all+="${now}" + ((na += 1)) + KILL_TIMES_ALL=$all + + if ((n >= LOOP_KILLS)); then + record_action "event=kill-rate role=${role} kills=${n} window_s=${LOOP_WINDOW} budget_mb=$((${BUDGET[$role]:-0} / MIB)) enforcing=yes detail=@this role is being recycled often enough to be worth a look; enforcement continues - raise WATCHDOG_BUDGET_${role}, change the envelope, or make the process smaller@" + fi + if ((na >= GLOBAL_LOOP_KILLS)); then + record_action "event=kill-rate role=all kills=${na} window_s=${LOOP_WINDOW} enforcing=yes detail=@kills across several roles in one window; the apportionment may not fit this workload@" + fi + return 0 +} + +# --------------------------------------------------------------------------- # +# the sweep - measure every process, decide, act, record +# --------------------------------------------------------------------------- # + +# Prunes dwell and kill bookkeeping for keys whose process is gone. Without this +# the maps grow for the life of the pod, and - worse - a recycled pid could +# inherit an entry. Keys are pid:starttime, so recycling changes the key. +prune_state() { + local key pid start + local -A live=() + for pid in "${PIDS[@]}"; do + live["${pid}:${P_START[$pid]:-0}"]=1 + done + for key in "${!OVER_SINCE[@]}"; do + [[ -n ${live[$key]:-} ]] || unset 'OVER_SINCE[$key]' + done + for key in "${!KILLED_AT[@]}"; do + [[ -n ${live[$key]:-} ]] || unset 'KILLED_AT[$key]' + done + for key in "${!FIT_SEEN[@]}"; do + [[ -n ${live[$key]:-} ]] || unset 'FIT_SEEN[$key]' + done + return 0 +} + +# On aggregation, and why a budget is compared per process rather than per role. +# +# Every budget below is checked against one pid's PSS, and nothing sums a role. +# A workspace running N same-role processes can therefore sit at N x +# (just-under-budget) indefinitely and trip nothing, which is a real hole and is +# recorded here rather than fixed, because fixing it costs more than it is worth +# on the evidence available: +# +# - The evidence says the case is not occurring. The only role that runs more +# than one process on this workspace is tsserver - two of them, measured at +# 112 and 141 MB PSS against a 768 MiB budget each. The sum is a third of one +# budget. +# - A per-role total changes what a kill *means*. If three siblings each hold a +# third of the total, none of them is the offender, and killing the largest is +# an arbitrary choice dressed as a decision. +# - It does not compose with the dwell clock, which is keyed per pid:starttime +# precisely so that load can be told from drift. A role-level clock cannot +# distinguish "one process drifting" from "processes being created", and the +# second is a supervisor's business, not this loop's. +# +# The envelope makes this hole more visible rather than less, and it is worth +# saying exactly what it does and does not claim. 2048 MiB is enforced as an +# *apportionment*: seven shares that add up to it, each compared against one +# process at a time. It is not enforced as a sum, so a role with three members +# can hold three times its share without any single process being over budget. +# +# So the sum is measured instead. TREE_PSS is computed every sweep, published in +# the census as tree_pss_mb/envelope_pct and in the summary, and written into the +# TOTAL row of every sweep. An envelope whose total was never computed would be a +# number nobody could check; one that is computed and published is a claim a +# reviewer can falsify from the log. Acting on it - shedding by total rather than +# by process - is deliberately not attempted: that is the graded ladder this file +# used to contain, which was removed because a poll loop cannot see the events it +# was reacting to. +sweep_once() { + local now=$1 + local pid role budget pss key over_since over_s state guard id cmd age + local -a rows=() + local policed_n=0 over_n=0 killed_n=0 total_pss=0 oversize_n=0 + + read_uptime + read_process_table + build_server_tree + compute_protected + compute_policed + read_usage "${PIDS[@]}" + prune_state + + ROLE_SUM_PSS=() + TREE_PSS=0 + + TOP_ROLE="" + TOP_ID="" + TOP_PSS=0 + TOP_BUDGET=0 + TOP_PCT=0 + + for pid in "${!POLICED[@]}"; do + role=${POLICED[$pid]} + budget=${BUDGET[$role]:-0} + pss=${P_PSS[$pid]:-0} + age=${P_AGE[$pid]:-0} + key="${pid}:${P_START[$pid]:-0}" + ((policed_n += 1)) + total_pss=$((total_pss + pss)) + # Per-role totals and peaks. Deliberately reported and not acted on: see + # the aggregation note above sweep_once. + ROLE_SUM_PSS[$role]=$((${ROLE_SUM_PSS[$role]:-0} + pss)) + ((pss > ${ROLE_PEAK_PSS[$role]:-0})) && ROLE_PEAK_PSS[$role]=$pss + # What the envelope is a claim about. Enforcement is per process (see the + # aggregation note above) so this total is measured and published rather than + # acted on - but an envelope whose total was never computed would be a number + # nobody could check. + [[ ${ENVELOPE_ROLES} == *" ${role} "* ]] && TREE_PSS=$((TREE_PSS + pss)) + + # The oversize rule, stated once: a process counts as having fitted only if + # it was seen at or below its budget while old enough to have finished + # starting up. Without the age test the flag would be set by whichever sweep + # happened to catch a process a second after exec, when everything is small - + # which would make the classification a function of sweep timing rather than + # of the process. + if ((budget > 0 && age >= MIN_AGE && pss <= budget)); then + FIT_SEEN[$key]=1 + fi + + id="$(identity_of "$pid" "$role")" + if ((budget > 0)) && ((pss * 100 / budget > TOP_PCT)); then + TOP_ROLE=$role + TOP_PSS=$pss + TOP_BUDGET=$budget + TOP_PCT=$((pss * 100 / budget)) + TOP_ID=$id + fi + + state=ok + over_s=0 + if ((budget > 0 && pss > budget)); then + ((over_n += 1)) + : "${OVER_SINCE[$key]:=$now}" + over_since=${OVER_SINCE[$key]} + over_s=$((now - over_since)) + state=over + if ((over_s >= DWELL_SECONDS)) && ((age >= MIN_AGE)); then + state=over-dwell + if [[ -n ${KILLED_AT[$key]:-} ]]; then + # Already asked politely. Escalate once the grace has elapsed. + # + # This is tested *before* the disarm check, and that ordering is the + # decision rather than an accident of layout. A drill caught the other + # ordering: the breaker tripped between a process's SIGTERM and its + # SIGKILL, the disarm branch short-circuited, and the process was still + # alive and still over budget at the end of the run - signalled, + # abandoned, and recorded as neither killed nor spared. + # + # Disarming means "stop deciding to kill this role", not "abandon a + # kill already decided". Completing an escalation cannot start the + # loop the breaker exists to stop, because it finishes one action + # rather than beginning another - record_kill is deliberately not + # called again here, since this is the same kill it already counted. + # Leaving a process half-signalled is the worst of both outcomes: a + # helper that caught SIGTERM and began shutting down is not something + # to walk away from, and one that ignored it goes on holding the + # memory with the watchdog having decided not to look again. + if ((now - KILLED_AT[$key] >= KILL_GRACE)); then + if signal_pid KILL "$pid" "$role" "drift ${role}"; then + state=killed-9 + record_action "event=kill sig=KILL pid=${pid} role=${role} id=${id} pss_mb=$((pss / MIB)) budget_mb=$((budget / MIB)) over_s=${over_s} detail=@still over budget ${KILL_GRACE}s after SIGTERM@" + fi + else + state=terminating + fi + elif [[ -z ${FIT_SEEN[$key]:-} ]]; then + # Structurally over budget rather than drifted into it: this process + # has never been observed inside its share since it started, so killing + # it accomplishes nothing - the replacement arrives over budget too, and + # that is a kill loop with a supervisor's restart in the middle of it. + # + # It is reported instead, on the same once-per-dwell cadence as + # would-kill, and it is reported forever: the loop cannot silence + # itself, cannot widen the budget, and re-arms the moment a replacement + # is seen fitting. The known gap is a process that is born far too big + # and then grows further; the watchdog will record it and never signal + # it. That is accepted, because a process born at ten times its share is + # not the drift this loop polices, and the sweep log names it either way. + state=oversize + ((oversize_n += 1)) + record_action "event=oversize pid=${pid} role=${role} id=${id} pss_mb=$((pss / MIB)) budget_mb=$((budget / MIB)) over_s=${over_s} age_s=${age} detail=@never observed inside its share since it started; killing it would not make the replacement smaller@" + OVER_SINCE[$key]=$now + elif ! role_is_armed "$role"; then + # Observe mode, or a role this mode does not arm. Report the kill that + # would have happened, then restart the dwell clock so the same process + # is reported once per dwell period rather than on every sweep for the + # rest of its life. + state=would-kill + record_action "event=would-kill armed=no pid=${pid} role=${role} id=${id} pss_mb=$((pss / MIB)) budget_mb=$((budget / MIB)) over_s=${over_s} age_s=${age}" + OVER_SINCE[$key]=$now + else + if signal_pid TERM "$pid" "$role" "drift ${role}"; then + KILLED_AT[$key]=$now + state=killed + ((killed_n += 1)) + record_action "event=kill sig=TERM pid=${pid} role=${role} id=${id} pss_mb=$((pss / MIB)) budget_mb=$((budget / MIB)) over_s=${over_s} age_s=${age}" + record_kill "$role" "$now" + fi + fi + fi + else + unset 'OVER_SINCE[$key]' + unset 'KILLED_AT[$key]' + fi + + guard=${PROTECT_REASON[$pid]:-'-'} + cmd="$(redact "${P_CMD[$pid]:-}")" + rows+=("${now}"$'\t'"${SWEEPS}"$'\t'"${pid}"$'\t'"${P_START[$pid]:-0}"$'\t'"${role}"$'\t'"${state}"$'\t'"$((pss / 1024))"$'\t'"$((${P_RSS[$pid]:-0} / 1024))"$'\t'"${age}"$'\t'"$((budget / 1024))"$'\t'"${over_s}"$'\t'"${guard}"$'\t'"${id}"$'\t'"${cmd:0:160}") + done + + # Everything else that is large enough to matter, policed or not. This is what + # makes the log answerable about the processes the watchdog does *not* manage - + # which, on the evidence of every OOM recorded for this workspace, is where the + # memory actually was. + for pid in "${PIDS[@]}"; do + [[ -n ${POLICED[$pid]:-} ]] && continue + pss=${P_PSS[$pid]:-0} + ((pss >= SWEEP_LOG_FLOOR)) || continue + guard=${PROTECT_REASON[$pid]:-'-'} + cmd="$(redact "${P_CMD[$pid]:-}")" + rows+=("${now}"$'\t'"${SWEEPS}"$'\t'"${pid}"$'\t'"${P_START[$pid]:-0}"$'\t'"unmanaged"$'\t'"seen"$'\t'"$((pss / 1024))"$'\t'"$((${P_RSS[$pid]:-0} / 1024))"$'\t'"${P_AGE[$pid]:-0}"$'\t'"0"$'\t'"0"$'\t'"${guard}"$'\t'"${P_COMM[$pid]:-unknown}"$'\t'"${cmd:0:160}") + done + + rows+=("${now}"$'\t'"${SWEEPS}"$'\t'"0"$'\t'"0"$'\t'"TOTAL"$'\t'"${PRESSURE}"$'\t'"$((total_pss / 1024))"$'\t'"0"$'\t'"0"$'\t'"0"$'\t'"0"$'\t'"-"$'\t'"policed=${policed_n},over=${over_n},oversize=${oversize_n},killed=${killed_n},procs=${#PIDS[@]},tree=${#SERVER_TREE[@]},sessions=${#CLAUDE_ROOTS[@]},tree_mb=$((TREE_PSS / MIB)),envelope_mb=$((ENVELOPE / MIB))"$'\t'"h=${M_H},u=${M_U},psi=${M_PSI_CENTI},mode=${MODE}") + append_sweep "${rows[@]}" + printf '%s\n' "$SWEEP_HEADER" "${rows[@]}" >"${STATE_DIR}/sweep.latest.tmp" && + mv -f "${STATE_DIR}/sweep.latest.tmp" "${STATE_DIR}/sweep.latest" + + POLICED_N=$policed_n + OVER_N=$over_n + OVERSIZE_N=$oversize_n + publish_top + return 0 +} + +# The falsification the census used to provide, generalised. A watchdog that +# manages nothing looks exactly like a watchdog with nothing to do, and the only +# difference is whether anything was ever there to manage. Nothing is *wrong* +# with a workspace where VS Code is closed and no session is running - but it is +# worth one line in the log, because the alternative reading is that selection is +# broken, and that reading has been correct before. +# +# The latch is per *episode*, not per process lifetime. It used to be neither: a +# one-shot flag was set at the warmup sweep whether or not it warned, and the +# condition it tested was a lifetime high-water mark, so a watchdog that policed +# something once in its first half hour could never warn again however blind it +# went afterwards. A pod lives for days; the interesting blind spot is the one +# that appears on day two, when an upgrade changes a launch shape and selection +# quietly stops matching. So the clock restarts whenever the blindness does, and +# re-arms as soon as anything is policed again - one line per episode, not one +# line per pod. +VISIBILITY_WARMUP="${WATCHDOG_VISIBILITY_WARMUP:-30}" # sweeps + +check_visibility() { + if ((${POLICED_N:-0} > 0)); then + BLIND_SINCE=-1 + VISIBILITY_WARNED=0 + return 0 + fi + ((BLIND_SINCE >= 0)) || BLIND_SINCE=$SWEEPS + ((VISIBILITY_WARNED)) && return 0 + ((SWEEPS - BLIND_SINCE >= VISIBILITY_WARMUP)) || return 0 + VISIBILITY_WARNED=1 + record_action "event=warning reason=nothing-policed sweeps=${SWEEPS} blind_since_sweep=${BLIND_SINCE} detail=@either nothing this watchdog manages has run, or selection is finding nothing; sweep.latest shows what was there@" + return 0 +} + +# One line an hour, whatever is happening. Two reasons it earns its place in a +# log pipeline that the sweep does not: it answers "was the watchdog alive and +# what was it seeing" for any window in the past, which no in-pod file can once +# the pod is gone; and it is the trend series for the standing population - the +# thing this watchdog exists to bound - at 24 lines a day rather than 40,000. +emit_census() { + local now=$1 role sums="" peaks="" + # The per-role total is what makes the un-fixed aggregation hole observable + # (see the note above sweep_once), and the per-role peak is how a share that no + # longer matches the workload becomes visible. Neither is acted on: a watchdog + # that re-derived a limit from what it observed would raise the limit it is + # enforcing every time the thing it enforces against grew, which is the one + # move this design forbids outright. + for role in "${!ROLE_SUM_PSS[@]}"; do + sums+="${role}=$((${ROLE_SUM_PSS[$role]} / MIB)) " + done + for role in "${!ROLE_PEAK_PSS[@]}"; do + peaks+="${role}=$((${ROLE_PEAK_PSS[$role]} / MIB)) " + done + LAST_CENSUS_AT=$now + record_action "event=census sweeps=${SWEEPS} uptime_s=$((now - STARTED_AT))" \ + "policed=${POLICED_N:-0} over_budget=${OVER_N:-0} sessions=${#CLAUDE_ROOTS[@]}" \ + "tree=${#SERVER_TREE[@]} kills_total=${KILLS_TOTAL} oversize=${OVERSIZE_N:-0}" \ + "envelope_mb=$((ENVELOPE / MIB)) tree_pss_mb=$((${TREE_PSS:-0} / MIB)) envelope_pct=$((${TREE_PSS:-0} * 100 / ENVELOPE))" \ + "h_mb=$((M_H / MIB)) u_mb=$((M_U / MIB)) pressure=${PRESSURE} psi_full10=${M_PSI_CENTI}" \ + "top_role=${TOP_ROLE:-none} top_id=${TOP_ID:-none} top_pss_mb=$((${TOP_PSS:-0} / MIB))" \ + "top_budget_pct=${TOP_PCT:-0} role_pss_mb=@${sums% }@ role_peak_mb=@${peaks% }@" + return 0 +} + +publish_summary() { + local now=$1 + local up=$((now - STARTED_AT)) + ((up > 0)) || up=1 + local role line="" + { + printf 'mode=%s uptime_s=%d sweeps=%d\n' "$MODE" "$up" "$SWEEPS" + printf 'h=%s h_min=%s h_max=%s pressure=%s\n' \ + "$(fmt_gib "$M_H")" "$(fmt_gib "$H_MIN_SEEN")" "$(fmt_gib "$H_MAX_SEEN")" "$PRESSURE" + printf 'policed=%d over_budget=%d sessions=%d tree=%d\n' \ + "${POLICED_N:-0}" "${OVER_N:-0}" "${#CLAUDE_ROOTS[@]}" "${#SERVER_TREE[@]}" + for role in "${!BUDGET[@]}"; do + line+="${role}=$((${BUDGET[$role]} / MIB))M " + done + printf 'budgets %s\n' "$line" + printf 'envelope=%dM tree_pss=%dM tree_pct=%d oversize=%d\n' \ + "$((ENVELOPE / MIB))" "$((${TREE_PSS:-0} / MIB))" \ + "$((${TREE_PSS:-0} * 100 / ENVELOPE))" "${OVERSIZE_N:-0}" + line="" + for role in "${!KILLS[@]}"; do line+="${role}:${KILLS[$role]} "; done + printf 'kills total=%d by_role=%s last=%s\n' \ + "$KILLS_TOTAL" "${line:-none}" "${LAST_KILL_AT:-0}" + # Per day rather than per hour: the operator's question is "not every + # fifteen minutes", and an hourly rate over a short uptime reads as a huge + # number for one event. + line="" + for role in "${!ROLE_PEAK_PSS[@]}"; do line+="${role}=$((${ROLE_PEAK_PSS[$role]} / MIB))M "; done + # The largest each role has been since this watchdog started. It is the + # standing evidence that a share no longer matches the workload, and it is + # reported for a human to act on rather than fed back into the budgets. + printf 'peak_pss %s\n' "${line:-none}" + printf 'kill_rate_per_day=%d.%02d\n' \ + "$((KILLS_TOTAL * 86400 / up))" "$((KILLS_TOTAL * 86400 * 100 / up % 100))" + printf 'dwell_s=%d min_age_s=%d loop_window_s=%d loop_kills=%d pss=%s\n' \ + "$DWELL_SECONDS" "$MIN_AGE" "$LOOP_WINDOW" "$LOOP_KILLS" \ + "$((PSS_UNAVAILABLE ? 0 : 1))" + } >"${STATE_DIR}/summary.tmp" && + mv -f "${STATE_DIR}/summary.tmp" "${STATE_DIR}/summary" + return 0 +} + +# --------------------------------------------------------------------------- # +# lifecycle +# --------------------------------------------------------------------------- # + +# A coder_script re-runs when the agent restarts without the pod restarting, so +# two watchdogs are otherwise entirely possible. +# +# Liveness decides, not the existence of a file. The previous version treated a +# failed O_EXCL create as evidence that another watchdog held the lock, and only +# then looked at whether the recorded pid was alive. That inverts the reliable +# test and the unreliable one, and it failed in exactly the situation this daemon +# exists for: the pod was OOM-killed, the watchdog died by SIGKILL without +# running its EXIT trap, and the pidfile survived on the NFS-backed home +# directory. Every restart afterwards logged "another instance is already +# running" and exited - observed on the test workspace, not theorised. +acquire_singleton() { + local pidfile="${STATE_DIR}/watchdog.pid" + local self="${BASH_SOURCE[0]}" + local other="" a + local -a argv=() + + if [[ -s $pidfile ]]; then + read -r other 2>/dev/null <"$pidfile" + if [[ $other =~ ^[0-9]+$ ]] && ((other != $$)) && + [[ -r ${PROC_DIR}/${other}/cmdline ]]; then + mapfile -d '' -t argv 2>/dev/null <"${PROC_DIR}/${other}/cmdline" + for a in "${argv[@]}"; do + [[ $a == "$self" ]] && return 1 + done + fi + fi + + printf '%s\n' "$$" >"$pidfile" 2>/dev/null || return 1 + # Two watchdogs started at the same instant would both get this far. Settle it + # by reading back who actually owns the file rather than by trusting the write. + /usr/bin/sleep 1 + read -r other 2>/dev/null <"$pidfile" + [[ $other == "$$" ]] +} + +release_singleton() { + local pidfile="${STATE_DIR}/watchdog.pid" owner="" + # Both traps can fire, so the second one finds no file. A redirection that + # fails is reported by the shell before `2>/dev/null` on the same command has + # taken effect, so the test is done here rather than swallowed there. + [[ -r $pidfile ]] || return 0 + read -r owner 2>/dev/null <"$pidfile" + # Never remove a pidfile another instance owns - that would hand the lock to a + # third one while the second is still running. + [[ $owner == "$$" ]] && rm -f "$pidfile" + return 0 +} + +# Returns 2 when the cgroup has no memory limit and there is nothing to measure +# against. +cycle_once() { + local now=$1 rc + + read_cgroup_memory + rc=$? + if ((rc == 2)); then + record_action "event=exit reason=no-memory-limit detail=@memory.max is unlimited, so there is nothing to budget against@" + return 2 + fi + ((rc == 0)) || return 1 + + if ((${#BUDGET[@]} == 0)); then + derive_budgets + local role in_force + in_force=$(envelope_in_force) + for role in ${BUDGET_ORDER}; do + record_action "event=budget role=${role} budget_mb=$((${BUDGET[$role]} / MIB)) envelope=$([[ ${ENVELOPE_ROLES} == *" ${role} "* ]] && printf yes || printf no) envelope_pct=$((${BUDGET[$role]} * 100 / ENVELOPE)) armed=$(role_is_armed "$role" && printf yes || printf no)" + done + record_action "event=budgets_derived memory_max_mb=$((M_MAX / MIB)) envelope_mb=$((ENVELOPE / MIB)) shares_sum_mb=$((in_force / MIB)) dwell_s=${DWELL_SECONDS} min_age_s=${MIN_AGE} loop_window_s=${LOOP_WINDOW} loop_kills=${LOOP_KILLS} armed=@$( + for role in ${BUDGET_ORDER}; do role_is_armed "$role" && printf '%s ' "$role"; done + )@" + # The shares are constants that must add up to the envelope. An override, or + # an edit that changes one number without changing another, breaks that - and + # it must say so rather than leaving a total nobody can reconstruct from the + # log. + ((in_force == ENVELOPE)) || record_action "event=warning reason=envelope-mismatch shares_sum_mb=$((in_force / MIB)) envelope_mb=$((ENVELOPE / MIB)) detail=@the shares in force do not add up to the envelope; a budget override or an edit has changed the total@" + fi + + read_cgroup_pressure + ((M_H > H_MAX_SEEN)) && H_MAX_SEEN=$M_H + ((H_MIN_SEEN == 0 || M_H < H_MIN_SEEN)) && H_MIN_SEEN=$M_H + + local dt=$((now - PREV_AT)) + ((dt > 0)) || dt=1 + local refault_rate=0 pgscan_rate=0 du_rate=0 + if ((PREV_AT > 0)); then + refault_rate=$(((M_REFAULT_FILE - PREV_REFAULT) / dt)) + pgscan_rate=$(((M_PGSCAN_DIRECT - PREV_PGSCAN) / dt)) + du_rate=$(((M_U - PREV_U) / dt)) + fi + + append_calibration "${now},${M_MAX},${M_CURRENT},${M_U},${M_H},${M_ANON},${M_SHMEM},${M_UNEVICTABLE},${M_SLAB_UNRECLAIMABLE},${M_SLAB_RECLAIMABLE},${M_KERNEL_STACK},${M_PAGETABLES},${M_SEC_PAGETABLES},${M_PERCPU},${M_SOCK},${M_FILE},${M_PSI_CENTI},${refault_rate},${pgscan_rate},${PRESSURE},${du_rate}" + publish_headroom "$M_H" "$PRESSURE" + + if ((CYCLE % SWEEP_EVERY == 0)); then + sweep_once "$now" + ((SWEEPS += 1)) + check_visibility + publish_summary "$now" + # The first census goes out immediately, so that a watchdog that starts and + # then dies has still said what it saw. + if ((LAST_CENSUS_AT == 0 || now - LAST_CENSUS_AT >= CENSUS_EVERY)); then + emit_census "$now" + fi + fi + + PREV_AT=$now + PREV_U=$M_U + PREV_REFAULT=$M_REFAULT_FILE + PREV_PGSCAN=$M_PGSCAN_DIRECT + return 0 +} + +# Rotation counts lines written by *this* process, so a daemon that restarts +# every few hours would append to a file it believes is empty and the cap would +# never be reached. One fork each at startup fixes that; there is no way to ask +# bash for a file's size. +count_lines() { + local n=0 + [[ -r $1 ]] || { + printf '0' + return 0 + } + n=$(/usr/bin/wc -l 2>/dev/null <"$1") + printf '%s' "${n:-0}" +} + +main() { + mkdir -p "$STATE_DIR" || exit 1 + LOG_LINES=$(count_lines "${STATE_DIR}/actions.log") + CSV_LINES=$(count_lines "${STATE_DIR}/calibration.csv") + SWEEP_LINES=$(count_lines "${STATE_DIR}/sweep.log") + + normalise_mode + + if ! acquire_singleton; then + printf 'memory-watchdog: another instance is already running\n' >&2 + exit 0 + fi + trap 'release_singleton; exit 0' HUP INT TERM + trap release_singleton EXIT + + local now + STARTED_AT=${WATCHDOG_NOW:-$EPOCHSECONDS} + record_action "event=started pid=$$ cgroup=${CGROUP_DIR} sweep_every_s=$((SAMPLE_INTERVAL * SWEEP_EVERY)) dwell_s=${DWELL_SECONDS} stdout=${STDOUT_PATH}" + + while :; do + now=${WATCHDOG_NOW:-$EPOCHSECONDS} + cycle_once "$now" + (($? == 2)) && break + ((CYCLE += 1)) + + [[ $ONESHOT == "1" ]] && break + /usr/bin/sleep "$SAMPLE_INTERVAL" + done + return 0 +} + +# Sourcing with WATCHDOG_SOURCE_ONLY=1 exposes the functions to the test harness +# without starting the loop. +if [[ ${WATCHDOG_SOURCE_ONLY:-0} != "1" ]]; then + main "$@" +fi diff --git a/templates/kubernetes/homelab-workspace/script-prepare-workspace.sh b/templates/kubernetes/homelab-workspace/scripts/script-prepare-workspace.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-prepare-workspace.sh rename to templates/kubernetes/homelab-workspace/scripts/script-prepare-workspace.sh diff --git a/templates/kubernetes/homelab-workspace/script-service-command.sh b/templates/kubernetes/homelab-workspace/scripts/script-service-command.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-service-command.sh rename to templates/kubernetes/homelab-workspace/scripts/script-service-command.sh diff --git a/templates/kubernetes/homelab-workspace/script-start-services.sh b/templates/kubernetes/homelab-workspace/scripts/script-start-services.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-start-services.sh rename to templates/kubernetes/homelab-workspace/scripts/script-start-services.sh diff --git a/templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh b/templates/kubernetes/homelab-workspace/scripts/script-vscode-server-gc.sh similarity index 100% rename from templates/kubernetes/homelab-workspace/script-vscode-server-gc.sh rename to templates/kubernetes/homelab-workspace/scripts/script-vscode-server-gc.sh