From aa4cae9ce836fe9fa921aee4070dcd69d0d21b66 Mon Sep 17 00:00:00 2001 From: michelhelsdingen Date: Thu, 3 Sep 2026 10:57:05 +0200 Subject: [PATCH 1/2] fix(collab): verlaten runtime-mappen opruimen en de service onder launchd houden /tmp/ensemble hield op 03-09-2026 34 mappen vast: naast tien afgeronde teams 22 mappen van launches die stierven voordat een agent iets zei, en veertig lock-mapjes die de testsuite zelf achterliet (de afterEach van de team-say-tests verwijderde alleen messages.jsonl, niet de map met het lockbestand ernaast). collab-cleanup.sh keek alleen naar mappen met een .finished-marker en liet de rest voor eeuwig staan. De cleanup kent nu ook verlaten mappen: geen marker en nooit een bericht, na 24 uur weg. Een map met berichten blijft altijd staan, want het team kan nog lopen en collab-history.py leest die berichten later. COLLAB_RUNTIME_ROOT maakt de root overschrijfbaar, zodat de tests niet in de echte map werken. De team-say- tests ruimen hun hele map op. De service draaide als losse `tsx server.ts` zonder launchd-agent: weg na een herstart van de Mac, en met de hand te herstarten na elke codewijziging. scripts/install-launchd.sh registreert hem als launchd-agent met KeepAlive en de PATH van de installerende shell, zodat de agent-CLI's gevonden worden. Nagemeten: na een kill staat hij binnen vijf seconden weer op, en hij leest zijn eigen bestanden in ~/Documents zonder EPERM. Preflight herstart een service ouder dan 24 uur nu zelf via launchctl kickstart in plaats van de gebruiker een shell-regel te geven. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 + docs/TROUBLESHOOTING.md | 9 +++ docs/architecture.md | 3 +- docs/collab-scripts.md | 8 ++- docs/getting-started.md | 5 ++ scripts/collab-cleanup.sh | 75 ++++++++++++++++++++++-- scripts/collab-preflight.sh | 27 ++++++++- scripts/install-launchd.sh | 107 ++++++++++++++++++++++++++++++++++ tests/collab-cleanup.test.ts | 85 +++++++++++++++++++++++++++ tests/ensemble.test.ts | 4 +- tests/install-launchd.test.ts | 61 +++++++++++++++++++ 11 files changed, 374 insertions(+), 12 deletions(-) create mode 100755 scripts/install-launchd.sh create mode 100644 tests/collab-cleanup.test.ts create mode 100644 tests/install-launchd.test.ts diff --git a/README.md b/README.md index 8df742d..f53ec2b 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ npm install # Start the server (keep this running) npm run dev +# ...or on macOS, once, to keep it running across reboots: +./scripts/install-launchd.sh ``` ### Verify (in a second terminal) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 39a95e8..602ee06 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -21,6 +21,15 @@ cd ~/Documents/ensemble && nohup ./node_modules/.bin/tsx server.ts > /tmp/ensemb Then re-run `/collab` from the same shell where `claude --print "ok"` and `codex login status` both succeed. +On macOS you can hand the service to launchd instead, once: +```bash +./scripts/install-launchd.sh +``` +It then starts at login, comes back after a crash, and restarts on demand with +`launchctl kickstart -k gui/$(id -u)/dev.ensemble.server`. The agent carries the +PATH of the shell that installed it, so install from a shell where the agent CLIs +work. When preflight finds the service older than 24h it does the kickstart itself. + ### Why the preflight catches it `scripts/collab-preflight.sh` runs before every team spawn, and checks only the CLIs the run actually needs (`collab-preflight.sh codex,claude,grok`, or `COLLAB_AGENTS`): diff --git a/docs/architecture.md b/docs/architecture.md index cd9d3c0..3eb62f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -65,7 +65,8 @@ ensemble/ │ ├── collab-livefeed.sh # Continuous live feed │ ├── collab-status.sh # Multi-team dashboard │ ├── collab-replay.sh # Session replay -│ ├── collab-cleanup.sh # Temp file cleanup +│ ├── collab-cleanup.sh # Finished + abandoned runtime dir cleanup +│ ├── install-launchd.sh # macOS launchd agent for the server │ ├── team-say.sh # Agent message send │ ├── team-read.sh # Agent message read │ ├── ensemble-bridge.sh # File→HTTP message bridge diff --git a/docs/collab-scripts.md b/docs/collab-scripts.md index 40e7520..7f486d2 100644 --- a/docs/collab-scripts.md +++ b/docs/collab-scripts.md @@ -196,13 +196,19 @@ Shows: team name, status (active/finished/stale), message count, last message, d ## collab-cleanup.sh -**Remove finished team runtime directories** from `/tmp/ensemble/`. Dry-run by default. +**Remove finished and abandoned team runtime directories** from `/tmp/ensemble/`. Dry-run by default. ```bash ./scripts/collab-cleanup.sh # list what would be deleted ./scripts/collab-cleanup.sh --force # actually delete ``` +Finished directories (with a `.finished` marker) are removed after 24h, the latest +three are always kept. Abandoned directories, without a marker and without a single +message (a launch that died before the agents spoke, a stray lock directory), are +removed after 24h as well. A directory that holds messages is never touched: the +team may still be running, and `collab-history.py` reads those messages later. + This does not disband a running team. To end one, press `d` in the monitor or `POST /api/ensemble/teams//disband`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 405ccfe..e414e8d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -102,6 +102,11 @@ npm run dev You should see: `[Ensemble] Server running on http://127.0.0.1:23000` +On macOS, `./scripts/install-launchd.sh` registers the server as a launchd agent +instead: it starts at login, restarts after a crash, and logs to +`/tmp/ensemble-server.log`. Run it from a shell where your agent CLIs work; the +agent inherits that shell's PATH. Remove it again with `--uninstall`. + ### 3. Verify (in a second terminal) ```bash diff --git a/scripts/collab-cleanup.sh b/scripts/collab-cleanup.sh index ef8fe32..1c8835b 100755 --- a/scripts/collab-cleanup.sh +++ b/scripts/collab-cleanup.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash -# collab-cleanup.sh — Clean up old finished collab runtime directories. +# collab-cleanup.sh — Clean up old finished and abandoned collab runtime directories. # Usage: collab-cleanup.sh [--force] +# +# Finished: has a .finished marker (written on disband). The latest 3 and anything +# under 24h are kept. Abandoned: no marker and never a message (a launch that +# died before the agents spoke, or a stray lock directory); removed after 24h. +# COLLAB_RUNTIME_ROOT overrides the root, for tests. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=./collab-paths.sh source "$SCRIPT_DIR/collab-paths.sh" -ENSEMBLE_ROOT="/tmp/ensemble" +ENSEMBLE_ROOT="${COLLAB_RUNTIME_ROOT:-/tmp/ensemble}" KEEP_RECENT=3 MIN_AGE_SECONDS=$((24 * 60 * 60)) MODE="dry-run" @@ -55,6 +60,29 @@ finished_entries() { done | sort -rn } +# Newest mtime of a directory and everything in it. +newest_mtime() { + local dir="${1:?dir required}" newest=0 ts + while IFS= read -r -d '' entry; do + ts="$(mtime_epoch "$entry")" + [ "$ts" -gt "$newest" ] && newest="$ts" + done < <(find "$dir" -print0) + printf '%s\n' "$newest" +} + +# Directories without a .finished marker that never held a message. A team that +# spoke keeps its directory whatever its state: it may still be running, and its +# messages are what collab-history.py reads later. +abandoned_entries() { + [ -d "$ENSEMBLE_ROOT" ] || return 0 + find "$ENSEMBLE_ROOT" -mindepth 1 -maxdepth 1 -type d -print0 | + while IFS= read -r -d '' runtime_dir; do + [ -f "$runtime_dir/.finished" ] && continue + [ -s "$runtime_dir/messages.jsonl" ] && continue + printf '%s\t%s\n' "$(newest_mtime "$runtime_dir")" "$runtime_dir" + done | sort -rn +} + for arg in "$@"; do case "$arg" in --force) @@ -78,7 +106,15 @@ while IFS= read -r line; do ENTRIES+=("$line") done < <(finished_entries) +ABANDONED=() +while IFS= read -r line; do + ABANDONED+=("$line") +done < <(abandoned_entries) + TOTAL_FINISHED="${#ENTRIES[@]}" +TOTAL_ABANDONED="${#ABANDONED[@]}" +ABANDONED_ELIGIBLE=0 +ABANDONED_REMOVED=0 PRESERVED_RECENT=0 PRESERVED_FRESH=0 ELIGIBLE=0 @@ -98,8 +134,8 @@ if [ ! -d "$ENSEMBLE_ROOT" ]; then exit 0 fi -if [ "$TOTAL_FINISHED" -eq 0 ]; then - echo -e " ${Y}No finished collabs found.${R}" +if [ "$TOTAL_FINISHED" -eq 0 ] && [ "$TOTAL_ABANDONED" -eq 0 ]; then + echo -e " ${Y}No finished or abandoned collabs found.${R}" exit 0 fi @@ -142,18 +178,45 @@ for idx in "${!ENTRIES[@]}"; do fi done +for entry in "${ABANDONED[@]}"; do + newest_ts="${entry%%$'\t'*}" + runtime_dir="${entry#*$'\t'}" + runtime_name="$(basename "$runtime_dir")" + age_seconds=$((NOW - newest_ts)) + age_hours=$((age_seconds / 3600)) + + if [ "$age_seconds" -lt "$MIN_AGE_SECONDS" ]; then + echo -e " ${C}skip${R} ${runtime_name} ${D}(no messages yet, ${age_hours}h old, may still be starting)${R}" + continue + fi + + ABANDONED_ELIGIBLE=$((ABANDONED_ELIGIBLE + 1)) + if [ "$MODE" = "force" ]; then + if rm -rf "$runtime_dir"; then + ABANDONED_REMOVED=$((ABANDONED_REMOVED + 1)) + echo -e " ${G}remove${R} ${runtime_name} ${D}(abandoned, never a message, ${age_hours}h old)${R}" + else + FAILED=$((FAILED + 1)) + echo -e " ${Y}failed${R} ${runtime_name} ${D}(abandoned, ${age_hours}h old)${R}" + fi + else + echo -e " ${Y}would rm${R} ${runtime_name} ${D}(abandoned, never a message, ${age_hours}h old)${R}" + fi +done + echo "" echo -e " ${BD}Stats${R}" echo -e " finished dirs: ${TOTAL_FINISHED}" echo -e " kept (latest 3): ${PRESERVED_RECENT}" echo -e " kept (<24h): ${PRESERVED_FRESH}" echo -e " eligible old dirs: ${ELIGIBLE}" +echo -e " abandoned dirs: ${TOTAL_ABANDONED} (${ABANDONED_ELIGIBLE} older than 24h)" if [ "$MODE" = "force" ]; then - echo -e " removed dirs: ${REMOVED}" + echo -e " removed dirs: ${REMOVED} finished, ${ABANDONED_REMOVED} abandoned" echo -e " failed removals: ${FAILED}" echo -e " reclaimed: $(human_kb "$REMOVED_KB")" else - echo -e " would remove: ${ELIGIBLE}" + echo -e " would remove: ${ELIGIBLE} finished, ${ABANDONED_ELIGIBLE} abandoned" echo -e " reclaimable: $(human_kb "$REMOVED_KB")" fi echo -e " finished footprint: $(human_kb "$TOTAL_KB")" diff --git a/scripts/collab-preflight.sh b/scripts/collab-preflight.sh index 4b733a5..7a8a7ef 100755 --- a/scripts/collab-preflight.sh +++ b/scripts/collab-preflight.sh @@ -92,11 +92,32 @@ if [ -n "$SERVER_PID" ]; then if [ -n "$AGE_SECS" ]; then AGE_HRS=$((AGE_SECS / 3600)) if [ "$AGE_HRS" -gt "$SERVICE_MAX_AGE_HOURS" ]; then - fail 2 "Ensemble service is ${AGE_HRS}h old (>${SERVICE_MAX_AGE_HOURS}h threshold) + # Under launchd (scripts/install-launchd.sh) a stale service is a restart + # away, so do that here instead of sending the user to a shell one-liner. + LAUNCHD_LABEL="${ENSEMBLE_LAUNCHD_LABEL:-dev.ensemble.server}" + LAUNCHD_TARGET="gui/$(id -u)/$LAUNCHD_LABEL" + if command -v launchctl > /dev/null 2>&1 && launchctl print "$LAUNCHD_TARGET" > /dev/null 2>&1; then + warn "Ensemble service is ${AGE_HRS}h old — restarting it through launchd ($LAUNCHD_LABEL)" + launchctl kickstart -k "$LAUNCHD_TARGET" + RESTARTED=0 + for _ in $(seq 1 10); do + sleep 1 + if curl -sf "$API/api/v1/health" > /dev/null 2>&1; then RESTARTED=1; break; fi + done + if [ "$RESTARTED" = 1 ]; then + ok "Ensemble service restarted (fresh process under launchd)" + else + fail 2 "Ensemble service did not come back within 10s after launchctl kickstart; check /tmp/ensemble-server.log" + fi + else + fail 2 "Ensemble service is ${AGE_HRS}h old (>${SERVICE_MAX_AGE_HOURS}h threshold) This is the 2026-05-08 stale-state issue: agents will spawn with broken auth. - Fix: pkill -f 'tsx server.ts' && cd ~/Documents/ensemble && nohup ./node_modules/.bin/tsx server.ts > /tmp/ensemble-server.log 2>&1 &" + Fix: pkill -f 'tsx server.ts' && cd ~/Documents/ensemble && nohup ./node_modules/.bin/tsx server.ts > /tmp/ensemble-server.log 2>&1 & + Or install the launchd agent once (scripts/install-launchd.sh) and preflight restarts it for you." + fi + else + ok "Ensemble service age: ${AGE_HRS}h (within ${SERVICE_MAX_AGE_HOURS}h limit)" fi - ok "Ensemble service age: ${AGE_HRS}h (within ${SERVICE_MAX_AGE_HOURS}h limit)" else warn "Could not determine service age (no usable ps) — stale-service check skipped" fi diff --git a/scripts/install-launchd.sh b/scripts/install-launchd.sh new file mode 100755 index 0000000..9aa1b31 --- /dev/null +++ b/scripts/install-launchd.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# install-launchd.sh — Keep the ensemble service running as a macOS launchd agent. +# Usage: install-launchd.sh [--uninstall] [--no-load] +# +# Writes ~/Library/LaunchAgents/