From a53ffc17fe5b4c6c2ba0de80e82adbbf16ec9253 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Thu, 30 Jul 2026 02:32:03 -0700 Subject: [PATCH 01/23] fix(tests): replace source-content .gitignore assertion with behavioral coverage (#1304) The second assertion in fm-gitignore-config.test.sh (added by #1261) greps .gitignore for a specific spelling of the config/ ignore pattern. It fails on a semantically equivalent pattern like config/** and does not prove Git actually ignores anything, per the completed source-content-test audit. Replace it with a real git check-ignore control test on a generated unrelated path, and strengthen the existing directory-coverage test with generated unpredictable direct and nested config/ paths. --- tests/fm-gitignore-config.test.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/fm-gitignore-config.test.sh b/tests/fm-gitignore-config.test.sh index d64e362880..5b864dd646 100755 --- a/tests/fm-gitignore-config.test.sh +++ b/tests/fm-gitignore-config.test.sh @@ -18,22 +18,30 @@ pass() { printf 'ok - %s\n' "$1" } +random_leaf() { + printf '%s-%s' "$1" "$$-$RANDOM-$RANDOM" +} + test_config_dir_ignored_as_category() { - local sample - for sample in config/anything config/nested/dir/file config/some-new-key.admin; do + local direct nested sample + direct="$(random_leaf config/unlisted-key)" + nested="config/$(random_leaf nested-dir)/$(random_leaf deep-file)" + for sample in "$direct" "$nested" config/some-new-key.admin; do git -C "$ROOT" check-ignore -q "$sample" \ || fail "git does not ignore $sample (config/ must be ignored as a directory)" done - pass "config/ is ignored as a directory, covering unlisted paths" + pass "config/ is ignored as a directory, covering unlisted and nested paths" } -test_config_not_ignored_by_name_by_name_list() { - # Regression guard: .gitignore must not go back to enumerating config/ entries - # by exact filename, since that reintroduces the same silent-drift failure. - grep -qE '^config/[^/]+$' "$ROOT/.gitignore" \ - && fail ".gitignore lists config/ entries by exact filename instead of ignoring the directory" - pass "no name-by-name config/ entries remain in .gitignore" +test_unrelated_path_stays_visible() { + # Control: a path outside config/ must remain visible to Git, so the + # coverage above is proven by contrast rather than an always-ignoring rule. + local sibling + sibling="$(random_leaf not-config)" + git -C "$ROOT" check-ignore -q "$sibling" \ + && fail "git unexpectedly ignores $sibling (outside config/)" + pass "an unrelated path outside config/ remains visible to git" } test_config_dir_ignored_as_category -test_config_not_ignored_by_name_by_name_list +test_unrelated_path_stays_visible From 79e62b8ba5c5bb2080c6e424a9e8af982849d78e Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:49:39 -0700 Subject: [PATCH 02/23] feat: bound and consolidate startup memory during stow (#1303) * Add bounded startup memory curation * no-mistakes(review): Record reproducible stow verification evidence * no-mistakes(review): Validate inherited secondmate stow evidence * no-mistakes(document): Document editable startup-memory budget propagation --- .agents/skills/bootstrap-diagnostics/SKILL.md | 3 +- .../skills/secondmate-provisioning/SKILL.md | 4 +- .agents/skills/stow/SKILL.md | 104 +++--- AGENTS.md | 3 +- bin/fm-bootstrap.sh | 21 ++ bin/fm-config-inherit-lib.sh | 49 ++- bin/fm-startup-memory-budget-lib.sh | 224 +++++++++++++ bin/fm-startup-memory-budget.sh | 94 ++++++ bin/fm-test-run.sh | 5 + docs/configuration.md | 16 +- docs/documentation-audiences.json | 4 + docs/verification/stow-memory.md | 217 ++++++++++++ .../fm-backend-herdr-presentation-e2e.test.sh | 2 +- tests/fm-bootstrap.test.sh | 1 + tests/fm-secondmate-harness.test.sh | 33 +- tests/fm-startup-memory-budget.test.sh | 310 ++++++++++++++++++ 16 files changed, 1026 insertions(+), 64 deletions(-) create mode 100644 bin/fm-startup-memory-budget-lib.sh create mode 100755 bin/fm-startup-memory-budget.sh create mode 100644 docs/verification/stow-memory.md create mode 100755 tests/fm-startup-memory-budget.test.sh diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 2b70879941..477980b8df 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -2,7 +2,7 @@ name: bootstrap-diagnostics description: >- Agent-only handling playbook for session-start bootstrap diagnostics. - Use whenever the session-start digest's bootstrap section prints an actionable diagnostic line - MISSING, MISSING_MANUAL, BACKEND_INVALID, NEEDS_GH_AUTH, TANGLE, CREW_DISPATCH invalid, FLEET_SYNC, PR_CHECK_MIGRATION, SECONDMATE_SYNC, SECONDMATE_LIVENESS, NUDGE_SECONDMATES, or FMX - or when a standalone bin/fm-bootstrap.sh run prints one of those lines. + Use whenever the session-start digest's bootstrap section prints an actionable diagnostic line - MISSING, MISSING_MANUAL, BACKEND_INVALID, NEEDS_GH_AUTH, TANGLE, STARTUP_MEMORY_BUDGET, CREW_DISPATCH invalid, FLEET_SYNC, PR_CHECK_MIGRATION, SECONDMATE_SYNC, SECONDMATE_LIVENESS, NUDGE_SECONDMATES, or FMX - or when a standalone bin/fm-bootstrap.sh run prints one of those lines. A silent bootstrap section, or a BOOTSTRAP_INFO fact, means no skill load. user-invocable: false metadata: @@ -27,6 +27,7 @@ When any diagnostic needs captain attention, report the plain consequence and re - `TANGLE: ` - the primary checkout is stranded on a feature branch instead of its default branch; `AGENTS.md` section 8 explains why this guard exists and what it protects. The work is safe on that branch ref; restore the primary to its default branch with the printed `git -C checkout `, then re-validate that branch in a proper worktree. This is the only sanctioned firstmate-initiated git write to the primary, and it is a non-destructive branch switch that strands nothing. +- `STARTUP_MEMORY_BUDGET: invalid config/startup-memory-budget - ` - the visible startup-memory budget is not a safe one-line positive decimal file; do not infer the default or propagate it. Correct the local primary file, then rerun session start so the normal convergence path can deliver the validated value to secondmate homes. - `CREW_DISPATCH: invalid config/crew-dispatch.json - ` - the optional dispatch profile file exists but failed low-cost bootstrap validation; stop profile-based dispatch, report the actionable error, and require correction of the malformed schema, unverified harness name, or invalid harness/effort pair rather than falling back around it or selecting a bad profile. - `FLEET_SYNC: : skipped: ` - a benign one-off skip (offline, no origin, local-only); bootstrap continued, investigate only if it blocks work. A skip can also report the bounded fleet-refresh timeout (`FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT`, or a fleet-size-aware default with a 20 second floor); a timeout never blocks startup. diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index f9e68937ab..44caa0cb1c 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -78,7 +78,7 @@ This section is the single owner of the secondmate sync and inherited-local-mate Before launch, `fm-spawn.sh --secondmate` locally fast-forwards the home to the primary firstmate checkout's current default-branch commit when it is safe; dirty, diverged, or in-flight homes launch unchanged with a warning. The locked session-start bootstrap sweep runs the same guarded fast-forward for every live secondmate home, discovered from `state/.meta` records with `kind=secondmate` (`data/secondmates.md` only backfills `home=` for older records). That no-fetch path is a purely local fast-forward of tracked files, never an origin fetch, and it never touches the gitignored operational dirs, so a secondmate's backlog, projects, and in-flight work are never disturbed; a linked worktree advances immediately, while a standalone clone that lacks the target receives firstmate updates through `/updatefirstmate`'s origin refresh. -The same launch and the same locked bootstrap sweep also propagate the primary's declared inherited local material: `config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, `config/backend`, `config/herdr-presentation-spaces`, and the one shared captain-preference file `data/captain-shared.md`. +The same launch and the same locked bootstrap sweep also propagate the primary's declared inherited local material: `config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, `config/backend`, `config/herdr-presentation-spaces`, `config/startup-memory-budget`, and the one shared captain-preference file `data/captain-shared.md`. Because these paths are gitignored, that propagation is a separate, primary-authoritative copy independent of the tracked-files fast-forward: it re-converges every live home whether or not its tracked files advanced, and it touches only the declared items. Propagation failures warn without blocking secondmate launch or session-start continuation, and the destination keeps whatever safely validated state the helper left behind. Inheritance copies the literal `config/crew-harness` file, so a secondmate's own crewmates use the primary's crewmate harness only when it names a concrete adapter such as `codex`; an unset or `default` value has nothing concrete to inherit, and the secondmate's own crewmates fall back to the secondmate's own or detected harness instead. @@ -99,7 +99,7 @@ Keep every `data/learnings.md` fully local by captain decision; route fleet-gene No AGENTS.md reread nudge is needed at spawn or respawn because the agent reads instructions fresh on launch; only the bootstrap sweep's running-home instruction-surface advance needs that AGENTS.md re-read. Bootstrap reports successful AGENTS.md re-read sends as `BOOTSTRAP_INFO:` and only emits `NUDGE_SECONDMATES:` when that send fails and needs retry. A separate, literal-content config reread is required whenever inherited `config/*` material changes under an already-running secondmate. -After each successful allowlisted config write, both the locked bootstrap convergence path and mid-session `bin/fm-config-push.sh` use the shared propagation report to build one per-home generation-specific private instruction file from the validated destination post-write bytes for only the allowlisted config items that actually changed for that home (`config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, `config/backend`, `config/herdr-presentation-spaces`), in deterministic allowlist order. +After each successful allowlisted config write, both the locked bootstrap convergence path and mid-session `bin/fm-config-push.sh` use the shared propagation report to build one per-home generation-specific private instruction file from the validated destination post-write bytes for only the allowlisted config items that actually changed for that home (`config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, `config/backend`, `config/herdr-presentation-spaces`, `config/startup-memory-budget`), in deterministic allowlist order. Each changed path is printed with clear begin/end delimiters and the destination file's full exact new bytes unparsed, or the explicit token `ABSENT` when propagation removed the destination copy. The instruction uses only minimal framing that these are defaults/rules and do not remove judgment; it never includes SHA values, selected profiles, parsed summaries, or any other generated interpretation. `data/captain-shared.md` is not a config file and is never inlined into this instruction file or message. diff --git a/.agents/skills/stow/SKILL.md b/.agents/skills/stow/SKILL.md index 4c2c2a337a..672894bd56 100644 --- a/.agents/skills/stow/SKILL.md +++ b/.agents/skills/stow/SKILL.md @@ -10,56 +10,76 @@ metadata: # stow -Sweep this session for durable knowledge that only exists in conversation right now, and write it to the disk locations firstmate already prints in the next session-start context digest. -The goal is a session that is safe to reset or destroy because everything durable has already been captured. +Sweep this session for durable knowledge that exists only in conversation, then leave the next session with a compact current operating map rather than an accumulating journal. +This skill writes only through the existing Firstmate ownership and write boundaries. -## What it does +## Required startup-memory pass -1. **Sweep the session for uncaptured durable knowledge.** - Read back over this conversation and look for: - - Operational learnings: fleet-local facts and gotchas discovered while operating firstmate (a script's sharp edge, a harness quirk, a recurring false alarm and its real cause). - - Captain preferences expressed in passing: a working-style or approval preference the captain stated conversationally rather than through the destination selected by AGENTS.md's knowledge-routing table. - - Project-intrinsic facts discovered: build, test, release, or architecture facts about a project that belong in that project's own `AGENTS.md`. - - Decisions made: a standing choice the captain made this session that should outlive it. - - Undone next steps: anything left open that has not yet been filed as backlog work. +Every `/stow` invocation performs this complete pass, even when the session contains no new finding: -2. **Route each finding using AGENTS.md's knowledge-routing table.** - AGENTS.md (section 6, "Knowledge routing") is the single source of truth for where each kind of knowledge belongs. - Read that table and route each finding there instead of re-deriving the mapping here. +1. Run `bin/fm-startup-memory-budget.sh report` before considering a write. + Record its effective budget and each file's estimated-token total. + The helper's stable estimate is the documented conservative local approximation, not provider-exact accounting. + If it rejects the setting or a memory file, do not infer a default or silently continue. + Report that concrete exception and do not call the session reset-safe. +2. Read every current memory file completely: `data/captain.md`, `data/captain-shared.md`, and `data/learnings.md`. + Treat an absent local file as absent, not as an invitation to manufacture content. + In a primary home, all three are curation inputs under their existing ownership rules. + In a secondmate home, `data/captain-shared.md` is a read-only primary-owned input: count it, never edit it, and curate only the editable local files. +3. Build one whole-file retention plan before editing. + Retain, in order: current captain preferences, authority and safety boundaries, and recurring working style; stable home-local operating facts that repeatedly affect future work and are expensive to rediscover; then concise pointers to an existing authoritative report, project document, configuration, or backlog item. + Retain lower-priority material only while budget remains. +4. Consolidate every editable memory file as needed, not only the file apparently related to a new finding. + Prefer one concise current rule or authoritative pointer over duplicate prose. + Remove, merge, or route completed incident and release chronology, stale versions and paths, transient task state, resolved alternatives, old metrics, superseded claims, duplicates, and report-sized procedures. + Do not remove a unique current fact unless it is preserved directly elsewhere through a stronger existing owner. +5. Run `bin/fm-startup-memory-budget.sh report` again after the complete pass. + Finish at or below the effective budget unless a concrete inability remains. + A secondmate must explicitly report `primary-owned-shared-file-alone-exceeds-budget` when the inherited shared file alone exceeds its allowance, because local curation cannot resolve it. + Any other unresolved excess must identify the fact that cannot safely be removed or routed and why. + +A net increase is allowed only for a genuinely new current fact with no stronger owner. +Before allowing it, consolidate enough lower-priority material to remain within budget. +Never describe the session as reset-safe while the memory total is over budget or an exception is unresolved. + +## Knowledge sweep and routing -3. **Write within firstmate's existing write boundaries.** - This skill does not grant any new write permission; it only prompts firstmate to use the boundaries that already exist (AGENTS.md section 1): - - Captain preferences and fleet-local operational facts: hand-write directly to the destination selected by AGENTS.md's knowledge-routing table, using inspect-then-update every time. - Before writing, inspect the destination, find the existing bullet or section the finding duplicates or supersedes, and rewrite it in place rather than adding a new trailing entry. - `data/learnings.md` may not exist yet; create it on first local learning, in the same dated, evidence-backed, curated style as the captain-preference files. - - Project-intrinsic knowledge: never hand-write a project's `AGENTS.md`. - Route it through a normal ship task so a crewmate records it via `bin/fm-ensure-agents-md.sh` and commits it through that project's delivery pipeline, exactly as section 6 describes. - If the fleet is live, delegate this to a crewmate rather than doing it inline. - - Knowledge generalizable to every firstmate user: this repo's own `AGENTS.md` (or other shared, tracked material), shipped through the normal branch -> no-mistakes -> PR -> captain-merge pipeline for this repo (section 1), never hand-committed straight to `main`. - - Task-scoped notes: inspect the relevant backlog item with `tasks-axi show --full`, judge whether the new note is new, duplicate, superseding, or obsolete, then write a considered replacement body with `tasks-axi update --body-file `. - When the replacement intentionally supersedes prior state that should remain recoverable, add `--archive-body` to that update command so the prior body stays recoverable without copying it into the replacement. +1. **Sweep the session for uncaptured durable knowledge.** + Look for operational learnings, captain preferences expressed in passing, project-intrinsic facts, standing decisions, and undone next steps. +2. **Route each finding using AGENTS.md's knowledge-routing table.** + AGENTS.md section 6 is the source of truth for destinations. + Do not re-derive or duplicate that mapping here. +3. **Write within the existing boundaries.** + - Captain preferences and fleet-local operational facts belong in the destination selected by AGENTS.md after the required whole-file curation pass. + Create `data/learnings.md` only for a genuinely new local learning with no stronger owner. + - In a primary home, curate shared captain preferences only under the existing primary-authoritative shared-preference contract. + In a secondmate home, route a newly discovered shared preference to the main firstmate through marked status or a document pointer instead of editing the inherited file. + - Project-intrinsic knowledge never goes directly into a project's `AGENTS.md`. + Route it through a normal ship task so a crewmate records it with `bin/fm-ensure-agents-md.sh` and the project's delivery path. + - Knowledge general to every Firstmate user belongs in this repo's shared tracked material through the normal branch, no-mistakes, PR, and captain-merge path. + - For task-scoped notes, inspect the item with `tasks-axi show --full`, classify the change as new, duplicate, superseding, or obsolete, then use a considered replacement body through `tasks-axi update --body-file `. + Use `--archive-body` when recoverability matters. Never append. - If hand-editing `data/backlog.md` per the active backend, make the same inspect-then-update edit in place. - - Undone next steps: file each as a queued backlog item (section 10), with `blocked-by` recorded if it genuinely depends on something else. + - File each undone next step as a queued backlog item with a genuine `blocked-by` dependency when applicable. +4. **Use inspect-then-update.** + For every retained fact, ask which current statement it supersedes, whether it can be a one-sentence rewrite, and whether a stale entry should be deleted, retired, or routed to an existing stronger owner. + The only graduation moves are promotion to tracked shared material through a PR, folding a learning into the captain-preference destination selected by AGENTS.md, or deletion of a stale entry. + Do not invent another graduation path. + +## Completion receipt + +Report the outcome in plain captain-facing language with all of these facts: -4. **Curate with inspect-then-update.** - Every write starts by reading the current destination and deciding how the finding changes what is already there. - Use this checklist before writing: - - Which existing bullet, section, or task body does this supersede? - - Can this be a one-sentence rewrite instead of a new entry? - - Should an older bullet or note be deleted, retired, or archived because it is now obsolete? - When a finding overlaps or supersedes something already on disk, rewrite or prune the existing entry instead of piling on a new one. - Graduation moves are limited to exactly three: promote a learning to the shared `AGENTS.md` via PR, fold it into the captain-preference destination selected by AGENTS.md, or delete a stale entry. - Do not invent other graduation paths. +- effective startup-memory budget and total estimated tokens before and after; +- one or more actions for each of `data/captain.md`, `data/captain-shared.md`, and `data/learnings.md`: `unchanged`, `added`, `rewritten`, `pruned`, or `routed`; +- each durable finding filed outside memory and its authoritative owner; +- every unresolved exception, including a primary-owned shared-file constraint in a secondmate home; +- whether the session is safe to reset, only when all durable findings are captured and the post-pass result is within budget with no exception. -5. **Report to the captain.** - Summarize, in plain outcome language (section 9): what was stowed and where, what was filed to the backlog, and whether the session is now safe to reset or destroy - i.e. whether every durable finding from this sweep now lives on disk rather than only in this conversation. - If something could not be captured yet (for example, project-intrinsic knowledge waiting on a crewmate to land it), say so explicitly rather than reporting the session fully safe. +Do not hide an over-budget result behind a reset-safe claim. ## Scope exclusion: no skill storage -`/stow` must **never** store, create, or edit a skill as a destination for any finding. +`/stow` must never store, create, or edit a skill as a destination for any finding. There is no "graduate this to a skill" move in this skill's routing. -This is a deliberate, standing exclusion, not an oversight: even with the two-tier skill layout, a stow sweep is a memory-routing operation, not a way to author or mutate skills. -Writing learnings into either `.agents/skills/` or public `skills/` would still risk mixing fleet-local material with shared firstmate behavior or standalone installer-facing behavior. -Until a human deliberately scopes a skill change as firstmate repo work, route generalizable knowledge to the shared `AGENTS.md` (or other shared, tracked material) via the pipeline, and fleet-local knowledge to `data/`, never to a skill. +Until a human deliberately scopes a skill change as Firstmate repository work, route generalizable knowledge to shared tracked material through its pipeline and fleet-local knowledge to `data/`, never to `.agents/skills/` or public `skills/`. diff --git a/AGENTS.md b/AGENTS.md index ee23c840dc..c75eeb7759 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,6 +70,7 @@ config/secondmate-harness harness the PRIMARY uses to launch SECONDMATE agents, config/backlog-backend backlog backend override; LOCAL, gitignored; absent or "tasks-axi" = default tasks-axi backend, "manual" = force routine backlog updates to hand-editing; inherited by secondmate homes (section 10) config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), while herdr, zellij, orca, and cmux are experimental spawn backends (docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; inherited by secondmate homes under the primary-authoritative contract in secondmate-provisioning config/calm Pi Calm presentation preference; LOCAL, gitignored, and not inherited; see docs/configuration.md "Pi Calm preference" +config/startup-memory-budget primary-authoritative per-home startup-memory budget; LOCAL, gitignored, materialized as 7,500 estimated tokens by locked primary bootstrap and inherited into secondmate homes; see docs/configuration.md "Startup memory budget" config/herdr-presentation-spaces optional presence flag for Herdr's default-off disposable single-task visual projection; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Optional presentation spaces" config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md @@ -473,7 +474,7 @@ It performs guarded fast-forward updates of firstmate and registered secondmate These skills are not captain-invocable; load them only at their precise triggers. -- `bootstrap-diagnostics` - load whenever the session-start digest's bootstrap section prints an actionable diagnostic line (`MISSING:`, `MISSING_MANUAL:`, `BACKEND_INVALID:`, `NEEDS_GH_AUTH`, `TANGLE:`, `CREW_DISPATCH: invalid`, `FLEET_SYNC:`, `PR_CHECK_MIGRATION:`, `SECONDMATE_SYNC:`, `SECONDMATE_LIVENESS:`, `NUDGE_SECONDMATES:`, or `FMX:`); silence and `BOOTSTRAP_INFO:` need no load. +- `bootstrap-diagnostics` - load whenever the session-start digest's bootstrap section prints an actionable diagnostic line (`MISSING:`, `MISSING_MANUAL:`, `BACKEND_INVALID:`, `NEEDS_GH_AUTH`, `TANGLE:`, `STARTUP_MEMORY_BUDGET:`, `CREW_DISPATCH: invalid`, `FLEET_SYNC:`, `PR_CHECK_MIGRATION:`, `SECONDMATE_SYNC:`, `SECONDMATE_LIVENESS:`, `NUDGE_SECONDMATES:`, or `FMX:`); silence and `BOOTSTRAP_INFO:` need no load. - `diagnostic-reasoning` - load before scoping a reported bug and before acting on a diagnostic report. - `ask-user-authority` - load before deciding any ask-user finding, regardless of the project's `yolo` posture. - `quota-array-dispatch` - load before choosing among a matched crew-dispatch profile array from current quota-axi output. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index fef86ba383..16102adfa4 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -8,6 +8,7 @@ # Lines: "MISSING: (install: )", # "MISSING_MANUAL: (instructions: )", "NEEDS_GH_AUTH", # "BACKEND_INVALID: (known: )", +# "STARTUP_MEMORY_BUDGET: invalid config/startup-memory-budget - ", # "CREW_DISPATCH: invalid config/crew-dispatch.json - ", # "FLEET_SYNC: : skipped|recovered|STUCK: ", # "PR_CHECK_MIGRATION: ", @@ -55,6 +56,11 @@ # tasks-axi default backend is silent. quota-axi is required for the # agent-owned dispatch-profile array procedure in AGENTS.md section 4 # and .agents/skills/quota-array-dispatch/SKILL.md. +# On a primary home, the locked mutable path materializes the visible +# default config/startup-memory-budget=7500 when absent. It never +# guesses at malformed or unsafe existing files, and secondmate homes +# await the primary-authoritative inherited value instead of creating +# their own. # X mode is OPTIONAL and inert unless FM_HOME/.env has a non-empty # FMX_PAIRING_TOKEN. When opted in, bootstrap requires curl+jq, writes # the relay poll shim and 30s cadence config, and prints an FMX line. @@ -97,6 +103,8 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" . "$SCRIPT_DIR/fm-ff-lib.sh" # shellcheck source=bin/fm-config-inherit-lib.sh disable=SC1091 . "$SCRIPT_DIR/fm-config-inherit-lib.sh" +# shellcheck source=bin/fm-startup-memory-budget-lib.sh disable=SC1091 +. "$SCRIPT_DIR/fm-startup-memory-budget-lib.sh" # shellcheck source=bin/fm-x-lib.sh disable=SC1091 . "$SCRIPT_DIR/fm-x-lib.sh" # shellcheck source=bin/fm-backend.sh disable=SC1091 @@ -805,6 +813,18 @@ crew_dispatch_validate() { fi } +startup_memory_budget_setup() { + # Primary bootstrap owns default publication. A secondmate is deliberately + # passive here because its setting must converge from the primary through the + # inherited-local-material contract rather than becoming a local authority. + if [ -e "$FM_HOME/.fm-secondmate-home" ] || [ -L "$FM_HOME/.fm-secondmate-home" ]; then + return 0 + fi + if ! fm_startup_memory_budget_materialize "$CONFIG"; then + echo "STARTUP_MEMORY_BUDGET: invalid config/$FM_STARTUP_MEMORY_BUDGET_FILE - $FM_STARTUP_MEMORY_BUDGET_ERROR" + fi +} + if [ "${1:-}" = "install" ]; then shift [ $# -gt 0 ] || { echo "usage: fm-bootstrap.sh install ..." >&2; exit 1; } @@ -827,6 +847,7 @@ fi # runnable. Detect-only sessions never touch state. if [ "${FM_BOOTSTRAP_DETECT_ONLY:-0}" != 1 ]; then "$SCRIPT_DIR/fm-pr-check-migrate.sh" || true + startup_memory_budget_setup fi if [ "$BACKEND_VALID" -eq 0 ]; then diff --git a/bin/fm-config-inherit-lib.sh b/bin/fm-config-inherit-lib.sh index 22109aa87a..bffbd5234d 100644 --- a/bin/fm-config-inherit-lib.sh +++ b/bin/fm-config-inherit-lib.sh @@ -6,7 +6,8 @@ # profile rules, primary config/crew-harness=codex makes a secondmate's crewmates # spawn on codex too, primary config/backlog-backend=manual makes that home # hand-edit backlog files too, primary config/backend pins that home's local -# runtime-backend default for future spawns, and primary +# runtime-backend default for future spawns, primary config/startup-memory-budget +# bounds that home's startup-memory curation, and primary # config/herdr-presentation-spaces enables the same default-off Herdr presentation # projection). It also pushes the one primary-authoritative shared # captain-preference file, data/captain-shared.md, into each secondmate home's @@ -32,6 +33,9 @@ # is deliberately NOT in the list: it is the primary's own setting for launching # secondmates, and a secondmate never spawns secondmates, so it must not flow # downstream. +# +# shellcheck source=bin/fm-startup-memory-budget-lib.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/fm-startup-memory-budget-lib.sh" # The one shared data file in this inheritance contract. There is deliberately # no shared learnings file. @@ -42,7 +46,7 @@ FM_SHARED_CAPTAIN_MODE="444" # The declared inheritable set (space-separated, config-dir-relative item paths). # Extend here to inherit more of the primary's local config; override via the # environment only in tests. Items must not contain whitespace. -FM_INHERITABLE_CONFIG="${FM_INHERITABLE_CONFIG:-crew-dispatch.json crew-harness backlog-backend backend herdr-presentation-spaces}" +FM_INHERITABLE_CONFIG="${FM_INHERITABLE_CONFIG:-crew-dispatch.json crew-harness backlog-backend backend herdr-presentation-spaces startup-memory-budget}" fm_inherit_file_mode() { if [ "$(uname)" = Darwin ]; then @@ -401,6 +405,47 @@ propagate_inheritable_config() { esac src="$src_config/$item" dest="$dest_config/$item" + # This one scalar config is consumed as a local safety boundary, so reject + # every unsafe or malformed source/destination artifact before the generic + # byte-copy behavior below can treat it as ordinary inherited material. + if [ "$item" = "$FM_STARTUP_MEMORY_BUDGET_FILE" ]; then + if [ -e "$src_config" ] || [ -L "$src_config" ]; then + if ! fm_startup_memory_budget_config_dir_safe "$src_config"; then + reason="unsafe primary config directory: $FM_STARTUP_MEMORY_BUDGET_ERROR" + warn_inheritable_config_error "$item" "$src_config" "$reason" + record_inheritable_config_result "$item" error "$reason" + rc=1 + continue + fi + fi + if [ -e "$dest_config" ] || [ -L "$dest_config" ]; then + if ! fm_startup_memory_budget_config_dir_safe "$dest_config"; then + reason="unsafe destination config directory: $FM_STARTUP_MEMORY_BUDGET_ERROR" + warn_inheritable_config_error "$item" "$dest_config" "$reason" + record_inheritable_config_result "$item" error "$reason" + rc=1 + continue + fi + fi + if [ -e "$src" ] || [ -L "$src" ]; then + if ! fm_startup_memory_budget_file_valid "$src"; then + reason="unsafe or invalid primary source: $FM_STARTUP_MEMORY_BUDGET_ERROR" + warn_inheritable_config_error "$item" "$src" "$reason" + record_inheritable_config_result "$item" error "$reason" + rc=1 + continue + fi + fi + if [ -e "$dest" ] || [ -L "$dest" ]; then + if ! fm_startup_memory_budget_file_valid "$dest"; then + reason="unsafe or invalid destination: $FM_STARTUP_MEMORY_BUDGET_ERROR" + warn_inheritable_config_error "$item" "$dest" "$reason" + record_inheritable_config_result "$item" error "$reason" + rc=1 + continue + fi + fi + fi if [ -f "$src" ]; then if ! destination_allows_inherited_item "$dest_config" "$item"; then reason=$(inheritable_config_skip_reason) diff --git a/bin/fm-startup-memory-budget-lib.sh b/bin/fm-startup-memory-budget-lib.sh new file mode 100644 index 0000000000..f2c06014b8 --- /dev/null +++ b/bin/fm-startup-memory-budget-lib.sh @@ -0,0 +1,224 @@ +# shellcheck shell=bash +# Startup-memory budget primitives. +# Usage: . bin/fm-startup-memory-budget-lib.sh +# +# The local, primary-authoritative config/startup-memory-budget setting is one +# strictly formatted positive decimal value followed by one newline. The +# locked primary bootstrap owns first materialization. This library owns safe +# parsing, default publication, and the portable prompt-memory estimate used by +# bin/fm-startup-memory-budget.sh and the internal /stow skill. + +FM_STARTUP_MEMORY_BUDGET_FILE="startup-memory-budget" +FM_STARTUP_MEMORY_BUDGET_DEFAULT="7500" +FM_STARTUP_MEMORY_BUDGET_ERROR="" +FM_STARTUP_MEMORY_BUDGET_VALUE="" +FM_STARTUP_MEMORY_MEASURE_BYTES="" +FM_STARTUP_MEMORY_MEASURE_TOKENS="" +FM_STARTUP_MEMORY_MEASURE_PRESENCE="" + +fm_startup_memory_budget_fail() { + FM_STARTUP_MEMORY_BUDGET_ERROR=$1 + return 1 +} + +fm_startup_memory_budget_link_count() { + if [ "$(uname)" = Darwin ]; then + stat -f %l "$1" 2>/dev/null + else + stat -c %h "$1" 2>/dev/null + fi +} + +fm_startup_memory_budget_config_dir_safe() { + local dir=$1 + if [ -L "$dir" ]; then + fm_startup_memory_budget_fail "config directory is symlinked" + return 1 + fi + if [ ! -d "$dir" ]; then + fm_startup_memory_budget_fail "config directory is not a directory" + return 1 + fi + return 0 +} + +# fm_startup_memory_budget_file_valid +# Sets FM_STARTUP_MEMORY_BUDGET_VALUE only for a regular, single-linked file +# containing exactly one positive decimal value and one terminating newline. +fm_startup_memory_budget_file_valid() { + local path=$1 links value + FM_STARTUP_MEMORY_BUDGET_VALUE="" + if [ -L "$path" ]; then + fm_startup_memory_budget_fail "file is symlinked" + return 1 + fi + if [ ! -e "$path" ]; then + fm_startup_memory_budget_fail "file is absent" + return 1 + fi + if [ ! -f "$path" ]; then + fm_startup_memory_budget_fail "file is not a regular file" + return 1 + fi + links=$(fm_startup_memory_budget_link_count "$path") || { + fm_startup_memory_budget_fail "could not inspect file link count" + return 1 + } + if [ "$links" != 1 ]; then + fm_startup_memory_budget_fail "file is hardlinked" + return 1 + fi + value=$(<"$path") || { + fm_startup_memory_budget_fail "could not read file" + return 1 + } + case "$value" in + ''|0|*[!0-9]*|0*) + fm_startup_memory_budget_fail "value must be one positive decimal integer" + return 1 + ;; + esac + if ! printf '%s\n' "$value" | cmp -s "$path" -; then + fm_startup_memory_budget_fail "file must contain exactly one value followed by one newline" + return 1 + fi + FM_STARTUP_MEMORY_BUDGET_VALUE=$value + return 0 +} + +# fm_startup_memory_budget_read +# Prints the validated decimal value. It never treats an absent or unsafe file +# as an implicit default because callers need a visible, auditable setting. +fm_startup_memory_budget_read() { + local config_dir=$1 path + fm_startup_memory_budget_config_dir_safe "$config_dir" || return 1 + path="$config_dir/$FM_STARTUP_MEMORY_BUDGET_FILE" + fm_startup_memory_budget_file_valid "$path" || return 1 + printf '%s\n' "$FM_STARTUP_MEMORY_BUDGET_VALUE" +} + +# fm_startup_memory_budget_materialize +# Atomically publishes the visible default only when the file is absent. A +# concurrent valid creator is accepted; every unsafe or malformed existing +# artifact is rejected without replacement. +fm_startup_memory_budget_materialize() { + local config_dir=$1 path tmp + if [ -e "$config_dir" ] || [ -L "$config_dir" ]; then + fm_startup_memory_budget_config_dir_safe "$config_dir" || return 1 + else + mkdir -p "$config_dir" 2>/dev/null || { + fm_startup_memory_budget_fail "could not create config directory" + return 1 + } + fm_startup_memory_budget_config_dir_safe "$config_dir" || return 1 + fi + + path="$config_dir/$FM_STARTUP_MEMORY_BUDGET_FILE" + if [ -e "$path" ] || [ -L "$path" ]; then + fm_startup_memory_budget_read "$config_dir" >/dev/null || return 1 + return 0 + fi + + tmp=$(umask 077; mktemp "$config_dir/.startup-memory-budget.XXXXXX" 2>/dev/null) || { + fm_startup_memory_budget_fail "could not create default temporary file" + return 1 + } + if ! printf '%s\n' "$FM_STARTUP_MEMORY_BUDGET_DEFAULT" > "$tmp" \ + || ! fm_startup_memory_budget_file_valid "$tmp"; then + rm -f "$tmp" + [ -n "$FM_STARTUP_MEMORY_BUDGET_ERROR" ] \ + || fm_startup_memory_budget_fail "could not write default value" + return 1 + fi + + # link(2) gives no-clobber publication in this directory. Removing the + # temporary name leaves the published file with exactly one link. + if ln "$tmp" "$path" 2>/dev/null; then + rm -f "$tmp" + fm_startup_memory_budget_read "$config_dir" >/dev/null || return 1 + return 0 + fi + rm -f "$tmp" + # Another actor may have created the file. Accept it only if it now meets + # the same safe, exact format - never replace or guess at it. + fm_startup_memory_budget_read "$config_dir" >/dev/null +} + +# fm_startup_memory_estimated_tokens_for_bytes +# The estimate is ceil(UTF-8 bytes / 3): stable, dependency-free, and +# deliberately conservative for ordinary prompt text without claiming provider +# exactness. +fm_startup_memory_estimated_tokens_for_bytes() { + local bytes=$1 tokens + case "$bytes" in + ''|*[!0-9]*) return 1 ;; + esac + tokens=$((bytes / 3)) + if [ $((bytes % 3)) -ne 0 ]; then + tokens=$((tokens + 1)) + fi + printf '%s\n' "$tokens" +} + +# fm_startup_memory_measure_file +# Prints " ". Memory files must be +# ordinary files when present so a measurement never follows a symlink or reads +# a special file. +fm_startup_memory_measure_file() { + local path=$1 bytes tokens + FM_STARTUP_MEMORY_MEASURE_BYTES="" + FM_STARTUP_MEMORY_MEASURE_TOKENS="" + FM_STARTUP_MEMORY_MEASURE_PRESENCE="" + if [ ! -e "$path" ] && [ ! -L "$path" ]; then + FM_STARTUP_MEMORY_MEASURE_BYTES=0 + FM_STARTUP_MEMORY_MEASURE_TOKENS=0 + FM_STARTUP_MEMORY_MEASURE_PRESENCE=absent + printf '0 0 absent\n' + return 0 + fi + if [ -L "$path" ] || [ ! -f "$path" ]; then + fm_startup_memory_budget_fail "memory file is not an ordinary regular file: $path" + return 1 + fi + bytes=$(LC_ALL=C wc -c < "$path" 2>/dev/null | tr -d '[:space:]') || { + fm_startup_memory_budget_fail "could not measure memory file: $path" + return 1 + } + case "$bytes" in + ''|*[!0-9]*) + fm_startup_memory_budget_fail "invalid byte count for memory file: $path" + return 1 + ;; + esac + tokens=$(fm_startup_memory_estimated_tokens_for_bytes "$bytes") || { + fm_startup_memory_budget_fail "could not estimate memory tokens for: $path" + return 1 + } + # shellcheck disable=SC2034 # Public measurement result consumed by the caller after sourcing. + FM_STARTUP_MEMORY_MEASURE_BYTES=$bytes + # shellcheck disable=SC2034 # Public measurement result consumed by the caller after sourcing. + FM_STARTUP_MEMORY_MEASURE_TOKENS=$tokens + # shellcheck disable=SC2034 # Public measurement result consumed by the caller after sourcing. + FM_STARTUP_MEMORY_MEASURE_PRESENCE=present + printf '%s %s present\n' "$bytes" "$tokens" +} + +# fm_startup_memory_decimal_le +# Decimal comparison without shell arithmetic overflow. Inputs are normalized +# non-negative decimal strings. +fm_startup_memory_decimal_le() { + local left=$1 right=$2 left_len right_len + case "$left:$right" in + *[!0-9:]*|:*|*:) return 1 ;; + esac + left_len=${#left} + right_len=${#right} + if [ "$left_len" -lt "$right_len" ]; then + return 0 + fi + if [ "$left_len" -gt "$right_len" ]; then + return 1 + fi + [ "$left" = "$right" ] && return 0 + [[ "$left" < "$right" ]] +} diff --git a/bin/fm-startup-memory-budget.sh b/bin/fm-startup-memory-budget.sh new file mode 100755 index 0000000000..715da54948 --- /dev/null +++ b/bin/fm-startup-memory-budget.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Read and account for the local startup-memory budget. +# Usage: +# fm-startup-memory-budget.sh read +# fm-startup-memory-budget.sh report +# +# `read` prints the one validated effective budget from +# config/startup-memory-budget. `report` prints the stable local estimate for +# data/captain.md, data/captain-shared.md, and data/learnings.md together. +# Bootstrap owns default materialization; this command never creates or repairs +# configuration, so an absent, malformed, symlinked, hardlinked, or otherwise +# unsafe value is a concrete error rather than an inferred default. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" +DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" + +# shellcheck source=bin/fm-startup-memory-budget-lib.sh +. "$SCRIPT_DIR/fm-startup-memory-budget-lib.sh" + +usage() { + sed -n '2,11{s/^# \{0,1\}//;p;}' "$0" +} + +print_error() { + printf 'startup-memory-budget: %s\n' "$1" >&2 +} + +read_budget() { + if ! fm_startup_memory_budget_read "$CONFIG" >/dev/null; then + print_error "invalid config/$FM_STARTUP_MEMORY_BUDGET_FILE - $FM_STARTUP_MEMORY_BUDGET_ERROR" + return 1 + fi + printf '%s\n' "$FM_STARTUP_MEMORY_BUDGET_VALUE" +} + +report() { + local budget bytes tokens presence total=0 shared_tokens=0 role=primary + if ! budget=$(read_budget); then + return 2 + fi + + if [ -e "$FM_HOME/.fm-secondmate-home" ] || [ -L "$FM_HOME/.fm-secondmate-home" ]; then + role=secondmate + fi + + printf 'estimator=ceil(UTF-8 bytes / 3) conservative-local-estimate\n' + printf 'role=%s\n' "$role" + printf 'effective_budget_tokens=%s\n' "$budget" + for file in captain.md captain-shared.md learnings.md; do + if ! fm_startup_memory_measure_file "$DATA/$file" >/dev/null; then + print_error "$FM_STARTUP_MEMORY_BUDGET_ERROR" + return 2 + fi + bytes=$FM_STARTUP_MEMORY_MEASURE_BYTES + tokens=$FM_STARTUP_MEMORY_MEASURE_TOKENS + presence=$FM_STARTUP_MEMORY_MEASURE_PRESENCE + total=$((total + tokens)) + [ "$file" != captain-shared.md ] || shared_tokens=$tokens + printf 'file=data/%s bytes=%s estimated_tokens=%s status=%s\n' \ + "$file" "$bytes" "$tokens" "$presence" + done + printf 'total_estimated_tokens=%s\n' "$total" + if fm_startup_memory_decimal_le "$total" "$budget"; then + printf 'budget_status=within-budget\n' + else + printf 'budget_status=over-budget\n' + fi + if [ "$role" = secondmate ] \ + && ! fm_startup_memory_decimal_le "$shared_tokens" "$budget"; then + printf 'exception=primary-owned-shared-file-alone-exceeds-budget\n' + fi +} + +case "${1:-}" in + read) + [ "$#" -eq 1 ] || { usage >&2; exit 2; } + read_budget + ;; + report) + [ "$#" -eq 1 ] || { usage >&2; exit 2; } + report + ;; + -h|--help) + usage + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 314232ae88..f89a6bade5 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -145,6 +145,7 @@ family_for_basename() { ;; fm-backlog-handoff.test.sh|fm-secondmate-harness.test.sh|fm-secondmate-lifecycle-e2e.test.sh|\ fm-secondmate-liveness.test.sh|fm-secondmate-safety.test.sh|fm-secondmate-sync.test.sh|\ + fm-startup-memory-budget.test.sh|\ fm-send-secondmate-marker.test.sh|fm-shared-captain-inheritance.test.sh) printf '%s\n' secondmate ;; @@ -644,6 +645,10 @@ families_for_changed_path() { printf '%s\n' live-harness-optin printf '%s\n' afk ;; + bin/fm-startup-memory-budget.sh|bin/fm-startup-memory-budget-lib.sh) + printf '%s\n' secondmate + printf '%s\n' session-bootstrap + ;; bin/fm-secondmate*|bin/fm-home-seed.sh|bin/fm-backlog-handoff.sh|\ bin/fm-config-inherit-lib.sh|bin/fm-config-push.sh|bin/fm-shared*) printf '%s\n' secondmate diff --git a/docs/configuration.md b/docs/configuration.md index b2f80b8e6d..b226ec6888 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,20 @@ Fleet-local operational facts and gotchas live locally in `data/learnings.md`; i The file is created lazily on first learning and follows the same dated, evidence-backed, curated style as `data/captain.md`: inspect the current file first, then rewrite or prune stale entries instead of appending forever. There is no shared learnings file by captain decision. +## Startup memory budget (config/startup-memory-budget) + +`config/startup-memory-budget` is the primary-authoritative per-home allowance for the startup prompt-memory surface: `data/captain.md`, `data/captain-shared.md`, and `data/learnings.md` together. +The locked mutable bootstrap path materializes its visible default of `7500` estimated tokens in a primary home when the file is absent. +To select another allowance, replace the primary home's file with one valid positive value in the exact format below; the next locked bootstrap convergence or `bin/fm-config-push.sh` propagates it to registered secondmates. +A secondmate does not create an independent default and instead receives the primary value through the inherited-local-material contract in [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md). +The file must be one positive base-10 integer followed by exactly one newline in a regular, single-linked file beneath a non-symlinked `config/` directory. +Malformed, multi-line, symlinked, hardlinked, special, or otherwise unsafe values are rejected rather than treated as a default. +Use `bin/fm-startup-memory-budget.sh read` to validate and print the effective value, or `bin/fm-startup-memory-budget.sh report` to account for the three files. +The stable local estimate is `ceil(UTF-8 bytes / 3)` per file, a conservative portable approximation rather than a provider-exact tokenizer. +An inherited `data/captain-shared.md` counts in a secondmate's total but remains primary-owned and read-only there. +The internal `/stow` skill curates only the editable local files in that case and reports the primary-owned shared file as a concrete exception if it alone exceeds the budget. +The helper's header owns exact parsing, publication, and report output mechanics. + ## Secondmate routes (data/secondmates.md) Persistent secondmate routes live locally in `data/secondmates.md`. @@ -285,7 +299,7 @@ When a running home advances and its loaded instruction surface (`AGENTS.md`, `b If that send fails, bootstrap keeps an idempotent retry marker and emits `NUDGE_SECONDMATES:` with the failure reason. The same bootstrap run emits `SECONDMATE_LIVENESS:` only when a registered secondmate is skipped or its relaunch fails; already-live and successfully relaunched secondmates are handled silently. For a mid-session inherited local-material edit where tracked-file sync is not needed, run `bin/fm-config-push.sh`. -It uses the same live secondmate discovery and propagation helper as bootstrap, prints each live home's `crew-dispatch.json`, `crew-harness`, `backlog-backend`, `backend`, `herdr-presentation-spaces`, and `data/captain-shared.md` result as `pushed`, `unchanged`, `skipped`, or `error`, and exits non-zero for real propagation errors or config-reread send failures. +It uses the same live secondmate discovery and propagation helper as bootstrap, prints each live home's `crew-dispatch.json`, `crew-harness`, `backlog-backend`, `backend`, `herdr-presentation-spaces`, `startup-memory-budget`, and `data/captain-shared.md` result as `pushed`, `unchanged`, `skipped`, or `error`, and exits non-zero for real propagation errors or config-reread send failures. When an allowlisted config item changes for an already-running home, it sends the literal-content reread pointer described in [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md); unchanged allowlisted config sends no pointer unless a previous delivery is pending. The locked bootstrap inheritance pass uses the same per-home changed-set and reread path for already-running homes; see `secondmate-provisioning` for the single contract owner. That live discovery starts from `state/*.meta` records with `kind=secondmate`; `data/secondmates.md` only backfills `home=` for older or incomplete meta records. diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 54b2190f6c..8f70cd346c 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -307,6 +307,10 @@ "path": "docs/verification/runtime-backends.md", "audience": "maintainer-verification" }, + { + "path": "docs/verification/stow-memory.md", + "audience": "maintainer-verification" + }, { "path": "docs/verification/supervision.md", "audience": "maintainer-verification" diff --git a/docs/verification/stow-memory.md b/docs/verification/stow-memory.md new file mode 100644 index 0000000000..39e61eac4f --- /dev/null +++ b/docs/verification/stow-memory.md @@ -0,0 +1,217 @@ +# Startup-memory `/stow` verification + +Audience: maintainer verification. + +This record supports the active bounded-memory and whole-file curation guarantees for Firstmate's internal `/stow` skill. +[`docs/configuration.md`](../configuration.md) owns the current operator-facing setting and estimate. +The internal skill owns curation and completion-receipt behavior. +Task chronology, fixture paths, and delivery evidence remain outside this record. + +## Synthetic real-agent pass + +The development-only real-agent pass ran on 2026-07-30 with Pi 0.82.0 on `openai-codex/gpt-5.6-terra` at medium thinking. +It used disposable primary and secondmate-shaped `FM_HOME` directories under the repository worktree only. +No live Firstmate memory, project data, credential content, or external system was placed in either fixture or prompt. +The following exact Bash shell body created the sanitized fixtures, invoked the model-qualified skill twice per home, and captured reports, hashes, and file modes: + +```bash +set -eu +VERIFY_ROOT=$(mktemp -d "$PWD/.stow-verification.XXXXXX") +RUNTIME_ROOT="$VERIFY_ROOT/runtime-root" +PRIMARY="$VERIFY_ROOT/primary" +SECONDMATE="$VERIFY_ROOT/secondmate" +SECONDMATE_ID=stow-verification +mkdir -p "$RUNTIME_ROOT" "$PRIMARY/config" "$PRIMARY/data" \ + "$SECONDMATE/bin" "$SECONDMATE/config" "$SECONDMATE/data" +printf '%s\n' 350 >"$PRIMARY/config/startup-memory-budget" +printf '%s\n' "$SECONDMATE_ID" >"$SECONDMATE/.fm-secondmate-home" +printf '%s\n' '# Synthetic Firstmate home' >"$SECONDMATE/AGENTS.md" + +file_mode() { + if [ "$(uname)" = Darwin ]; then + stat -f %Lp "$1" + else + stat -c %a "$1" + fi +} + +record_shared_state() { + label=$1 + path=$2 + printf '%s sha256=%s mode=%s\n' "$label" \ + "$(shasum -a 256 "$path" | awk '{print $1}')" \ + "$(file_mode "$path")" +} + +cat >"$PRIMARY/data/captain.md" <<'EOF' +# Captain + +## Current preferences + +- Prefer the simplest direct end-to-end operational path. +- Preserve unique current facts when compacting memory. +- Use plain dashes in prose. + +## Duplicate and superseded material + +- Prefer the simplest direct end-to-end operational path. +- Old policy: build a wrapper before every one-off operation. +- Old policy copy: always build a wrapper for one-off work. +- Stale tool path: `/opt/old-firstmate/bin/fm`. +- Stale release version: 0.41.0. +- Completed task: migrated the demo fixture on Monday. +- Completed task detail: checked the demo fixture again on Tuesday. +- Metric from the completed task: 47 records moved. +EOF + +cat >"$PRIMARY/data/captain-shared.md" <<'EOF' +# Shared captain preferences + +This file is main-authoritative in the main firstmate home. +In secondmate homes it is read-only in secondmate homes and must not be edited there. +Route new captain-preference discoveries to the main firstmate through marked status or a document pointer. + +- Never expose secrets or weaken an accepted safety boundary. +- Prefer the simplest direct end-to-end operational path. +- Superseded policy: secondmates may rewrite shared memory when convenient. +- Duplicate safety note: do not expose secrets. +EOF + +cat >"$PRIMARY/data/learnings.md" <<'EOF' +# Learnings + +- Stable fact: startup-memory configuration is documented in `docs/configuration.md`. +- Authoritative pointer: incident detail belongs in `data/reports/synthetic-incident.md`. +- Stable fact copy: consult `docs/configuration.md` for startup-memory configuration. +- Completed chronology: first the synthetic incident was detected, then triaged, then assigned. +- Completed chronology continued: a patch was drafted, reviewed, merged, and announced. +- Old metric: the discarded prototype used 812 estimated tokens. +- Stale path: the discarded prototype lived at `/tmp/old-memory-prototype`. +- Superseded alternative: maintain both a JSON memory database and Markdown files. +- Report-sized procedure: create a staging directory, enumerate every file, copy each file, compare every line, write a status ledger, notify all operators, archive the ledger, and repeat the entire sequence after every prompt. +EOF + +FM_HOME="$PRIMARY" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/primary.before.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$PRIMARY/data/$file" +done >"$VERIFY_ROOT/primary.before.sha256" + +FM_HOME="$PRIMARY" pi -p --no-session --no-extensions --no-context-files \ + --model openai-codex/gpt-5.6-terra --thinking medium \ + --skill .agents/skills/stow/SKILL.md \ + 'Invoke /stow now against only the disposable synthetic Firstmate home in $FM_HOME. There are no new session facts to file. Follow every requirement in the loaded stow skill. Run the repository-owned bin/fm-startup-memory-budget.sh report command, with the existing FM_HOME environment, before and after curation; that executable is the only permitted path outside $FM_HOME. Retain the exact before total, preserve the complete main-authoritative routing header in data/captain-shared.md, and make the completion receipt state the effective budget, exact before and after totals, an action for each of the three files, every exception, and reset safety. Inspect all three startup-memory files completely, preserve every unique current preference, authority or safety boundary, stable fact, and authoritative pointer, and consolidate the supplied duplicate, superseded, stale, chronological, metric, and report-sized material. Do not access or modify any other home, credential, project data, or external system.' \ + >"$VERIFY_ROOT/primary.pass1.out" +FM_HOME="$PRIMARY" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/primary.after.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$PRIMARY/data/$file" +done >"$VERIFY_ROOT/primary.after.sha256" + +FM_HOME="$PRIMARY" pi -p --no-session --no-extensions --no-context-files \ + --model openai-codex/gpt-5.6-terra --thinking medium \ + --skill .agents/skills/stow/SKILL.md \ + 'Invoke /stow now against only the disposable synthetic Firstmate home in $FM_HOME. There are no new session facts to file. Follow every requirement in the loaded stow skill. Run the repository-owned bin/fm-startup-memory-budget.sh report command, with the existing FM_HOME environment, before and after curation; that executable is the only permitted path outside $FM_HOME. Retain the exact before total, preserve the complete main-authoritative routing header in data/captain-shared.md, and make the completion receipt state the effective budget, exact before and after totals, an action for each of the three files, every exception, and reset safety. Inspect all three startup-memory files completely, preserve every unique current preference, authority or safety boundary, stable fact, and authoritative pointer, and consolidate the supplied duplicate, superseded, stale, chronological, metric, and report-sized material. Do not access or modify any other home, credential, project data, or external system.' \ + >"$VERIFY_ROOT/primary.pass2.out" +FM_HOME="$PRIMARY" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/primary.repeat.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$PRIMARY/data/$file" +done >"$VERIFY_ROOT/primary.repeat.sha256" + +cat >"$SECONDMATE/data/captain.md" <<'EOF' +# Secondmate captain memory + +- Current preference: report concrete blockers instead of guessing. +- Current preference copy: never guess when a concrete blocker can be reported. +- Shared overlap: never expose secrets. +- Superseded preference: silently infer missing configuration. +- Stale version: the fleet uses 0.41.0. +- Completed task: inspected the synthetic queue yesterday. +- Completed task detail: closed the synthetic queue inspection after 19 checks. +EOF + +cat >"$SECONDMATE/data/learnings.md" <<'EOF' +# Secondmate learnings + +- Unique current learning: inherited shared memory counts against the local total. +- Authoritative pointer: startup-memory behavior is documented in `docs/configuration.md`. +- Duplicate learning: include inherited shared memory in the local total. +- Stale path: `/tmp/secondmate-memory-v1`. +- Superseded alternative: copy shared facts into every local file. +- Completed chronology: opened the sample, measured it, discussed it, revised it, remeasured it, and closed it. +- Old metric: the sample once measured 604 estimated tokens. +- Report-sized procedure: take a snapshot, copy it to a ledger, annotate every old measurement, preserve every discarded alternative, append a timestamp, and repeat after each completed task. +EOF + +FM_ROOT="$RUNTIME_ROOT" +FM_HOME="$PRIMARY" +. bin/fm-ff-lib.sh +. bin/fm-config-inherit-lib.sh +validate_secondmate_home "$SECONDMATE_ID" "$SECONDMATE" +printf 'secondmate_validation=accepted id=%s home=%s\n' \ + "$SECONDMATE_ID" "$VALIDATED_HOME" >"$VERIFY_ROOT/inheritance.out" +FM_CONFIG_INHERIT_REPORT="$VERIFY_ROOT/inheritance.report" \ + propagate_secondmate_inheritance \ + "$PRIMARY" "$VALIDATED_HOME" "$PRIMARY/config" "$PRIMARY/data" +cat "$VERIFY_ROOT/inheritance.report" >>"$VERIFY_ROOT/inheritance.out" +cmp -s "$PRIMARY/data/captain-shared.md" \ + "$SECONDMATE/data/captain-shared.md" +record_shared_state inherited "$SECONDMATE/data/captain-shared.md" \ + >>"$VERIFY_ROOT/inheritance.out" + +FM_HOME="$SECONDMATE" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/secondmate.before.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$SECONDMATE/data/$file" +done >"$VERIFY_ROOT/secondmate.before.sha256" +record_shared_state before "$SECONDMATE/data/captain-shared.md" \ + >"$VERIFY_ROOT/secondmate.shared-state" + +FM_HOME="$SECONDMATE" pi -p --no-session --no-extensions --no-context-files \ + --model openai-codex/gpt-5.6-terra --thinking medium \ + --skill .agents/skills/stow/SKILL.md \ + 'Invoke /stow now against only the validated disposable synthetic secondmate home in $FM_HOME. There are no new session facts to file. Follow every requirement in the loaded stow skill. Run the repository-owned bin/fm-startup-memory-budget.sh report command, with the existing FM_HOME environment, before and after curation; that executable is the only permitted path outside $FM_HOME. Retain the exact before total, and make the completion receipt state the effective budget, exact before and after totals, an action for each of the three files, every exception, and reset safety. Inspect all three startup-memory files completely, keep data/captain-shared.md byte-identical and filesystem read-only because it was installed through primary-authoritative inheritance, preserve every unique current preference, stable learning, and authoritative pointer, and consolidate the supplied duplicate, superseded, stale, chronological, metric, overlap, and report-sized material in editable local memory. Do not access or modify any other home, credential, project data, or external system.' \ + >"$VERIFY_ROOT/secondmate.pass1.out" +FM_HOME="$SECONDMATE" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/secondmate.after.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$SECONDMATE/data/$file" +done >"$VERIFY_ROOT/secondmate.after.sha256" +record_shared_state after "$SECONDMATE/data/captain-shared.md" \ + >>"$VERIFY_ROOT/secondmate.shared-state" + +FM_HOME="$SECONDMATE" pi -p --no-session --no-extensions --no-context-files \ + --model openai-codex/gpt-5.6-terra --thinking medium \ + --skill .agents/skills/stow/SKILL.md \ + 'Invoke /stow now against only the validated disposable synthetic secondmate home in $FM_HOME. There are no new session facts to file. Follow every requirement in the loaded stow skill. Run the repository-owned bin/fm-startup-memory-budget.sh report command, with the existing FM_HOME environment, before and after curation; that executable is the only permitted path outside $FM_HOME. Retain the exact before total, and make the completion receipt state the effective budget, exact before and after totals, an action for each of the three files, every exception, and reset safety. Inspect all three startup-memory files completely, keep data/captain-shared.md byte-identical and filesystem read-only because it was installed through primary-authoritative inheritance, preserve every unique current preference, stable learning, and authoritative pointer, and consolidate the supplied duplicate, superseded, stale, chronological, metric, overlap, and report-sized material in editable local memory. Do not access or modify any other home, credential, project data, or external system.' \ + >"$VERIFY_ROOT/secondmate.pass2.out" +FM_HOME="$SECONDMATE" bin/fm-startup-memory-budget.sh report \ + >"$VERIFY_ROOT/secondmate.repeat.report" +for file in captain.md captain-shared.md learnings.md; do + shasum -a 256 "$SECONDMATE/data/$file" +done >"$VERIFY_ROOT/secondmate.repeat.sha256" +record_shared_state repeat "$SECONDMATE/data/captain-shared.md" \ + >>"$VERIFY_ROOT/secondmate.shared-state" +``` + +Bounded observed output: + +```text +secondmate_validation=accepted id=stow-verification +startup-memory-budget pushed +data/captain-shared.md pushed +inherited sha256=d08ce8e35b17c8342773d551b5c1551a5a6ded5f45ab0f7ed5b6ef91ea1d408c mode=444 +primary: 699 -> 219 estimated tokens against a 350-token budget +primary repeat: 219 -> 219; all three files byte-identical +secondmate: 518 -> 192 estimated tokens against a 350-token budget +secondmate repeat: 192 -> 192; all three files byte-identical +before sha256=d08ce8e35b17c8342773d551b5c1551a5a6ded5f45ab0f7ed5b6ef91ea1d408c mode=444 +after sha256=d08ce8e35b17c8342773d551b5c1551a5a6ded5f45ab0f7ed5b6ef91ea1d408c mode=444 +repeat sha256=d08ce8e35b17c8342773d551b5c1551a5a6ded5f45ab0f7ed5b6ef91ea1d408c mode=444 +``` + +The first pass preserved current preferences, shared-memory and safety authority, a stable operating fact, and authoritative configuration and incident-report pointers while removing duplicate, superseded, stale, and chronological material. +The secondmate fixture passed the production home validator before the existing inheritance owner installed the main-authoritative file read-only. +Both secondmate passes preserved its unique local preference and learning while leaving those inherited bytes and mode untouched. +This verifies the real instruction path consolidates to budget, reports truthful deltas, preserves the primary-owned shared boundary, and does not grow on an identical second pass. diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 158e943211..fb44305e99 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -866,7 +866,7 @@ touch "$SECOND_HOME_A/state/.last-watcher-beat" "$SECOND_HOME_B/state/.last-watc # may write config/herdr-presentation-spaces. git -C "$SECOND_HOME_A" init -q git -C "$SECOND_HOME_B" init -q -printf 'config/herdr-presentation-spaces\nconfig/crew-harness\nconfig/crew-dispatch.json\nconfig/backlog-backend\nconfig/backend\n' \ +printf 'config/herdr-presentation-spaces\nconfig/crew-harness\nconfig/crew-dispatch.json\nconfig/backlog-backend\nconfig/backend\nconfig/startup-memory-budget\n' \ > "$SECOND_HOME_A/.gitignore" cp "$SECOND_HOME_A/.gitignore" "$SECOND_HOME_B/.gitignore" git -C "$SECOND_HOME_A" add .gitignore diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 219d74de58..f4e5739fe7 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -655,6 +655,7 @@ make_routine_bootstrap_fixture() { printf '%s\n' '.fm-secondmate-home' printf '%s\n' 'config/crew-harness' printf '%s\n' 'config/crew-dispatch.json' + printf '%s\n' 'config/startup-memory-budget' } > "$root/.gitignore" printf '%s\n' 'instructions' > "$root/AGENTS.md" mkdir -p "$root/bin" "$root/.agents/skills" diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index ae41c79351..446dcd93a1 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -14,7 +14,8 @@ # explicit per-spawn harness arg still wins. # B) Inheritance. The primary pushes a declared, extensible set of LOCAL # (gitignored) config items - config/crew-dispatch.json, config/crew-harness, -# config/backlog-backend, config/backend, and config/herdr-presentation-spaces - +# config/backlog-backend, config/backend, config/herdr-presentation-spaces, and +# config/startup-memory-budget - # down into each secondmate home's config/, so the secondmate's OWN crewmates, # dispatch profiles, backlog backend, runtime-backend default, and Herdr # presentation opt-in inherit the primary's settings. It is primary-authoritative @@ -167,7 +168,7 @@ SH [ "$got" = pi ] || fail "selected plain Pi resolved '$got', expected pi" got=$(PATH="$fakebin:$BASE_PATH" PI_CODING_AGENT=true FM_PI_HARNESS=pi-signed-helper "$ROOT/bin/fm-harness.sh") [ "$got" = pi ] || fail "inexact signed selection marker resolved '$got', expected pi" - got=$(PATH="$fakebin:$BASE_PATH" FM_PI_HARNESS=pi-signed "$ROOT/bin/fm-harness.sh") + got=$(env -u PI_CODING_AGENT PATH="$fakebin:$BASE_PATH" FM_PI_HARNESS=pi-signed "$ROOT/bin/fm-harness.sh") [ "$got" = pi ] || fail "signed selection marker without Pi's family marker resolved '$got', expected pi" got=$(PATH="$fakebin:$BASE_PATH" PI_CODING_AGENT=true FM_TEST_SIGNED_SHAPE=plain "$ROOT/bin/fm-harness.sh") [ "$got" = pi ] || fail "plain Pi marker resolved '$got', expected pi" @@ -883,7 +884,7 @@ new_world() { printf 'projects/\nstate/\ndata/\n.no-mistakes/\n' [ "$dispatch_ignore" = no ] || printf 'config/crew-dispatch.json\n' printf 'config/crew-harness\nconfig/secondmate-harness\nconfig/backlog-backend\n' - printf 'config/backend\nconfig/herdr-presentation-spaces\n' + printf 'config/backend\nconfig/herdr-presentation-spaces\nconfig/startup-memory-budget\n' } > "$w/main/.gitignore" printf 'v1\n' > "$w/main/AGENTS.md" printf 'r1\n' > "$w/main/README.md" @@ -1176,10 +1177,10 @@ test_bootstrap_sweep_defers_dispatch_on_stale_unignored_home() { pass "B9 bootstrap sweep defers new inherited config until the home ignores it" } -# Backward-compat: with no inherited config set, the sweep is a no-op for the -# home's config/ - exactly as before this feature - and ordinary sweep behavior -# (fast-forward) is unaffected. -test_bootstrap_sweep_no_inheritance_is_noop() { +# The primary bootstrap always materializes the startup-memory default, so an +# otherwise empty inherited surface converges that one visible value while +# ordinary tracked-file fast-forward behavior remains unchanged. +test_bootstrap_sweep_materializes_and_inherits_memory_default() { local w c1 w=$(new_world boot-noop) c1=$(git -C "$w/main" rev-parse HEAD) @@ -1193,13 +1194,16 @@ test_bootstrap_sweep_no_inheritance_is_noop() { run_bootstrap "$w" >/dev/null - [ -e "$w/sm/config/crew-dispatch.json" ] && fail "no-inheritance sweep created a home crew-dispatch.json" - [ -e "$w/sm/config/crew-harness" ] && fail "no-inheritance sweep created a home crew-harness" - [ -e "$w/sm/config/backend" ] && fail "no-inheritance sweep created a home backend" - [ -e "$w/sm/config" ] && fail "no-inheritance sweep created a home config/ dir" + [ -e "$w/sm/config/crew-dispatch.json" ] && fail "default-only sweep created a home crew-dispatch.json" + [ -e "$w/sm/config/crew-harness" ] && fail "default-only sweep created a home crew-harness" + [ -e "$w/sm/config/backend" ] && fail "default-only sweep created a home backend" + [ "$(cat "$w/home/config/startup-memory-budget")" = 7500 ] \ + || fail "primary bootstrap did not materialize the startup-memory default" + [ "$(cat "$w/sm/config/startup-memory-budget")" = 7500 ] \ + || fail "default-only sweep did not converge startup-memory-budget" [ "$(git -C "$w/sm" rev-parse HEAD)" = "$head" ] \ - || fail "no-inheritance sweep did not still fast-forward the tracked files" - pass "B10 bootstrap sweep with no inherited config is a config no-op and still fast-forwards" + || fail "default-only sweep did not still fast-forward the tracked files" + pass "B10 bootstrap sweep materializes and inherits the startup-memory default while fast-forwarding" } # config/backend: present and absent primary state converges exactly. @@ -2193,6 +2197,7 @@ cat > "$w/main/bin/fm-spawn.sh" <> '$log' printf '%s' codex > '$w/sm/config/crew-harness' +printf '%s\n' 7500 > '$w/sm/config/startup-memory-budget' SH chmod +x "$w/main/bin/fm-spawn.sh" fakebin=$(make_fake_toolchain "$w") @@ -2317,7 +2322,7 @@ test_spawn_fallback_chain_and_crew_scout_unaffected test_bootstrap_sweep_propagates_and_reconverges test_bootstrap_sweep_propagates_when_tracked_current test_bootstrap_sweep_defers_dispatch_on_stale_unignored_home -test_bootstrap_sweep_no_inheritance_is_noop +test_bootstrap_sweep_materializes_and_inherits_memory_default test_backend_inheritance_present_and_absent test_bootstrap_sweep_surfaces_config_propagation_failure test_bootstrap_rereads_after_partial_propagation diff --git a/tests/fm-startup-memory-budget.test.sh b/tests/fm-startup-memory-budget.test.sh new file mode 100755 index 0000000000..813a273ba6 --- /dev/null +++ b/tests/fm-startup-memory-budget.test.sh @@ -0,0 +1,310 @@ +#!/usr/bin/env bash +# Behavioral coverage for the visible startup-memory budget, its safe parser, +# accounting command, primary-to-secondmate convergence, and exact reread bytes. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} +TMP_ROOT=$(fm_test_tmproot fm-startup-memory-budget) +BUDGET="$ROOT/bin/fm-startup-memory-budget.sh" +BOOTSTRAP="$ROOT/bin/fm-bootstrap.sh" +CONFIG_PUSH="$ROOT/bin/fm-config-push.sh" + +make_fake_toolchain() { + local dir=$1 fakebin + fakebin=$(fm_fakebin "$dir") + fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi quota-axi + cat > "$fakebin/gh" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + cat > "$fakebin/treehouse" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = get ] && [ "${2:-}" = --help ]; then + printf '%s\n' 'Usage: treehouse get [--lease]' +fi +SH + cat > "$fakebin/no-mistakes" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' 'no-mistakes version v1.31.2 (fake)' +fi +SH + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +case "${1:-}:${2:-}" in + --version:*) printf '%s\n' '0.2.3' ;; + update:--help) printf '%s\n' '--archive-body' ;; + mv:--help) printf '%s\n' 'usage: tasks-axi mv [...]' ;; +esac +SH + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +[ -z "${FM_FAKE_TMUX_LOG:-}" ] || printf '%s\n' "$*" >> "$FM_FAKE_TMUX_LOG" +case "$*" in + *display-message*'#{pane_current_command}'*) printf '%s\n' codex ;; + *display-message*'#{pane_id}'*) printf '%s\n' '%1' ;; + *display-message*'#{cursor_y}'*) printf '%s\n' 0 ;; + *capture-pane*) printf '\n' ;; +esac +exit 0 +SH + chmod +x "$fakebin"/* + printf '%s\n' "$fakebin" +} + +new_bootstrap_world() { + local name=$1 world root home + world="$TMP_ROOT/$name" + root="$world/root" + home="$world/home" + mkdir -p "$home/config" "$home/data" "$home/state" "$root/bin" + git init -q -b main "$root" + printf '%s\n' 'config/' > "$root/.gitignore" + printf '%s\n' '# Firstmate test root' > "$root/AGENTS.md" + printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "$root/bin/placeholder.sh" + chmod +x "$root/bin/placeholder.sh" + git -C "$root" add -A + git -C "$root" -c user.name=fmtest -c user.email=fmtest@example.invalid commit -qm initial + printf '%s|%s\n' "$root" "$home" +} + +run_bootstrap() { + local root=$1 home=$2 fakebin=$3 + PATH="$fakebin:$BASE_PATH" FM_BACKEND=tmux FM_HOME="$home" FM_ROOT_OVERRIDE="$root" \ + "$BOOTSTRAP" +} + +test_primary_bootstrap_materializes_visible_default() { + local rec root home fakebin out second + rec=$(new_bootstrap_world materialize) + root=${rec%%|*} + home=${rec#*|} + fakebin=$(make_fake_toolchain "$TMP_ROOT/materialize") + + out=$(run_bootstrap "$root" "$home" "$fakebin") + [ -z "$out" ] || fail "default materialization should stay quiet, got: $out" + [ "$(<"$home/config/startup-memory-budget")" = 7500 ] \ + || fail "bootstrap did not materialize the visible 7500 default" + [ "$(FM_HOME="$home" "$BUDGET" read)" = 7500 ] \ + || fail "read command did not expose the generated default" + + printf '321\n' > "$home/config/startup-memory-budget" + run_bootstrap "$root" "$home" "$fakebin" >/dev/null + [ "$(<"$home/config/startup-memory-budget")" = 321 ] \ + || fail "bootstrap replaced a valid captain-selected budget" + + second="$TMP_ROOT/materialize/secondmate" + mkdir -p "$second/config" "$second/data" "$second/state" + printf '%s\n' sm > "$second/.fm-secondmate-home" + run_bootstrap "$root" "$second" "$fakebin" >/dev/null + [ ! -e "$second/config/startup-memory-budget" ] \ + || fail "secondmate bootstrap created an independent budget instead of awaiting inheritance" + pass "primary bootstrap materializes only the visible default and preserves valid captain choices" +} + +expect_rejected_read() { + local home=$1 expected=$2 out rc + set +e + out=$(FM_HOME="$home" "$BUDGET" read 2>&1) + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "unsafe budget unexpectedly parsed: $expected" + assert_contains "$out" "$expected" "unsafe budget rejection was not specific" +} + +test_safe_parser_rejects_ambiguous_and_unsafe_values() { + local home outside + home="$TMP_ROOT/parser-home" + mkdir -p "$home/config" "$home/data" + printf '42\n' > "$home/config/startup-memory-budget" + [ "$(FM_HOME="$home" "$BUDGET" read)" = 42 ] || fail "valid positive decimal budget was rejected" + + printf '0\n' > "$home/config/startup-memory-budget" + expect_rejected_read "$home" 'value must be one positive decimal integer' + printf '42\nextra\n' > "$home/config/startup-memory-budget" + expect_rejected_read "$home" 'value must be one positive decimal integer' + printf '+42\n' > "$home/config/startup-memory-budget" + expect_rejected_read "$home" 'value must be one positive decimal integer' + + outside="$TMP_ROOT/parser-outside" + printf '77\n' > "$outside" + rm -f "$home/config/startup-memory-budget" + ln -s "$outside" "$home/config/startup-memory-budget" + expect_rejected_read "$home" 'file is symlinked' + [ "$(<"$outside")" = 77 ] || fail "symlink rejection changed its external target" + + rm -f "$home/config/startup-memory-budget" + ln "$outside" "$home/config/startup-memory-budget" + expect_rejected_read "$home" 'file is hardlinked' + [ "$(<"$outside")" = 77 ] || fail "hardlink rejection changed its external source" + + rm -f "$home/config/startup-memory-budget" + rm -rf "$home/config" + ln -s "$TMP_ROOT/parser-config-target" "$home/config" + mkdir -p "$TMP_ROOT/parser-config-target" + printf '88\n' > "$TMP_ROOT/parser-config-target/startup-memory-budget" + expect_rejected_read "$home" 'config directory is symlinked' + pass "budget parser accepts one exact positive value and rejects malformed or unsafe inputs" +} + +test_budget_accounting_reports_all_three_files_and_safe_failure() { + local home out rc outside + home="$TMP_ROOT/accounting-home" + mkdir -p "$home/config" "$home/data" + printf '10\n' > "$home/config/startup-memory-budget" + printf 'abc\n' > "$home/data/captain.md" + printf 'abcdef\n' > "$home/data/captain-shared.md" + + out=$(FM_HOME="$home" "$BUDGET" report) + assert_contains "$out" 'estimator=ceil(UTF-8 bytes / 3) conservative-local-estimate' \ + "report did not name the stable estimator" + assert_contains "$out" 'file=data/captain.md bytes=4 estimated_tokens=2 status=present' \ + "report did not account for captain memory" + assert_contains "$out" 'file=data/captain-shared.md bytes=7 estimated_tokens=3 status=present' \ + "report did not account for shared memory" + assert_contains "$out" 'file=data/learnings.md bytes=0 estimated_tokens=0 status=absent' \ + "report did not account for absent learnings" + assert_contains "$out" 'total_estimated_tokens=5' "report total was not the sum of all three files" + assert_contains "$out" 'budget_status=within-budget' "report did not classify the initial total" + + printf 'abcdefabcdefabcdefabcdef\n' > "$home/data/learnings.md" + out=$(FM_HOME="$home" "$BUDGET" report) + assert_contains "$out" 'budget_status=over-budget' "report did not surface an over-budget total" + + outside="$TMP_ROOT/accounting-outside" + printf 'outside\n' > "$outside" + rm -f "$home/data/captain.md" + ln -s "$outside" "$home/data/captain.md" + set +e + out=$(FM_HOME="$home" "$BUDGET" report 2>&1) + rc=$? + set -e + expect_code 2 "$rc" "unsafe memory input should fail the accounting command" + assert_contains "$out" 'memory file is not an ordinary regular file' \ + "accounting failure did not identify the unsafe memory file" + [ "$(<"$outside")" = outside ] || fail "accounting failure changed a symlink target" + pass "budget accounting sums the three startup files and reports safe failures" +} + +new_propagation_world() { + local world=$1 root="$1/root" home="$1/home" sm="$1/sm" head + mkdir -p "$home/config" "$home/data" "$home/state" "$root/bin" + touch "$home/state/.last-watcher-beat" + git init -q -b main "$root" + printf '%s\n' 'config/' > "$root/.gitignore" + printf '%s\n' '# Firstmate test root' > "$root/AGENTS.md" + printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "$root/bin/placeholder.sh" + chmod +x "$root/bin/placeholder.sh" + git -C "$root" add -A + git -C "$root" -c user.name=fmtest -c user.email=fmtest@example.invalid commit -qm initial + head=$(git -C "$root" rev-parse HEAD) + git -C "$root" worktree add -q --detach "$sm" "$head" + printf '%s\n' sm > "$sm/.fm-secondmate-home" + mkdir -p "$sm/config" "$sm/data" "$sm/state" "$sm/projects" + { + printf 'window=firstmate:fm-sm\n' + printf 'kind=secondmate\n' + printf 'harness=codex\n' + printf 'home=%s\n' "$sm" + } > "$home/state/sm.meta" + printf '%s|%s|%s\n' "$root" "$home" "$sm" +} + +latest_reread_instruction() { + local home=$1 state path latest= + state=$(cd "$home/state" && pwd -P) || return 1 + for path in "$state"/.fm-inherited-config-reread.*; do + case "$path" in *.pending) continue ;; esac + [ -f "$path" ] && [ ! -L "$path" ] || continue + latest=$path + done + [ -n "$latest" ] || return 1 + printf '%s\n' "$latest" +} + +run_config_push() { + local root=$1 home=$2 fakebin=$3 log=$4 + PATH="$fakebin:$BASE_PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$root" FM_SEND_SETTLE=0 \ + FM_FAKE_TMUX_LOG="$log" "$CONFIG_PUSH" +} + +test_primary_budget_converges_with_exact_reread_and_safe_failures() { + local world="$TMP_ROOT/propagation" rec root home sm fakebin log out rc instruction expected outside + mkdir -p "$world" + rec=$(new_propagation_world "$world") + root=${rec%%|*} + rec=${rec#*|} + home=${rec%%|*} + sm=${rec#*|} + fakebin=$(make_fake_toolchain "$world") + log="$world/tmux.log" + + printf '321\n' > "$home/config/startup-memory-budget" + out=$(run_config_push "$root" "$home" "$fakebin" "$log") + assert_contains "$out" 'startup-memory-budget: pushed' \ + "config push did not report the new budget as inherited" + [ "$(<"$sm/config/startup-memory-budget")" = 321 ] \ + || fail "secondmate did not receive the primary budget bytes" + instruction=$(latest_reread_instruction "$sm") || fail "budget propagation did not publish a reread instruction" + expected=$(printf '%s\n\n%s\n%s\n321\n%s' \ + 'These inherited config files changed. Re-read and apply their exact contents at every future intake. They are defaults/rules and do not remove your judgment to choose differently when warranted.' \ + 'config/startup-memory-budget' \ + '-----BEGIN config/startup-memory-budget-----' \ + '-----END config/startup-memory-budget-----') + [ "$(<"$instruction")" = "$expected" ] \ + || fail "budget reread payload was not the exact destination bytes" + assert_contains "$(<"$log")" "CONFIG_REREAD: $instruction" \ + "budget propagation did not send the pointer to its exact reread generation" + + outside="$world/unsafe-budget" + printf '555\n' > "$outside" + rm -f "$sm/config/startup-memory-budget" + ln "$outside" "$sm/config/startup-memory-budget" + set +e + out=$(run_config_push "$root" "$home" "$fakebin" "$log" 2>&1) + rc=$? + set -e + expect_code 1 "$rc" "unsafe inherited destination should stop propagation" + assert_contains "$out" 'startup-memory-budget: error - unsafe or invalid destination: file is hardlinked' \ + "unsafe inherited destination did not produce a concrete propagation error" + [ "$(<"$outside")" = 555 ] || fail "unsafe destination handling changed its hardlinked source" + rm -f "$sm/config/startup-memory-budget" + run_config_push "$root" "$home" "$fakebin" "$log" >/dev/null + [ "$(<"$sm/config/startup-memory-budget")" = 321 ] \ + || fail "safe retry did not restore the converged primary budget" + + rm -f "$home/config/startup-memory-budget" + out=$(run_config_push "$root" "$home" "$fakebin" "$log") + assert_contains "$out" 'startup-memory-budget: pushed - mirrored primary absence' \ + "primary absence was not reported as a converging removal" + [ ! -e "$sm/config/startup-memory-budget" ] \ + || fail "primary absence did not remove the inherited budget" + instruction=$(latest_reread_instruction "$sm") || fail "budget absence did not publish a reread instruction" + assert_contains "$(<"$instruction")" $'-----BEGIN config/startup-memory-budget-----\nABSENT\n-----END config/startup-memory-budget-----' \ + "budget absence reread did not use the explicit ABSENT payload" + + rm -f "$sm/config/startup-memory-budget" + printf '555\n' > "$outside" + ln -s "$outside" "$home/config/startup-memory-budget" + set +e + out=$(run_config_push "$root" "$home" "$fakebin" "$log" 2>&1) + rc=$? + set -e + expect_code 1 "$rc" "unsafe primary budget should stop propagation" + assert_contains "$out" 'startup-memory-budget: error - unsafe or invalid primary source: file is symlinked' \ + "unsafe primary budget did not produce a concrete propagation error" + [ ! -e "$sm/config/startup-memory-budget" ] \ + || fail "unsafe primary budget changed the converged secondmate copy" + [ "$(<"$outside")" = 555 ] || fail "unsafe primary budget handling changed its symlink target" + pass "budget propagation converges through config push with exact rereads, absence, and safe rejection" +} + +test_primary_bootstrap_materializes_visible_default +test_safe_parser_rejects_ambiguous_and_unsafe_values +test_budget_accounting_reports_all_three_files_and_safe_failure +test_primary_budget_converges_with_exact_reread_and_safe_failures + +echo '# all fm-startup-memory-budget tests passed' From f0d7cbe91a4734f189a5d85b27d02d3ef58a7d23 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:32:03 -0700 Subject: [PATCH 03/23] fix(herdr): place workers in the launching workspace (#1328) * fix(herdr): place workers in the launching agent's exact workspace Herdr enforces no workspace-label uniqueness, and spawn resolved its container by taking the FIRST workspace whose label matched the home label. With two workspaces both labeled "firstmate", a worker launched from the second one was created in the first, so it appeared in a different space than the Firstmate the captain was watching. Reproduced end to end on Herdr 0.7.5 protocol 17 by running the real bin/fm-spawn.sh inside a launcher pane in the second "firstmate" workspace: the worker landed in w1 while its launcher was in w2, with an unrelated third workspace focused throughout, which also rules out any dependence on the focused workspace. Placement now binds to the launching process's own Herdr identity. Herdr injects HERDR_PANE_ID, HERDR_SESSION, and HERDR_SOCKET_PATH into every process it manages a pane for, and fm_backend_herdr_launcher_identity resolves that pane's current owning tab and workspace live from Herdr, cross-checking the pane against its tab and confirming the workspace exists exactly once in the session. The injected HERDR_TAB_ID and HERDR_WORKSPACE_ID are creation-time snapshots and are deliberately not read as current identity. Labels are no longer placement authority. A claimed parent identity that is unreadable, contradictory, stale, or from another named session or Herdr server stops the spawn before any worker endpoint exists, rather than degrading to a label search. A launcher with no Herdr ancestry has no workspace to inherit and keeps the per-home labeled container, which must now resolve to exactly one workspace; two same-labeled candidates refuse instead of adopting either. A --secondmate launch keeps standing up that home's own workspace by design. With presentation spaces enabled, the projected child is created and bound under that same exact parent and anchors its ordering on it, so a duplicated home label no longer makes the layout ambiguous. Projection, focus restoration, restart binding, and quarantine rules are unchanged, and children are never collapsed into the parent. tmux, Zellij, cmux, Orca, and the away-mode daemon terminal were each inspected and are not affected: none resolves a container by searching mutable labels. tests/fm-backend-herdr-launcher-workspace-e2e.test.sh drives the real spawn and teardown against an isolated Herdr lab, with its headline case running fm-spawn.sh inside a real Herdr pane so the identity comes from Herdr's own injection. The refusal matrix and the ordering anchor are covered deterministically in tests/fm-backend-herdr.test.sh. Eight existing real-Herdr suites inherited the developer terminal's own Herdr pane into their isolated lab sessions, which the new cross-session check correctly refuses. tests/herdr-test-safety.sh now owns herdr_forget_inherited_pane and those suites call it, so what they assert no longer depends on where they were launched from. Two unrelated fixes found along the way. tests/fm-secondmate-harness.test.sh had the same class of environment leak through CLAUDECODE, which outranks PI_CODING_AGENT in bin/fm-harness.sh and made its pi-signed ancestry case resolve "claude" whenever the suite ran inside Claude Code. And fm-spawn.sh's usage() printed a fixed line range that had already been truncating its own help mid-sentence. * no-mistakes(review): Enforce exact Herdr launcher and projection identity * no-mistakes(document): Document exact Herdr launcher workspace placement --- bin/backends/herdr.sh | 311 +++++++++++-- bin/fm-spawn.sh | 44 +- bin/fm-test-run.sh | 1 + docs/architecture.md | 4 +- docs/configuration.md | 4 +- docs/herdr-backend.md | 35 +- docs/verification/runtime-backends.md | 62 ++- tests/fm-afk-inject-herdr-e2e.test.sh | 5 + tests/fm-backend-autodetect-smoke.test.sh | 8 + .../fm-backend-herdr-eventwait-smoke.test.sh | 5 + ...ckend-herdr-launcher-workspace-e2e.test.sh | 429 ++++++++++++++++++ .../fm-backend-herdr-presentation-e2e.test.sh | 8 + .../fm-backend-herdr-prune-safety-e2e.test.sh | 5 + .../fm-backend-herdr-respawn-idem-e2e.test.sh | 5 + tests/fm-backend-herdr-smoke.test.sh | 5 + ...ckend-herdr-workspace-per-home-e2e.test.sh | 5 + tests/fm-backend-herdr.test.sh | 282 +++++++++++- tests/fm-secondmate-harness.test.sh | 9 + tests/herdr-test-safety.sh | 19 + 19 files changed, 1176 insertions(+), 70 deletions(-) create mode 100755 tests/fm-backend-herdr-launcher-workspace-e2e.test.sh diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 237d2348c5..30f420d07b 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -462,18 +462,15 @@ fm_backend_herdr_presentation_lock_namespace_valid() { # it would turn JSON null into the literal string "null"). Canonicalizes the # parent directory when that directory exists so symlink parents such as /tmp # -> /private/tmp cannot yield two lock identities for the same socket. -fm_backend_herdr_presentation_session_socket_path() { # - local session=$1 sessions socket sock_dir sock_base - [ -n "$session" ] || return 1 - sessions=$(fm_backend_herdr_cli "$session" session list --json 2>/dev/null) || return 1 - socket=$(printf '%s' "$sessions" | jq -er --arg want "$session" ' - [.sessions[]? - | select(.name == $want and .running == true) - | select((.socket_path | type) == "string") - | select((.socket_path | length) > 0) - | .socket_path] - | if length == 1 then .[0] else empty end - ' 2>/dev/null) || return 1 +# fm_backend_herdr_canonical_socket_path: normalize one absolute Unix-socket +# path so two spellings of the same socket compare equal. Refuses a relative +# or empty path. An unresolvable directory is left as-is rather than treated as +# a failure, so a socket whose directory was removed still compares by its own +# literal path. Single owner for every socket-identity comparison in this +# adapter (the presentation session lock and the launcher-identity same-session +# proof both use it). +fm_backend_herdr_canonical_socket_path() { # + local socket=$1 sock_dir sock_base [ -n "$socket" ] || return 1 case "$socket" in /*) ;; @@ -489,6 +486,21 @@ fm_backend_herdr_presentation_session_socket_path() { # printf '%s' "$socket" } +fm_backend_herdr_presentation_session_socket_path() { # + local session=$1 sessions socket + [ -n "$session" ] || return 1 + sessions=$(fm_backend_herdr_cli "$session" session list --json 2>/dev/null) || return 1 + socket=$(printf '%s' "$sessions" | jq -er --arg want "$session" ' + [.sessions[]? + | select(.name == $want and .running == true) + | select((.socket_path | type) == "string") + | select((.socket_path | length) > 0) + | .socket_path] + | if length == 1 then .[0] else empty end + ' 2>/dev/null) || return 1 + fm_backend_herdr_canonical_socket_path "$socket" +} + fm_backend_herdr_presentation_session_lock_path() { # local session=$1 socket key dir hash [ -n "$session" ] || return 1 @@ -630,6 +642,12 @@ fm_backend_herdr_projection_close_pane_focus_preserving() { # is the owning FM_HOME label (firstmate or 2ndmate-). +# Optional is that parent's EXACT id, which the caller +# already resolved from the launching agent's own herdr identity. When given it +# anchors the owning parent by id, so two workspaces sharing the home label no +# longer make the whole layout ambiguous; when omitted the parent is located by +# label exactly as before. With a unique label the two select the same +# workspace, so ordering behavior is unchanged in the ordinary case. # New-format └ ... · p: children and, for compatibility only, already # adjacent old-format firstmate/... or 2ndmate-/... projections may extend # the block read-only; they are never renamed or moved. @@ -644,8 +662,8 @@ fm_backend_herdr_projection_close_pane_focus_preserving() { # - local session=$1 created=$2 parent=$3 list analysis current desired protocol schema socket mover response move_status focus_before +fm_backend_herdr_projection_order_best_effort() { # [] + local session=$1 created=$2 parent=$3 parent_ws=${4:-} list analysis current desired protocol schema socket mover response move_status focus_before local before_existing after_existing [ -n "$parent" ] || { echo "warning: herdr presentation ordering missing owning parent label; leaving worker in Herdr's current order" >&2 @@ -655,9 +673,12 @@ fm_backend_herdr_projection_order_best_effort() { # &2 return 0 } - analysis=$(printf '%s' "$list" | jq -c --arg created "$created" --arg parent "$parent" ' + analysis=$(printf '%s' "$list" | jq -c --arg created "$created" --arg parent "$parent" --arg parent_ws "$parent_ws" ' def is_parent: - (.label | type) == "string" and .label == $parent; + if ($parent_ws | length) > 0 + then .workspace_id == $parent_ws + else (.label | type) == "string" and .label == $parent + end; def is_top_level_parent: (.label | type) == "string" and ((.label == "firstmate") or (.label | test("^2ndmate-[^/]+$"))); @@ -779,14 +800,19 @@ fm_backend_herdr_projection_order_best_effort() { # &2 return 0 fi - if ! printf '%s' "$response" | jq -e --arg created "$created" --arg parent "$parent" --argjson desired "$desired" ' + if ! printf '%s' "$response" | jq -e --arg created "$created" --arg parent "$parent" --arg parent_ws "$parent_ws" --argjson desired "$desired" ' + def is_parent: + if ($parent_ws | length) > 0 + then .workspace_id == $parent_ws + else (.label | type) == "string" and .label == $parent + end; .result.type == "workspace_list" and (.result.workspaces | type) == "array" and .result.workspaces[$desired].workspace_id == $created - and ([.result.workspaces[] | select(.label == $parent)] | length) == 1 + and ([.result.workspaces[] | select(is_parent)] | length) == 1 and ( [range(0; .result.workspaces | length) as $i - | select(.result.workspaces[$i].label == $parent) + | select(.result.workspaces[$i] | is_parent) | $i][0] < $desired ) ' >/dev/null 2>&1; then @@ -821,14 +847,20 @@ fm_backend_herdr_server_ensure() { # return 1 } -# fm_backend_herdr_workspace_find: this HOME's own workspace id inside -# (fm_backend_herdr_workspace_label), or empty (never creates). -# Read-only, safe for recovery/list paths. Label-collision semantics -# (docs/herdr-backend.md "Label collisions"): herdr enforces no label -# uniqueness at all, so this adopts the FIRST matching workspace `jq` returns -# (list order, normally creation order/oldest) rather than disambiguating - -# identical in spirit to the pre-existing tab duplicate-label check below. -fm_backend_herdr_workspace_find() { # +# fm_backend_herdr_workspace_find_all: EVERY workspace id inside +# whose label equals this HOME's own label (fm_backend_herdr_workspace_label), +# one per line, in herdr's own list order (normally creation order, oldest +# first). Empty when none match. Never creates anything. +# +# Single owner of the home-label workspace query. Herdr enforces no workspace +# label uniqueness at all (docs/herdr-backend.md "Label collisions"), so this +# can legitimately return MORE THAN ONE id: a captain-owned workspace can +# collide by label, a cwd-basename-derived label can coincide, and concurrent +# first spawns can mint two same-labeled home workspaces. Callers decide what a +# duplicate means for them - fm_backend_herdr_workspace_ensure refuses to guess +# which one is the caller's, while the read-only recovery path below keeps its +# historical first-match behavior. +fm_backend_herdr_workspace_find_all() { # local session=$1 label list label=$(fm_backend_herdr_workspace_label) list=$(fm_backend_herdr_cli "$session" workspace list 2>/dev/null) || return 0 @@ -838,7 +870,143 @@ fm_backend_herdr_workspace_find() { # # ALWAYS return empty and every spawn mint a fresh "firstmate" workspace # (the workspace leak). printf '%s' "$list" | jq -r --arg want "$label" \ - '.result.workspaces[]? | select(.label == $want) | .workspace_id' 2>/dev/null | head -1 + '.result.workspaces[]? | select(.label == $want) | .workspace_id' 2>/dev/null +} + +# fm_backend_herdr_workspace_find: this HOME's own workspace id inside +# , or empty (never creates). Read-only, safe for recovery/list +# paths, which address panes they already recorded and only need a container +# to scan. Keeps the historical FIRST-match behavior on a label collision - +# identical in spirit to the pre-existing tab duplicate-label check below. +# NOT the spawn-time resolver: placing a new worker by first label match is +# exactly the defect fm_backend_herdr_workspace_ensure now refuses. +fm_backend_herdr_workspace_find() { # + fm_backend_herdr_workspace_find_all "$1" | head -1 +} + +# fm_backend_herdr_launcher_identity: the EXACT herdr workspace that the +# process making this spawn is itself running in. +# +# Herdr 0.7.5 injects HERDR_ENV=1, HERDR_PANE_ID, HERDR_SESSION, +# HERDR_SOCKET_PATH, HERDR_TAB_ID, and HERDR_WORKSPACE_ID into every process it +# manages a pane for (docs/verification/runtime-backends.md), and a firstmate +# or secondmate agent's own tool calls inherit them. Older injection shapes are +# unverified and cannot establish launcher ancestry without both pane and +# socket identity. Workspace LABELS are mutable and herdr enforces no +# uniqueness on them, so a label search cannot tell one `firstmate` workspace +# from another, and herdr's globally focused workspace is whatever the captain +# happens to be looking at, not the launcher's. +# +# The injected HERDR_TAB_ID/HERDR_WORKSPACE_ID are deliberately NOT read as the +# answer. They are a snapshot taken when the pane's process started, and herdr +# can move a pane between tabs and workspaces afterwards without being able to +# rewrite a running process's environment. Only a live read is the CURRENT +# parent, which is what placement has to bind to. +# +# Sets, only on a 0 return: +# FM_BACKEND_HERDR_LAUNCHER_PANE_ID +# FM_BACKEND_HERDR_LAUNCHER_TAB_ID +# FM_BACKEND_HERDR_LAUNCHER_WORKSPACE_ID +# +# Returns: +# 0 - one exact, self-consistent launcher pane/tab/workspace in . +# 2 - this process is NOT running in a herdr pane (no HERDR_PANE_ID at all), +# so there is no launcher workspace to inherit and the caller falls back +# to its per-home container. HERDR_ENV=1 on its own is only a backend +# SELECTION marker (bin/fm-backend.sh's fm_backend_detect), never a +# parent binding - herdr always injects the pane id alongside it. +# 1 - a launcher pane IS claimed but its binding is missing, stale, +# contradictory, or belongs to another herdr session. The caller must +# refuse before creating or publishing any worker endpoint rather than +# degrading to a label search. +fm_backend_herdr_launcher_identity() { # + local session=$1 pane=${HERDR_PANE_ID:-} claimed_session claimed_socket session_socket + local pane_out tab_out list tab workspace + FM_BACKEND_HERDR_LAUNCHER_PANE_ID="" + FM_BACKEND_HERDR_LAUNCHER_TAB_ID="" + FM_BACKEND_HERDR_LAUNCHER_WORKSPACE_ID="" + [ -n "$pane" ] || return 2 + + # Same-session proof, before the pane id is trusted at all: herdr pane ids + # ("w2:p1") restart at the same low numbers in every session, so a pane id + # borrowed from another session can silently resolve to a real but unrelated + # workspace here. The injected socket path is the server identity herdr + # exposes, and the session name independently binds the named session. + claimed_session=$(fm_backend_herdr_session) + if [ "$claimed_session" != "$session" ]; then + echo "error: herdr launcher pane '$pane' reports session '$claimed_session' but this spawn targets session '$session'; refusing to place a worker from a cross-session parent identity" >&2 + return 1 + fi + claimed_socket=${HERDR_SOCKET_PATH:-} + if [ -z "$claimed_socket" ]; then + echo "error: herdr launcher pane '$pane' has no injected socket identity; refusing to place a worker from an unverifiable parent identity" >&2 + return 1 + fi + claimed_socket=$(fm_backend_herdr_canonical_socket_path "$claimed_socket") || { + echo "error: herdr launcher pane '$pane' reports an unusable socket path; refusing to place a worker from an unverifiable parent identity" >&2 + return 1 + } + session_socket=$(fm_backend_herdr_presentation_session_socket_path "$session") || { + echo "error: herdr session '$session' has no unambiguous socket to match against the launcher pane's own; refusing to place a worker from an unverifiable parent identity" >&2 + return 1 + } + if [ "$claimed_socket" != "$session_socket" ]; then + echo "error: herdr launcher pane '$pane' belongs to the server at '$claimed_socket', not session '$session' at '$session_socket'; refusing to place a worker from a cross-session parent identity" >&2 + return 1 + fi + + pane_out=$(fm_backend_herdr_cli "$session" pane get "$pane" 2>/dev/null) || { + echo "error: herdr launcher pane '$pane' could not be read in session '$session'; refusing to place a worker without its exact parent workspace" >&2 + return 1 + } + tab=$(printf '%s' "$pane_out" | jq -r --arg pane "$pane" ' + select(.result.pane.pane_id == $pane) + | select((.result.pane.tab_id | type) == "string" and (.result.pane.tab_id | length) > 0) + | .result.pane.tab_id + ' 2>/dev/null) + workspace=$(printf '%s' "$pane_out" | jq -r --arg pane "$pane" ' + select(.result.pane.pane_id == $pane) + | select((.result.pane.workspace_id | type) == "string" and (.result.pane.workspace_id | length) > 0) + | .result.pane.workspace_id + ' 2>/dev/null) + if [ -z "$tab" ] || [ -z "$workspace" ]; then + echo "error: herdr launcher pane '$pane' returned an ambiguous tab or workspace identity in session '$session'; refusing to place a worker without its exact parent workspace" >&2 + return 1 + fi + + # Independent second read: the tab must agree that it lives in the same + # workspace the pane just claimed. A restored-but-stale pane record that + # disagrees with its own tab is exactly the contradictory binding this must + # refuse rather than resolve. + tab_out=$(fm_backend_herdr_cli "$session" tab get "$tab" 2>/dev/null) || { + echo "error: herdr launcher tab '$tab' could not be read in session '$session'; refusing to place a worker without its exact parent workspace" >&2 + return 1 + } + if ! printf '%s' "$tab_out" | jq -e --arg tab "$tab" --arg workspace "$workspace" ' + .result.tab.tab_id == $tab and .result.tab.workspace_id == $workspace + ' >/dev/null 2>&1; then + echo "error: herdr launcher pane '$pane' and tab '$tab' disagree about their workspace in session '$session'; refusing to place a worker from a contradictory parent identity" >&2 + return 1 + fi + + list=$(fm_backend_herdr_cli "$session" workspace list 2>/dev/null) || { + echo "error: could not list herdr workspaces in session '$session' to confirm the launcher's own workspace '$workspace'; refusing to place a worker without its exact parent workspace" >&2 + return 1 + } + if ! printf '%s' "$list" | jq -e --arg workspace "$workspace" ' + (.result.workspaces | type) == "array" + and ([.result.workspaces[] | select(.workspace_id == $workspace)] | length) == 1 + ' >/dev/null 2>&1; then + echo "error: herdr launcher workspace '$workspace' is missing or duplicated in session '$session'; refusing to place a worker from a stale parent identity" >&2 + return 1 + fi + + # shellcheck disable=SC2034 # callers consume the verified binding's parts + FM_BACKEND_HERDR_LAUNCHER_PANE_ID=$pane + # shellcheck disable=SC2034 # callers consume the verified binding's parts + FM_BACKEND_HERDR_LAUNCHER_TAB_ID=$tab + FM_BACKEND_HERDR_LAUNCHER_WORKSPACE_ID=$workspace + return 0 } # fm_backend_herdr_workspace_prune_seeded_default_tab: close EXACTLY @@ -903,11 +1071,13 @@ fm_backend_herdr_workspace_prune_seeded_default_tab() { # , creating it in if absent. Must be called as a PLAIN -# STATEMENT, never through command substitution ($(...)) - it communicates -# through these globals, not solely through stdout, and a command -# substitution forks a subshell that would discard them: +# fm_backend_herdr_workspace_ensure: the workspace this spawn's task tab +# belongs in inside - the launching agent's own exact workspace when +# it has one, otherwise this HOME's persistent workspace, created in if +# absent. Must be called as a PLAIN STATEMENT, never through command +# substitution ($(...)) - it communicates through these globals, not solely +# through stdout, and a command substitution forks a subshell that would +# discard them: # FM_BACKEND_HERDR_WS_ID - the resolved workspace_id (also echoed, # for callers that only need the id) # FM_BACKEND_HERDR_WS_SEEDED_TAB_ID - non-empty ONLY when THIS call just @@ -919,11 +1089,14 @@ fm_backend_herdr_workspace_prune_seeded_default_tab() { # - local session=$1 cwd=$2 wsid out label +# +# (3rd arg, default "launcher-home") says whether the +# container being ensured belongs to the SAME firstmate home as the process +# calling this: +# launcher-home - a crewmate or scout for the caller's own home. When the +# caller is itself running in a herdr pane, the worker MUST +# land in that exact workspace +# (fm_backend_herdr_launcher_identity), never in whichever +# same-labeled workspace happens to sort first. +# other-home - a --secondmate launch, which stands up a DIFFERENT home's +# own per-home workspace by design. The launcher's workspace +# is deliberately not inherited here. +# With no herdr ancestry at all there is no launcher workspace to inherit, so +# the per-home label lookup below stays the resolver - but it must then resolve +# to exactly ONE workspace. Two same-labeled home workspaces with no launcher +# identity to disambiguate them is an unresolvable placement, and adopting +# either one is the very defect this refuses. +# +# Returns 0 on success, 3 for a refusal whose exact reason is already on +# stderr, and 1 for a failed or unparseable herdr call. +fm_backend_herdr_workspace_ensure() { # [] + local session=$1 cwd=$2 relationship=${3:-launcher-home} wsid out label matches count status FM_BACKEND_HERDR_WS_ID="" FM_BACKEND_HERDR_WS_SEEDED_TAB_ID="" - wsid=$(fm_backend_herdr_workspace_find "$session") + if [ "$relationship" = launcher-home ]; then + fm_backend_herdr_launcher_identity "$session" && status=0 || status=$? + case "$status" in + 0) + FM_BACKEND_HERDR_WS_ID=$FM_BACKEND_HERDR_LAUNCHER_WORKSPACE_ID + printf '%s' "$FM_BACKEND_HERDR_WS_ID" + return 0 + ;; + 2) ;; + *) return 3 ;; + esac + fi + label=$(fm_backend_herdr_workspace_label) + matches=$(fm_backend_herdr_workspace_find_all "$session") + count=$(printf '%s' "$matches" | grep -c '[^[:space:]]' || true) + if [ "$count" -gt 1 ]; then + echo "error: ${count} herdr workspaces in session '$session' are labeled '$label' (${matches//$'\n'/ }) and this spawn has no herdr parent pane to identify which one is its own; rename or close the extras, or run firstmate inside the workspace its workers belong in" >&2 + return 3 + fi + wsid=${matches%%$'\n'*} if [ -n "$wsid" ]; then FM_BACKEND_HERDR_WS_ID=$wsid printf '%s' "$wsid" return 0 fi - label=$(fm_backend_herdr_workspace_label) out=$(fm_backend_herdr_cli "$session" workspace create --cwd "$cwd" --label "$label" --no-focus 2>/dev/null) || return 1 wsid=$(printf '%s' "$out" | jq -r '.result.workspace.workspace_id // empty' 2>/dev/null) [ -n "$wsid" ] || return 1 @@ -974,13 +1185,18 @@ fm_backend_herdr_workspace_ensure() { # # CONTAINER=${RAW%%$'\t'*}; SEEDED_TAB_ID=${RAW#*$'\t'}. The seeded tab id # must be threaded through to fm_backend_herdr_create_task, which is the only # function allowed to prune it (fm_backend_herdr_workspace_prune_seeded_default_tab). -fm_backend_herdr_container_ensure() { # - local cwd=${1:-$PWD} session label +# is passed straight through to +# fm_backend_herdr_workspace_ensure, which owns its meaning. +fm_backend_herdr_container_ensure() { # [] + local cwd=${1:-$PWD} relationship=${2:-launcher-home} session label status fm_backend_herdr_version_check || return 1 session=$(fm_backend_herdr_session) fm_backend_herdr_server_ensure "$session" || return 1 - fm_backend_herdr_workspace_ensure "$session" "$cwd" >/dev/null || { label=$(fm_backend_herdr_workspace_label); echo "error: failed to ensure herdr workspace '$label' in session '$session'" >&2; return 1; } - if [ -z "$FM_BACKEND_HERDR_WS_ID" ]; then + fm_backend_herdr_workspace_ensure "$session" "$cwd" "$relationship" >/dev/null && status=0 || status=$? + # A 3 already reported the exact placement it refused to guess at; adding the + # generic message here would bury it. + [ "$status" -ne 3 ] || return 1 + if [ "$status" -ne 0 ] || [ -z "$FM_BACKEND_HERDR_WS_ID" ]; then label=$(fm_backend_herdr_workspace_label) echo "error: failed to ensure herdr workspace '$label' in session '$session'" >&2 return 1 @@ -1347,7 +1563,7 @@ fm_backend_herdr_projection_parent_workspace_exact() { # local session=$1 token=$2 workspace=$3 tab=$4 pane=$5 parent_workspace=$6 @@ -1373,7 +1589,6 @@ fm_backend_herdr_projection_live_binding_matches() { # .herdr-presentation atomically, then creates a disposable @@ -115,7 +124,10 @@ set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" usage() { - sed -n '2,78p' "$0" | sed 's/^# \{0,1\}//' + # The whole leading comment block, ending at the first line that is not a + # comment. Derived rather than a fixed line range, which silently truncated + # this help mid-sentence every time the header above grew. + sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//' } case "${1:-}" in @@ -985,9 +997,18 @@ case "$BACKEND" in # to PROJ_ABS for just these two calls (bash restores it automatically # after each prefixed simple-command call) so the secondmate's tab lands # in the secondmate's own workspace, not the primary's "firstmate" one. + # + # Placement, separately from labeling: a crewmate/scout belongs in the + # EXACT herdr workspace this launching process is itself running in, which + # only its own herdr pane identity can name (a same-labeled sibling + # workspace must never be adopted). A --secondmate launch is the exception - + # it stands up a DIFFERENT home's own workspace by design - so it asks for + # the per-home container instead of inheriting this launcher's. HERDR_LABEL_HOME=$FM_HOME + HERDR_LAUNCHER_RELATIONSHIP=launcher-home if [ "$KIND" = secondmate ]; then HERDR_LABEL_HOME=$PROJ_ABS + HERDR_LAUNCHER_RELATIONSHIP=other-home fi HERDR_PRESENTATION_JOURNAL=$(fm_backend_herdr_projection_journal_path "$STATE" "$ID") HERDR_PROJECTED=0 @@ -1042,8 +1063,21 @@ case "$BACKEND" in if ! fm_backend_herdr_server_ensure "$HERDR_SES"; then echo "warning: herdr presentation could not ensure its session server; using the ordinary flat layout without projection" >&2 elif spawn_herdr_presentation_order_lock_acquire "$HERDR_SES"; then - HERDR_PARENT_WORKSPACE_ID=$(fm_backend_herdr_projection_parent_workspace_exact \ - "$HERDR_SES" "$HERDR_PARENT_LABEL" 2>/dev/null || true) + # The projected child is placed and bound UNDER this launcher's exact + # parent workspace. Its own herdr pane identity names that workspace + # directly; the label lookup is only the fallback for a launcher with + # no herdr ancestry at all. A claimed-but-broken identity refuses here + # rather than projecting under a guessed parent. + set +e + fm_backend_herdr_launcher_identity "$HERDR_SES" + HERDR_LAUNCHER_STATUS=$? + set -e + case "$HERDR_LAUNCHER_STATUS" in + 0) HERDR_PARENT_WORKSPACE_ID=$FM_BACKEND_HERDR_LAUNCHER_WORKSPACE_ID ;; + 2) HERDR_PARENT_WORKSPACE_ID=$(fm_backend_herdr_projection_parent_workspace_exact \ + "$HERDR_SES" "$HERDR_PARENT_LABEL" 2>/dev/null || true) ;; + *) spawn_herdr_presentation_order_lock_release; exit 1 ;; + esac if [ -z "$HERDR_PARENT_WORKSPACE_ID" ]; then echo "warning: herdr presentation parent is absent or ambiguous; using the ordinary flat layout without projection" >&2 spawn_herdr_presentation_order_lock_release @@ -1071,7 +1105,7 @@ case "$BACKEND" in HERDR_PROJECTION_ABORT_TASK_PANE=$HERDR_PANE_ID HERDR_PROJECTION_ABORT_SEEDED_PANE=$FM_BACKEND_HERDR_PROJECTION_SEEDED_PANE_ID fm_backend_herdr_projection_order_best_effort \ - "$HERDR_SES" "$HERDR_WORKSPACE_ID" "$HERDR_PARENT_LABEL" + "$HERDR_SES" "$HERDR_WORKSPACE_ID" "$HERDR_PARENT_LABEL" "$HERDR_PARENT_WORKSPACE_ID" HERDR_HOME_ID=$(fm_backend_herdr_projection_home_identity "$HERDR_LABEL_HOME" 2>/dev/null || true) if [ -n "$HERDR_HOME_ID" ] \ && fm_backend_herdr_projection_live_binding_matches \ @@ -1093,7 +1127,7 @@ case "$BACKEND" in fi fi if [ "$HERDR_PROJECTED" -ne 1 ]; then - HERDR_CONTAINER_RAW=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_container_ensure "$PROJ_ABS") || exit 1 + HERDR_CONTAINER_RAW=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_container_ensure "$PROJ_ABS" "$HERDR_LAUNCHER_RELATIONSHIP") || exit 1 # fm_backend_herdr_container_ensure echoes ":\t" # (the second field empty when this call ADOPTED a pre-existing workspace # rather than creating a fresh one). Split on the guaranteed single tab diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index f89a6bade5..0ecfcd647c 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -138,6 +138,7 @@ family_for_basename() { ;; fm-afk-inject-herdr-e2e.test.sh|fm-afk-launch.test.sh|fm-backend-autodetect-smoke.test.sh|\ fm-backend-herdr-eventwait-smoke.test.sh|fm-backend-herdr-presentation-e2e.test.sh|\ + fm-backend-herdr-launcher-workspace-e2e.test.sh|\ fm-backend-herdr-prune-safety-e2e.test.sh|fm-backend-herdr-respawn-idem-e2e.test.sh|\ fm-herdr-session-cleanup-e2e.test.sh|\ fm-backend-herdr-smoke.test.sh|fm-backend-herdr-workspace-per-home-e2e.test.sh) diff --git a/docs/architecture.md b/docs/architecture.md index bf8b5cb3ec..f08c060ceb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -102,7 +102,7 @@ That poll loop is the default event source for backends with no native push even For capable Herdr sessions, the same watcher replaces its terminal sleep with a bounded native event wait that immediately surfaces `blocked`; [Push events and polling fallback](herdr-backend.md#push-events-and-polling-fallback) owns the current mechanism and capability gates, while [runtime backend verification](verification/runtime-backends.md#native-blocked-event) owns the active evidence. The deeper session-start agent-process liveness probe is separate from that busy-state poll: tmux and Herdr have verified classifiers for secondmate recovery, Zellij remains unverified, and Orca and cmux do not support secondmate spawns. Herdr is experimental and can be selected explicitly or by runtime auto-detection: Treehouse remains its worktree provider, [`herdr-backend.md`](herdr-backend.md) owns current setup and safety limits, and [`verification/runtime-backends.md`](verification/runtime-backends.md#herdr) owns active empirical evidence. -Herdr's durable default container shape is workspace-per-home plus tab-per-task: the primary home uses workspace label `firstmate`, secondmate homes use `2ndmate-`, and recovery/list-live scopes to the current `FM_HOME`'s workspace. +Herdr uses one tab per task; [Watching and task containers](herdr-backend.md#watching-and-task-containers) owns launcher-bound workspace placement, the label-only fallback, and recovery scope. Its optional default-off presentation projection may place one clean new task in a disposable workspace without changing endpoint authority or lifecycle ownership; [Optional presentation spaces](herdr-backend.md#optional-presentation-spaces) owns that conditional design and its narrow home-local restored-shell cleanup at locked session start. Zellij is experimental and selected only explicitly: Treehouse remains its worktree provider, [`zellij-backend.md`](zellij-backend.md) owns current setup and limits, and [`verification/runtime-backends.md`](verification/runtime-backends.md#zellij) owns active empirical evidence. Zellij's container shape is simpler than herdr's: one shared `firstmate` session, one tab per task, with no per-home workspace split; visible tab titles are scoped by the active home label plus a short hash of the resolved `FM_ROOT` path. @@ -157,7 +157,7 @@ That keeps spawn launch compatible across claude, codex, grok, pi, opencode, and `fm-home-seed.sh` provisions the isolated home, clones the listed PR-based projects into it, initializes newly cloned `no-mistakes` projects, copies the charter to `data/charter.md`, and `fm-spawn.sh --secondmate` launches it through the same session-provider and status-file path as any direct report. For a domain whose subject is the firstmate repo itself, a deliberate `--no-projects` seed creates a project-less home whose crews take pooled worktrees of that repo instead of separate clones. The signal cannot be mixed with project names or omitted accidentally, and a populated home cannot be converted in place; the full seed contract is in [configuration.md](configuration.md#secondmate-routes-datasecondmatesmd). -On the herdr backend, a secondmate launch lands in that secondmate home's labeled workspace, and crewmates spawned from that home land in the same workspace. +Herdr secondmate and child placement follows the launcher-binding contract in [Watching and task containers](herdr-backend.md#watching-and-task-containers). When seeded with `-`, the home is a durable treehouse lease under the secondmate id, so it survives with no live process and is not recycled by later `treehouse get` or pruning. Retirement or seed rollback returns the leased home; normal restart/recovery keeps it leased. If returning the lease fails during teardown, firstmate leaves the route and home intact instead of hiding a still-held lease. diff --git a/docs/configuration.md b/docs/configuration.md index b226ec6888..07b3bf39b4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,8 +81,8 @@ These five sentences are the single owner of the task-selector vocabulary; backe `fm-teardown.sh ` takes a task id directly and validates the complete metadata-only endpoint identity before any runtime dispatch or cleanup mutation. Missing, empty, duplicate, malformed, backend-inconsistent, or task-mismatched endpoint records are preserved and refused. Legacy tmux metadata remains cleanup-compatible when its exact window name is `fm-`; opaque non-tmux endpoints require their recorded `endpoint_task_id=` binding. -By default, Herdr workspaces are derived from `FM_HOME`: the primary home uses `firstmate`, and a secondmate home marked by `.fm-secondmate-home` uses `2ndmate-`. -The default-container spawn, list-live, and recovery paths read that label from the active home, so a secondmate's own crewmates stay inside that secondmate home's herdr space. +`FM_HOME` determines Herdr's home label: the primary home uses `firstmate`, and a secondmate home marked by `.fm-secondmate-home` uses `2ndmate-`. +[`herdr-backend.md`](herdr-backend.md#watching-and-task-containers) owns launcher-bound workspace placement, the label-only fallback, collision handling, and recovery behavior. The optional local `config/herdr-presentation-spaces` presence flag instead enables Herdr's default-off disposable single-task visual projection; [Optional presentation spaces](herdr-backend.md#optional-presentation-spaces) owns its behavior, safety limits, recovery contract, and narrow locked session-start cleanup of exact restored idle-shell children. The flag is default-off and inherited into secondmate homes under the primary-authoritative contract owned by [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md). For normal herdr operations, `HERDR_SESSION` selects the named session, but destructive test cleanup must not rely on `HERDR_SESSION` alone. diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 91047bcc6f..fd5d34dad2 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -33,10 +33,11 @@ Real harness credential tests remain opt-in rather than part of default CI. ## Watching and task containers -Each Firstmate home gets one durable workspace with one task tab per endpoint. -The primary workspace is `firstmate`. -A secondmate home uses `2ndmate-`, derived from its validated `.fm-secondmate-home` marker. -The secondmate process and every child it launches resolve the same home label; a secondmate launched by the primary receives a narrowly scoped home override during container creation. +The ordinary topology puts one task tab per endpoint in the exact workspace of the Firstmate or secondmate that launches it. +When the launcher has no Herdr workspace to inherit, the adapter maintains one durable home-labeled workspace instead. +The primary home label is `firstmate`. +A secondmate home label is `2ndmate-`, derived from its validated `.fm-secondmate-home` marker. +A secondmate launched by the primary receives a narrowly scoped home override during container creation. Attach to the selected named Herdr session and switch to the relevant home workspace to watch its task tabs. Routine supervision uses `bin/fm-peek.sh ` and `FM_HOME= bin/fm-send.sh ''` without attaching. @@ -44,10 +45,21 @@ Routine supervision uses `bin/fm-peek.sh ` and `FM_HOME= bin/fm-send.s Workspace and tab creation use `--no-focus`. The first workspace in a completely empty Herdr session must become focused because no prior target exists, but later task creation does not intentionally steal focus. -Herdr does not enforce workspace or tab label uniqueness. -Firstmate adopts the first workspace matching its derived home label and refuses duplicate task tabs inside it. -Avoid naming a personal workspace `firstmate` or `2ndmate-` because the adapter cannot distinguish that label collision from its own container. +Herdr does not enforce workspace or tab label uniqueness, so a label can never decide where a worker goes. +Herdr 0.7.5 exports `HERDR_ENV`, `HERDR_PANE_ID`, `HERDR_SESSION`, `HERDR_SOCKET_PATH`, `HERDR_TAB_ID`, and `HERDR_WORKSPACE_ID` into every process it manages a pane for, and a Firstmate or secondmate agent's own commands inherit them. +Older injection shapes are unverified, so a claimed launcher pane without the injected socket identity cannot be trusted. +With presentation spaces disabled, a crewmate or scout is created in the exact workspace that identity currently resolves to, read live from Herdr rather than from the injected snapshot, so the worker always appears beside the agent that launched it. +Duplicate labels elsewhere in the session are irrelevant, and the globally focused workspace is never the target. +A `--secondmate` launch is the deliberate exception: it stands up that secondmate home's own workspace instead of joining the launcher's. + +A claimed parent identity that cannot be resolved exactly stops the spawn before any worker endpoint exists, rather than falling back to a label search. +That covers a missing or unusable socket identity, a closed or unreadable launcher pane, a pane and tab that disagree about their workspace, a workspace missing from the session, and a pane belonging to another named session or Herdr server. + +Firstmate running outside Herdr entirely has no launcher workspace to inherit, so its workers use this home's own labeled workspace, created on first use. +That path needs the home label to identify exactly one workspace: two workspaces sharing it are an unresolvable placement and refuse rather than adopting either. +Avoid naming a personal workspace `firstmate` or `2ndmate-` for that reason, and because the adapter cannot distinguish that label collision from its own container. An older secondmate workspace using `firstmate-` is not migrated automatically; rename it manually before expecting new tasks or recovery to use it. +Recovery and list-live still scan the first workspace matching the home label, because they address panes they already recorded rather than choosing where new work goes. Existing task operations use recorded endpoint ids and do not move a live task when labels change. The per-home workspace is reused while it has task tabs. @@ -63,9 +75,12 @@ An absent or unconverged setting keeps the flat default. Presentation is a best-effort visual projection, never task ownership or lifecycle authority. Only a fresh task with neither metadata nor an existing presentation journal is eligible for projected creation. Firstmate atomically publishes a three-field version 1 journal containing a random 128-bit base64url token before asking Herdr to create anything. -After the new workspace converges to one exact task endpoint beneath one exact parent, the journal advances to a version 2 binding that records the physical home, named session, endpoint, parent, and immutable expected labels. +After the new workspace converges to one exact task endpoint beneath one exact parent workspace id, the journal advances to a version 2 binding that records the physical home, named session, endpoint, parent, and immutable expected labels. +Another parent with the same presentation label does not prevent publication or participate in restart reclaim. The token is visible in the workspace title because Herdr exposes no verified hidden persistent field, but neither token, title, nor journal authorizes send, capture, task ownership, Treehouse return, or general recovery. +The owning parent is the launcher's own exact workspace, resolved from the same identity the flat path uses, and falls back to a unique home-label lookup only for a Firstmate outside Herdr. +Projected children are never collapsed back into that parent; it is the placement and ordering reference the projection is bound under. The normal `fm-` task tab is created in the exact new workspace returned by Herdr. Only the exact seeded default tab returned by the same workspace-create response can be pruned. Before and after create, prune, order, abort cleanup, and normal cleanup, Firstmate verifies exact workspace, tab, pane, and active-focus ids. @@ -259,7 +274,8 @@ Tests use thin compatibility wrappers in `tests/herdr-test-safety.sh` and never - Herdr remains experimental. - Presentation ordering needs protocol 16 and Python and is best-effort only. -- Mutable labels can collide; they are never destructive authority. +- Mutable labels can collide; they are never placement or destructive authority. +- A Firstmate outside Herdr cannot resolve a launcher workspace, so a colliding home label refuses new spawns until the collision is cleared. - Ghost and placeholder recognition depends on ANSI de-emphasis and fails safely to pending when unavailable. - Mid-session secondmate liveness is not implemented. - OpenCode 1.18.4 can accept Enter while busy without clearing the composer. @@ -274,6 +290,7 @@ tests/fm-backend-herdr-smoke.test.sh tests/fm-backend-herdr-prune-safety-e2e.test.sh tests/fm-backend-herdr-respawn-idem-e2e.test.sh tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +tests/fm-backend-herdr-launcher-workspace-e2e.test.sh tests/fm-backend-herdr-presentation-e2e.test.sh tests/fm-backend-herdr-eventwait-smoke.test.sh tests/fm-herdr-session-cleanup.test.sh diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 65152100f4..71c559fa89 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -120,7 +120,8 @@ Claude, Codex, OpenCode, Pi, pi-signed, Grok, and Kimi share that backend cleanu ## Herdr The compatibility floor is protocol 14. -The latest active verification uses Herdr 0.7.5 protocol 16 on macOS aarch64, with earlier 0.7.4, protocol-14, and 0.7.3 evidence retained where they define current behavior or fallbacks. +The latest active verification uses Herdr 0.7.5 protocol 17 on macOS aarch64, with earlier 0.7.4, protocol-16, protocol-14, and 0.7.3 evidence retained where they define current behavior or fallbacks. +Protocol 17 keeps every protocol-16 feature gate satisfied; the event and workspace-move floors remain 16. Core read-only probes: @@ -134,7 +135,7 @@ Observed current shapes: ```text herdr 0.7.5 -{"client":16,"server":16} +{"client":17,"server":17} ["pane.output_matched","pane.agent_status_changed","pane.scroll_changed"] ``` @@ -173,6 +174,63 @@ HERDR_LAB_HELPER=bin/fm-herdr-lab.sh \ Observed guarantee: a restored no-agent tab was replaced create-before-close, while a registered live agent caused refusal. +### Launcher workspace placement + +Herdr exports its pane identity into every process it manages, checked on 2026-07-30 against Herdr 0.7.5 protocol 17 inside a guarded lab pane: + +```sh +HERDR_LAB_HELPER=bin/fm-herdr-lab.sh +"$HERDR_LAB_HELPER" run "$LAB" pane run "$PANE" "sh -c 'env | grep ^HERDR | sort > /tmp/env.txt'" +``` + +```text +HERDR_ENV=1 +HERDR_PANE_ID=w1:p1 +HERDR_SESSION=fm-lab-fm-herdr-env-pro-65961-25535 +HERDR_SOCKET_PATH=/Users/kunchen/.config/herdr/sessions/fm-lab-fm-herdr-env-pro-65961-25535/herdr.sock +HERDR_TAB_ID=w1:t1 +HERDR_WORKSPACE_ID=w1 +``` + +This complete injection shape is verified only for Herdr 0.7.5. +Firstmate requires both `HERDR_PANE_ID` and `HERDR_SOCKET_PATH` before accepting claimed launcher ancestry. + +`pane get` reports the pane's current owning tab and workspace, which is what placement resolves from; the injected `HERDR_TAB_ID` and `HERDR_WORKSPACE_ID` are creation-time snapshots and are not read as current identity: + +```sh +"$HERDR_LAB_HELPER" run "$LAB" pane get w1:p1 | jq -c '.result.pane | {pane_id,tab_id,workspace_id}' +``` + +```text +{"pane_id":"w1:p1","tab_id":"w1:t1","workspace_id":"w1"} +``` + +Placement is owned by: + +```sh +HERDR_LAB_HELPER=bin/fm-herdr-lab.sh \ + tests/fm-backend-herdr-launcher-workspace-e2e.test.sh +``` + +Observed guarantees on 2026-07-30 against Herdr 0.7.5 protocol 17: + +```text +ok - real herdr E2E: with one 'firstmate' workspace and no herdr parent, a crewmate still lands in this home's own workspace without stealing focus +ok - real herdr E2E: the normal unique-label path is unchanged when the launcher's own pane identifies the workspace +ok - real herdr E2E: presentation spaces still create the isolated child workspace and bind it under the launcher's exact parent, without stealing focus +ok - real herdr E2E: with two 'firstmate' workspaces, a worker spawned from inside the second one lands in that exact workspace +ok - real herdr E2E: the duplicate-labeled sibling workspace is left entirely untouched and focus is preserved +ok - real herdr E2E: with a duplicated home label, a projected worker still hangs off the launcher's exact workspace and the sibling stays untouched +ok - real herdr E2E: an ambiguous home label with no launcher identity refuses before any worker endpoint exists +ok - real herdr E2E: a launcher pane that no longer exists refuses before any worker endpoint exists +ok - real herdr E2E: a secondmate launching its own worker gets the same exact-workspace guarantee, and its same-labeled sibling is untouched +ok - real herdr E2E: a --secondmate launch still stands up that secondmate's own workspace instead of inheriting the launcher's +ok - real herdr E2E: teardown closes only the worker's own pane and leaves the launcher, its workspace, and the same-labeled sibling intact +``` + +That suite's headline case runs `bin/fm-spawn.sh` inside a real Herdr pane, so the parent identity comes from Herdr's own injection rather than a composed environment. +Cross-session and contradictory bindings are covered deterministically in `tests/fm-backend-herdr.test.sh`, which can script a second server's socket without provisioning one. + ### Per-home and presentation topology Per-home behavior is owned by: diff --git a/tests/fm-afk-inject-herdr-e2e.test.sh b/tests/fm-afk-inject-herdr-e2e.test.sh index 644d015086..9c5c66c5e8 100755 --- a/tests/fm-afk-inject-herdr-e2e.test.sh +++ b/tests/fm-afk-inject-herdr-e2e.test.sh @@ -41,6 +41,11 @@ command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the her # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +# This suite runs against its own isolated lab session, so a Herdr pane +# inherited from the terminal it was launched in must not follow spawn into it +# as a cross-session parent identity (tests/herdr-test-safety.sh). +herdr_forget_inherited_pane + fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } diff --git a/tests/fm-backend-autodetect-smoke.test.sh b/tests/fm-backend-autodetect-smoke.test.sh index b4c8c887fb..17fe88f617 100755 --- a/tests/fm-backend-autodetect-smoke.test.sh +++ b/tests/fm-backend-autodetect-smoke.test.sh @@ -43,6 +43,14 @@ command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (requi export FM_GATE_REFUSE_BYPASS=1 +# shellcheck source=tests/herdr-test-safety.sh +. "$ROOT/tests/herdr-test-safety.sh" +# This suite asserts that HERDR_ENV=1 alone selects the backend, and it runs +# against its own isolated lab session. A Herdr pane inherited from the terminal +# it was launched in must not follow spawn into that session as a cross-session +# parent identity; the spawn below sets HERDR_ENV explicitly. +herdr_forget_inherited_pane + # TMP_ROOT is physically resolved (mktemp -d "$(pwd -P)"-relative) to keep this # real-herdr smoke fixture free of unrelated OS symlink noise. # The old fm-spawn bug that originally motivated this fixture shape was fixed in diff --git a/tests/fm-backend-herdr-eventwait-smoke.test.sh b/tests/fm-backend-herdr-eventwait-smoke.test.sh index 5616a5bc75..b383176d7d 100755 --- a/tests/fm-backend-herdr-eventwait-smoke.test.sh +++ b/tests/fm-backend-herdr-eventwait-smoke.test.sh @@ -25,6 +25,11 @@ command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found (required # shellcheck source=tests/herdr-test-safety.sh . "$ROOT/tests/herdr-test-safety.sh" +# This suite runs against its own isolated lab session, so a Herdr pane +# inherited from the terminal it was launched in must not follow spawn into it +# as a cross-session parent identity (tests/herdr-test-safety.sh). +herdr_forget_inherited_pane + SESSION="fm-lab-eventwait-smoke-$$" export HERDR_SESSION="$SESSION" SCRATCH= diff --git a/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh b/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh new file mode 100755 index 0000000000..ca5cc4575e --- /dev/null +++ b/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash +# tests/fm-backend-herdr-launcher-workspace-e2e.test.sh - mandatory ISOLATED +# end-to-end real-Herdr test for worker PLACEMENT with presentation spaces +# disabled. +# +# The guarantee under test: a crewmate or scout is created in the exact Herdr +# workspace of the firstmate or secondmate process that launched it, identified +# from that process's own Herdr pane rather than from a workspace label. Herdr +# enforces no workspace-label uniqueness, so two workspaces can both be labeled +# "firstmate", and the previous label-first-match resolution put the worker in +# whichever one sorted first - visibly the wrong space whenever the launcher was +# not in it. +# +# This drives the REAL bin/fm-spawn.sh and bin/fm-teardown.sh, because the +# guarantee spans the whole spawn handoff (fm-spawn.sh's herdr arm -> +# fm_backend_herdr_container_ensure -> fm_backend_herdr_workspace_ensure -> +# fm_backend_herdr_launcher_identity) and no adapter primitive holds it alone. +# The headline duplicate-label case additionally runs fm-spawn.sh INSIDE a real +# Herdr pane, so the pane identity comes from Herdr's own injection rather than +# from an environment this test composed. +# +# Safety (2026-07-02 incident, see tests/herdr-test-safety.sh): every lifecycle +# operation goes through bin/fm-herdr-lab.sh, which appends the named session +# flag and verifies the default fleet session is unchanged after teardown. +set -u + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +assert_contains_local() { # + case "$1" in + *"$2"*) : ;; + *) fail "$3"$'\n'"--- got ---"$'\n'"$1" ;; + esac +} + +command -v herdr >/dev/null 2>&1 || { echo "skip: herdr not found"; exit 0; } +command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the herdr adapter)"; exit 0; } +command -v treehouse >/dev/null 2>&1 || { echo "skip: treehouse not found (required by fm-spawn.sh)"; exit 0; } + +# shellcheck source=tests/herdr-test-safety.sh +. "$ROOT/tests/herdr-test-safety.sh" + +# Every spawn below states its own launcher identity, so a pane inherited from +# the terminal this suite was started in must not leak into any of them. +herdr_forget_inherited_pane + +TMP_ROOT=$(mktemp -d "$(cd "${TMPDIR:-/tmp}" && pwd -P)/fm-herdr-launcher-e2e.XXXXXX") +HERDR_LAB_HELPER="$ROOT/bin/fm-herdr-lab.sh" +HERDR_LAB_SESSION=$("$HERDR_LAB_HELPER" name fm-herdr-launcher-ws) || { + rm -rf "$TMP_ROOT" + printf 'not ok - could not generate an isolated Herdr lab session name\n' >&2 + exit 1 +} +export HERDR_SESSION="$HERDR_LAB_SESSION" + +WORKTREES=() +CLEANED=0 +# Idempotent: fail() cleans up before exiting and the EXIT trap fires after it, +# so a second teardown would otherwise report the already-consumed fleet-state +# tripwire as if the lab had gone wrong. +cleanup_all() { + local wt status=0 + [ "$CLEANED" = 0 ] || return 0 + CLEANED=1 + for wt in ${WORKTREES[@]+"${WORKTREES[@]}"}; do + [ -n "$wt" ] && treehouse return --force "$wt" >/dev/null 2>&1 + done + WORKTREES=() + "$HERDR_LAB_HELPER" teardown "$HERDR_LAB_SESSION" || status=$? + rm -rf "$TMP_ROOT" + return "$status" +} +trap cleanup_all EXIT +"$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION" || fail "could not provision isolated Herdr lab session" + +lab() { "$HERDR_LAB_HELPER" run "$HERDR_LAB_SESSION" "$@"; } + +# --- helpers ---------------------------------------------------------------- + +make_scratch_project() { # + local dir=$1 + mkdir -p "$dir" + git -C "$dir" init -q + printf '# scratch\n' > "$dir/README.md" + git -C "$dir" add README.md + git -C "$dir" -c user.name='Firstmate Tests' -c user.email='tests@example.invalid' commit -qm initial +} + +# make_workspace