From 5bd5bb7d60f4dd0da467ef5871e0f6d3926a1171 Mon Sep 17 00:00:00 2001 From: allstargg <51008413+allstargg@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:03:00 -0400 Subject: [PATCH] feat: apply native W3C trace-context (PR 995) to the fork Apply only the isolated upstream PR-995 trace-context feature delta on top of the current fork main, which already carries canonical Firstmate content from the fork-sync merge. This makes the already-validated, default-off W3C trace-context capability available on the captain's integration fork without waiting for the canonical upstream merge of PR 995. The change is the exact PR-995 feature delta (a53ffc1..2ff8ab8, the validated integration merge of PR-995 head 2fbe2fd), applied with cherry-pick -m1 so the trace-context implementation is preserved with no semantic delta; every feature-only file is byte-identical to the PR-995 head. Three genuine overlaps with the newer canonical commit #1303 (bound and consolidate startup memory during stow) are resolved by preserving both behaviors: the inheritable-config set in bin/fm-config-inherit-lib.sh, the config-push list in docs/configuration.md, and the inheritance comment in tests/fm-secondmate-harness.test.sh now declare both startup-memory-budget and trace-context. No canonical #1303 content is dropped and no unrelated upstream content is reintroduced. Focused suites pass: fm-trace-context-lib, fm-trace-context-spawn, fm-session-start, fm-documentation-audiences, fm-secondmate-harness, and bin/fm-lint.sh. --- AGENTS.md | 3 +- bin/backends/cmux.sh | 10 +- bin/backends/zellij.sh | 12 +- bin/fm-bootstrap.sh | 2 +- bin/fm-config-inherit-lib.sh | 17 +- bin/fm-config-push.sh | 3 +- bin/fm-session-start.sh | 5 + bin/fm-spawn.sh | 60 +++- bin/fm-test-isolation-proof.sh | 3 +- bin/fm-test-run.sh | 6 +- bin/fm-trace-context-lib.sh | 229 ++++++++++++ docs/configuration.md | 11 +- docs/documentation-audiences.json | 12 + docs/trace-context.md | 102 ++++++ docs/verification/trace-context.md | 26 ++ tests/fm-backend-cmux.test.sh | 43 +++ tests/fm-backend-zellij.test.sh | 44 +++ tests/fm-secondmate-harness.test.sh | 24 +- tests/fm-session-start.test.sh | 38 +- tests/fm-test-run.test.sh | 22 +- tests/fm-trace-context-lib.test.sh | 311 +++++++++++++++++ tests/fm-trace-context-spawn.test.sh | 501 +++++++++++++++++++++++++++ 22 files changed, 1440 insertions(+), 44 deletions(-) create mode 100644 bin/fm-trace-context-lib.sh create mode 100644 docs/trace-context.md create mode 100644 docs/verification/trace-context.md create mode 100755 tests/fm-trace-context-lib.test.sh create mode 100755 tests/fm-trace-context-spawn.test.sh diff --git a/AGENTS.md b/AGENTS.md index c75eeb7759..6b31d9bced 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,6 +72,7 @@ config/backend runtime session-provider backend override for new tasks; LOCAL, config/calm Pi Calm presentation preference; LOCAL, gitignored, and not inherited; see docs/configuration.md "Pi Calm preference" config/startup-memory-budget primary-authoritative per-home startup-memory budget; LOCAL, gitignored, materialized as 7,500 estimated tokens by locked primary bootstrap and inherited into secondmate homes; see docs/configuration.md "Startup memory budget" config/herdr-presentation-spaces optional presence flag for Herdr's default-off disposable single-task visual projection; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Optional presentation spaces" +config/trace-context optional presence flag enabling default-off native W3C trace-context propagation to spawned agents; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Trace context propagation" and docs/trace-context.md config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md config/x-mode.env generated X-mode watcher cadence; LOCAL, gitignored; source before arming watcher when present @@ -90,7 +91,7 @@ state/ volatile runtime signals; gitignored .turn-ended touched by turn-end hooks .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown - .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; kind=secondmate also records home= and projects=; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) + .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Optional presentation spaces" .check.sh authenticated slow poll; the watcher dispatches validated PR data and the byte-identified X shim through trusted repository scripts, runs registered custom checks from hash-validated private snapshots, and rejects every other state check without execution .check-trust private content binding created by fm-check-register.sh for an intentional custom check diff --git a/bin/backends/cmux.sh b/bin/backends/cmux.sh index 12dc7629eb..745450b11c 100644 --- a/bin/backends/cmux.sh +++ b/bin/backends/cmux.sh @@ -501,14 +501,12 @@ fm_backend_cmux_send_key() { # [expected-label] fm_backend_cmux_cli send-key --workspace "$FM_BACKEND_CMUX_WORKSPACE" --surface "$FM_BACKEND_CMUX_SURFACE" "$key" >/dev/null 2>&1 } -# fm_backend_cmux_send_text_line: send one line of TEXT then submit. cmux has -# no single-call atomic "run and submit" primitive (like herdr's `pane run`), -# so this composes send (literal) + send-key enter, exactly like zellij's -# equivalent - used for the fixed spawn-time commands (treehouse get, the -# GOTMPDIR export). +# fm_backend_cmux_send_text_line: send one line of TEXT then submit. fm_backend_cmux_send_text_line() { # [expected-label] fm_backend_cmux_send_literal "$1" "$2" "${3:-}" || return 1 - fm_backend_cmux_send_key "$1" Enter "${3:-}" + fm_backend_cmux_send_key "$1" Enter "${3:-}" && return 0 + fm_backend_cmux_send_key "$1" C-c "${3:-}" >/dev/null 2>&1 && return 1 + return 2 } # fm_backend_cmux_capture: bounded plain-text surface capture. No herdr-style diff --git a/bin/backends/zellij.sh b/bin/backends/zellij.sh index 20d53a3c2d..a5d89ba502 100644 --- a/bin/backends/zellij.sh +++ b/bin/backends/zellij.sh @@ -460,16 +460,12 @@ fm_backend_zellij_send_key() { # [expected-label] fm_backend_zellij_cli "$FM_BACKEND_ZELLIJ_SESSION" action send-keys --pane-id "$FM_BACKEND_ZELLIJ_PANE" "$key" >/dev/null 2>&1 } -# fm_backend_zellij_send_text_line: send one line of TEXT then submit, -# ATOMICALLY - mirrors tmux's `send-keys -t T text Enter` / herdr's `pane -# run`. Used for the fixed spawn-time commands (treehouse get, the GOTMPDIR -# export). Zellij has no single-call atomic "run and submit" action, so this -# composes paste (literal) + send-keys Enter, exactly like send_literal + -# send_key are composed elsewhere - the two-step form is the ONLY form for -# this adapter, unlike tmux/herdr which have a genuinely atomic primitive. +# fm_backend_zellij_send_text_line: send one line of TEXT then submit. fm_backend_zellij_send_text_line() { # [expected-label] fm_backend_zellij_send_literal "$1" "$2" "${3:-}" || return 1 - fm_backend_zellij_send_key "$1" Enter "${3:-}" + fm_backend_zellij_send_key "$1" Enter "${3:-}" && return 0 + fm_backend_zellij_send_key "$1" C-c "${3:-}" >/dev/null 2>&1 && return 1 + return 2 } # fm_backend_zellij_capture: bounded plain-text pane capture. Mirrors diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 16102adfa4..4bcfdd6e4c 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -394,7 +394,7 @@ secondmate_sync() { fm_lock_release "$home_lock" || true continue } - if FM_CONFIG_INHERIT_REPORT="$report" \ + if FM_CONFIG_INHERIT_REPORT="$report" FM_CONFIG_INHERIT_LIVE=1 \ propagate_secondmate_inheritance "$FM_HOME" "$home_real" "$CONFIG" "$DATA"; then : else diff --git a/bin/fm-config-inherit-lib.sh b/bin/fm-config-inherit-lib.sh index bffbd5234d..d894c3b69c 100644 --- a/bin/fm-config-inherit-lib.sh +++ b/bin/fm-config-inherit-lib.sh @@ -9,9 +9,14 @@ # runtime-backend default for future spawns, primary config/startup-memory-budget # bounds that home's startup-memory curation, and primary # config/herdr-presentation-spaces enables the same default-off Herdr presentation -# projection). It also pushes the one primary-authoritative shared -# captain-preference file, data/captain-shared.md, into each secondmate home's -# data/ as a read-only copy. +# projection, and primary +# config/trace-context is copied at the launch convergence point as part of the +# default-off W3C trace-context setup, while live convergence leaves it unchanged. +# The primary passes its frozen home-session decision into a newly launched +# Secondmate; see docs/trace-context.md. +# It also pushes +# the one primary-authoritative shared captain-preference file, +# data/captain-shared.md, into each secondmate home's data/ as a read-only copy. # # Usage: . bin/fm-config-inherit-lib.sh (no FM_* setup required) # @@ -46,7 +51,7 @@ FM_SHARED_CAPTAIN_MODE="444" # The declared inheritable set (space-separated, config-dir-relative item paths). # Extend here to inherit more of the primary's local config; override via the # environment only in tests. Items must not contain whitespace. -FM_INHERITABLE_CONFIG="${FM_INHERITABLE_CONFIG:-crew-dispatch.json crew-harness backlog-backend backend herdr-presentation-spaces startup-memory-budget}" +FM_INHERITABLE_CONFIG="${FM_INHERITABLE_CONFIG:-crew-dispatch.json crew-harness backlog-backend backend herdr-presentation-spaces startup-memory-budget trace-context}" fm_inherit_file_mode() { if [ "$(uname)" = Darwin ]; then @@ -403,6 +408,10 @@ propagate_inheritable_config() { case "$item" in ''|/*|.|..|../*|*/../*|*/..) return 1 ;; esac + if [ "${FM_CONFIG_INHERIT_LIVE:-0}" = 1 ] && [ "$item" = trace-context ]; then + record_inheritable_config_result "$item" unchanged "session-scoped" + continue + fi src="$src_config/$item" dest="$dest_config/$item" # This one scalar config is consumed as a local safety boundary, so reject diff --git a/bin/fm-config-push.sh b/bin/fm-config-push.sh index b760666dd1..8867d3f4e5 100755 --- a/bin/fm-config-push.sh +++ b/bin/fm-config-push.sh @@ -167,7 +167,8 @@ while IFS='|' read -r id home _window meta; do continue } reports="$reports $report" - if FM_CONFIG_INHERIT_REPORT="$report" propagate_secondmate_inheritance "$FM_HOME" "$home_real" "$CONFIG" "$DATA"; then + if FM_CONFIG_INHERIT_REPORT="$report" FM_CONFIG_INHERIT_LIVE=1 \ + propagate_secondmate_inheritance "$FM_HOME" "$home_real" "$CONFIG" "$DATA"; then : else errors=1 diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 1abbace4bf..6a96ae6131 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -103,6 +103,8 @@ PRIMARY_HARNESS=$("$SCRIPT_DIR/fm-harness.sh" 2>/dev/null || printf unknown) . "$SCRIPT_DIR/fm-backend.sh" # shellcheck source=bin/fm-tasks-axi-lib.sh . "$SCRIPT_DIR/fm-tasks-axi-lib.sh" +# shellcheck source=bin/fm-trace-context-lib.sh +. "$SCRIPT_DIR/fm-trace-context-lib.sh" STATUS_TAIL=${FM_SESSION_START_STATUS_TAIL:-5} case "$STATUS_TAIL" in ''|*[!0-9]*) STATUS_TAIL=5 ;; esac @@ -264,6 +266,9 @@ if [ "$LOCK_RC" -ne 0 ]; then printf '%s\n' "$BAR" } fi +if [ "$READ_ONLY" -eq 0 ]; then + fm_trace_context_session_start "$CONFIG" "$STATE/.trace-context-effective" +fi # --- 2. bootstrap -------------------------------------------------------- subsection "BOOTSTRAP" diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 3526572550..883972fe2a 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -110,6 +110,11 @@ # On success prints: spawned harness= kind= mode= yolo= window= worktree= # mode/yolo are resolved per-project from data/projects.md for ship/scout tasks; # secondmate spawns record mode=secondmate, yolo=off, home=, and projects=. +# When the home session's frozen trace-context decision is enabled (see +# docs/configuration.md and bin/fm-trace-context-lib.sh), the meta also records +# one W3C traceparent= carrier, the same value injected into the pane as +# TRACEPARENT; the default-off path writes neither, leaving the generated meta +# and launch environment unchanged. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -161,6 +166,8 @@ SUB_HOME_MARKER=".fm-secondmate-home" . "$SCRIPT_DIR/fm-gate-refuse-lib.sh" # shellcheck source=bin/fm-pr-lib.sh . "$SCRIPT_DIR/fm-pr-lib.sh" +# shellcheck source=bin/fm-trace-context-lib.sh +. "$SCRIPT_DIR/fm-trace-context-lib.sh" # Fail closed before any fleet mutation: a no-mistakes gate agent must never spawn # a direct report (see bin/fm-gate-refuse-lib.sh). fm_refuse_if_gate_agent @@ -818,9 +825,10 @@ if [ "$KIND" = secondmate ]; then exit 1 fi CONFIG_INHERIT_LOCK_HELD=1 - # Inheritance propagation: push the primary-authoritative local inheritance + # Inheritance propagation: push the primary-authoritative live-safe inheritance # surface into this secondmate home (fm-config-inherit-lib.sh). - propagate_secondmate_inheritance "$FM_HOME" "$PROJ_ABS" "$CONFIG" "$DATA" \ + FM_CONFIG_INHERIT_LIVE=1 \ + propagate_secondmate_inheritance "$FM_HOME" "$PROJ_ABS" "$CONFIG" "$DATA" \ || echo "warning: secondmate $ID inheritance failed for $PROJ_ABS" >&2 if [ -f "$PROJ_ABS/data/charter.md" ]; then BRIEF="$PROJ_ABS/data/charter.md" @@ -1165,6 +1173,11 @@ EOF T="$ORCA_TERMINAL" ;; esac +if [ "$KIND" = secondmate ]; then + FM_INHERITABLE_CONFIG=trace-context \ + propagate_inheritable_config "$CONFIG" "$PROJ_ABS/config" \ + || echo "warning: secondmate $ID trace-context inheritance failed for $PROJ_ABS" >&2 +fi # #134 robustness: only tmux needs a worktree-detection target distinct from $T - # its rename-safe stable window id, set as WT_TARGET=$WID in the tmux branch above. # Every other backend addresses its pane/surface by the id already in $T, so default @@ -1451,6 +1464,23 @@ $("$FM_ROOT/bin/fm-project-mode.sh" "$PROJ_NAME") EOF fi +# Resolve the optional default-off W3C trace context (bin/fm-trace-context-lib.sh, +# docs/configuration.md): the one carrier both recorded in meta and injected into +# the pane, so an observer reads exactly what the child receives. Empty only when +# disabled or on entropy/validation failure; malformed or all-zero inherited +# context is treated as absent and roots a fresh trace. Reuses this task's +# already-recorded value on relaunch. Never aborts the spawn and adds only the +# cost of reading a few bytes of entropy. +# +# The session-start path owns input resolution. Spawn consumes only the frozen +# home-session state and reuses it for the carrier and Secondmate launch prefix. +SPAWN_TRACE_EFFECTIVE=$(fm_trace_context_session_effective "$STATE/.trace-context-effective") +if [ "$SPAWN_TRACE_EFFECTIVE" = on ]; then + SPAWN_TRACEPARENT=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CONFIG" "$STATE/$ID.meta" || true) +else + SPAWN_TRACEPARENT= +fi + META_WINDOW=$T [ "$BACKEND" = orca ] && META_WINDOW=$W { @@ -1465,6 +1495,7 @@ META_WINDOW=$T echo "tasktmp=$TASK_TMP" echo "model=${MODEL:-default}" echo "effort=${EFFORT:-default}" + # Default-off writes no traceparent= line (meta stays byte-identical). # backend= is written only for a non-default (non-tmux) backend, so the # default path's meta stays byte-identical (absent backend= means tmux; # data/fm-backend-design-d7's P1 compatibility contract). @@ -1523,12 +1554,35 @@ if [ "$HARNESS" = claude ] && [ -n "${CLAUDE_CONFIG_DIR:-}" ]; then fi if [ "$KIND" = secondmate ]; then sq_home=$(shell_quote "$PROJ_ABS") - LAUNCH="FM_ROOT_OVERRIDE= FM_STATE_OVERRIDE= FM_DATA_OVERRIDE= FM_PROJECTS_OVERRIDE= FM_CONFIG_OVERRIDE= FM_HOME=$sq_home $LAUNCH" + # Deliver the primary's EFFECTIVE trace-context decision as a normalized on/off + # literal (never the raw FM_TRACE_CONTEXT string) so a FM_TRACE_CONTEXT override + # on the primary reaches the secondmate's OWN workers, not just the copied + # config/trace-context file: otherwise off would not disable them and on would + # not enable them across the launch boundary (bin/fm-trace-context-lib.sh header). + # Reuse the single frozen decision from the carrier resolution above so the + # injected carrier and this on/off snapshot are guaranteed to agree. + LAUNCH="FM_ROOT_OVERRIDE= FM_STATE_OVERRIDE= FM_DATA_OVERRIDE= FM_PROJECTS_OVERRIDE= FM_CONFIG_OVERRIDE= FM_HOME=$sq_home FM_TRACE_CONTEXT=$SPAWN_TRACE_EFFECTIVE $LAUNCH" fi # Export GOTMPDIR into the crewmate's pane shell so the agent and every child # process (go build, go test, ...) inherit it. Sent before the launch command so # the env is set when the agent starts; the brief sleep lets the export land. spawn_send_text_line "$T" "export GOTMPDIR=$TASK_TMP/gotmp" +# Send through the exact channel that already ships GOTMPDIR, so every backend +# and harness - ship, scout, and secondmate - gets it before launch. Skipped +# entirely when trace context is off. +if [ -n "$SPAWN_TRACEPARENT" ]; then + if spawn_send_text_line "$T" "export TRACEPARENT=$SPAWN_TRACEPARENT"; then + if ! echo "traceparent=$SPAWN_TRACEPARENT" >> "$STATE/$ID.meta"; then + LAUNCH="unset TRACEPARENT; $LAUNCH" + fi + else + TRACE_SEND_STATUS=$? + if [ "$TRACE_SEND_STATUS" -eq 2 ]; then + echo "error: trace-context input could not be cleared for $W; refusing to append the launch command" >&2 + exit 1 + fi + fi +fi sleep 0.3 spawn_send_literal "$T" "$LAUNCH" sleep 0.3 diff --git a/bin/fm-test-isolation-proof.sh b/bin/fm-test-isolation-proof.sh index f84f8ed09e..7070e95e40 100755 --- a/bin/fm-test-isolation-proof.sh +++ b/bin/fm-test-isolation-proof.sh @@ -96,7 +96,7 @@ exclusion_reason() { fm-backend.test.sh) printf '%s\n' 'old-vs-new main checkout diff fixture; gray-zone concurrent git/worktree cost' ;; - fm-spawn-dispatch-profile.test.sh|fm-spawn-worktree-settle.test.sh) + fm-spawn-dispatch-profile.test.sh|fm-spawn-worktree-settle.test.sh|fm-trace-context-spawn.test.sh) printf '%s\n' 'real isolated git worktrees plus spawn settle loops; gray zone until dedicated proof' ;; fm-pr-check-security.test.sh) @@ -190,6 +190,7 @@ fm-backend-tmux-smoke.test.sh fm-backend.test.sh fm-spawn-dispatch-profile.test.sh fm-spawn-worktree-settle.test.sh +fm-trace-context-spawn.test.sh fm-pr-check-security.test.sh fm-teardown.test.sh fm-watcher-lock.test.sh diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index f89a6bade5..7c1f77ddb2 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -126,7 +126,8 @@ family_for_basename() { fm-operational-input.test.sh|fm-pi-primary-types.test.sh|\ fm-send-popup-settle.test.sh|fm-send-settle.test.sh|\ fm-subagent-pretool-check.test.sh|\ - fm-supervision-instructions.test.sh|fm-tmux-submit-busy.test.sh|fm-transition-lib.test.sh|\ + fm-supervision-instructions.test.sh|fm-tmux-submit-busy.test.sh|fm-trace-context-lib.test.sh|\ + fm-transition-lib.test.sh|\ fm-test-run.test.sh|fm-test-isolation-proof.test.sh) printf '%s\n' pure-contract-unit ;; @@ -162,7 +163,8 @@ family_for_basename() { ;; fm-backend-herdr.test.sh|fm-backend-tmux-smoke.test.sh|fm-backend.test.sh|\ fm-herdr-session-cleanup.test.sh|fm-send-strict.test.sh|fm-spawn-batch.test.sh|\ - fm-spawn-dispatch-profile.test.sh|fm-spawn-worktree-settle.test.sh|\ + fm-spawn-dispatch-profile.test.sh|\ + fm-trace-context-spawn.test.sh|fm-spawn-worktree-settle.test.sh|\ fm-teardown-endpoint-safety.test.sh) printf '%s\n' backend-dispatch ;; diff --git a/bin/fm-trace-context-lib.sh b/bin/fm-trace-context-lib.sh new file mode 100644 index 0000000000..55f8ed6718 --- /dev/null +++ b/bin/fm-trace-context-lib.sh @@ -0,0 +1,229 @@ +# shellcheck shell=bash +# Native W3C trace-context propagation for firstmate spawns (default-off). +# +# When enabled, firstmate resolves one W3C `traceparent` carrier for a task, +# injects it into the agent's pane shell as the TRACEPARENT environment variable +# before launch (bin/fm-spawn.sh, alongside GOTMPDIR, so it reaches every spawn +# backend and every harness for ship, scout, and secondmate spawns), and records +# the identical value as `traceparent=` in state/.meta. Because the injected +# carrier and the recorded carrier are the same string, an observer that reads +# the metadata sees exactly the identity the child received - no collector, +# storage, UI, or vendor coupling. +# +# TRACEPARENT here is a firstmate CONVENTION that carries a W3C-formatted +# traceparent value in the process environment. W3C Trace Context standardizes +# the `traceparent` HTTP header, not an environment variable, and OpenTelemetry +# SDKs do NOT read TRACEPARENT from the environment automatically. A downstream +# observer or instrumentation must explicitly read this env value (or the meta +# field); this library parents no SDK span by itself. +# +# Identity is per TASK, not per spawn: the carrier is minted on the first spawn, +# adopted as a child (fresh span, same trace) for a nested spawn whose parent +# already holds one, and REUSED verbatim from the meta on relaunch/recovery, so a +# task keeps one stable logical identity across restarts. +# +# Usage: . bin/fm-trace-context-lib.sh +# +# Public entry points: +# fm_trace_context_session_start +# Resolves config/trace-context plus FM_TRACE_CONTEXT once and atomically +# writes the normalized on/off decision bound to the locked home session. +# fm_trace_context_session_effective +# Echoes the normalized frozen decision only when its session binding matches +# the current lock, defaulting to off when the state is absent, stale, or invalid. +# fm_trace_context_resolve [] +# Echoes the traceparent to inject AND record, or nothing when the +# capability is off or when entropy or self-validation fails. A malformed or +# all-zero inherited value is NOT an omission: it is treated as absent and a +# fresh root is minted (see the root/child rules below). It ALWAYS returns 0: +# telemetry is omitted safely and never aborts the spawn. The third argument +# defaults to $TRACEPARENT so a nested secondmate or worker spawn continues +# its parent's trace; pass an explicit value (including empty) to override, +# mainly for tests. +# +# Enablement (see docs/configuration.md for the schema): +# config/trace-context presence flag under the home's config dir enables it. +# FM_TRACE_CONTEXT env override: 1/on/true/yes enables, any other +# non-empty value disables, and unset OR empty defers +# to the file. +# Each locked home session resolves these inputs once into +# state/.trace-context-effective. The record is atomically published through a +# same-directory temporary file and bound to state/.lock; a failed publication +# cannot reactivate a stale on decision. Every spawn reads only that frozen +# on/off value, so later config and environment edits take effect only after a +# new home session starts. +# At launch, the primary propagates config/trace-context into the secondmate +# home (FM_INHERITABLE_CONFIG in bin/fm-config-inherit-lib.sh) and passes its +# frozen on/off decision into the new process as a non-empty FM_TRACE_CONTEXT +# value in the launch prefix (bin/fm-spawn.sh). The Secondmate freezes that +# inherited decision when its own home session starts. +# +# Wire shape: version 00 only, "00-<32 hex trace>-<16 hex span>-<2 hex flags>", +# with the trace id and span id never all-zero (W3C rejects both). New roots use +# RANDOM ids from /dev/urandom. A `01` (sampled) flag on a root records a +# sampling DECISION that downstream parent-based samplers honor; it does not +# guarantee any collector stores a span, and firstmate emits no spans itself. A +# child preserves the inherited flag verbatim. +# +# Security / trust boundary. This feature adds no OTEL_* variables, no +# tracestate, no arbitrary environment injection, and no configurable or +# arbitrary command execution. It DOES run the fixed local utilities `od` and +# `tr` (resolved from PATH) to read a few bytes of entropy - a small local +# pipeline with no configured provider, network, or watchdog, and no hard latency +# guarantee; any resolver failure that returns omits the carrier without aborting +# the spawn. Carrier-delivery failure also omits telemetry and continues when the +# backend clears its input; if the backend reports that partial input could not be +# cleared, fm-spawn refuses to append the launch command. A firstmate-MINTED root +# is random and reads no prompt, path, +# task prose, credential, or arbitrary environment key. An INHERITED traceparent, +# by contrast, is opaque caller-controlled data: its 16-byte trace id and 8-byte +# span id are up to 24 bytes (48 hex chars) that firstmate accepts after syntax +# validation WITHOUT interpreting, so whoever set TRACEPARENT in firstmate's +# environment (a trusted local operator or observer) controls those bytes. +# Exposure is bounded to that fixed-width carrier - it is not a general content +# or secret channel, but it is not "structurally impossible to carry data" either. +# +# Root / child / recovery semantics (never mint an unrelated root by accident): +# recovery - a valid traceparent already recorded in the meta file is reused +# verbatim, so a relaunched or recovered task keeps one stable +# identity across restarts. +# child - a valid inherited traceparent contributes its trace id and flags +# while a fresh span id is minted, so nested spawns share one trace. +# root - with no valid inherited context a fresh random trace id, fresh +# span id, and sampled flags (01) begin a new trace. A malformed or +# all-zero inherited value is treated as absent, so garbage never +# propagates. + +# Strict W3C traceparent validator: version 00, 32-hex trace id, 16-hex span id, +# 2-hex flags, with neither id all-zero. The regex lives in a variable because +# bash 3.2 only honors an unquoted right-hand side for =~. +fm_trace_context_valid() { # + local tp=$1 + local re='^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$' + [[ $tp =~ $re ]] || return 1 + [ "${tp:3:32}" = "00000000000000000000000000000000" ] && return 1 + [ "${tp:36:16}" = "0000000000000000" ] && return 1 + return 0 +} + +# Echo random bytes as lowercase hex, or echo nothing and return 1 +# on any entropy failure (unreadable source, short read, non-hex). -v stops od +# from collapsing repeated byte lines to '*'; the explicit length and charset +# checks turn a masked pipeline failure into a clean omission upstream. +fm_trace_context_hex() { # + local bytes=$1 hex + hex=$(LC_ALL=C od -An -v -tx1 -N "$bytes" /dev/urandom 2>/dev/null | tr -d ' \n') || return 1 + case "$hex" in + '' | *[!0-9a-f]*) return 1 ;; + esac + [ "${#hex}" -eq "$((bytes * 2))" ] || return 1 + printf '%s' "$hex" +} + +# True when the capability is enabled for this home. The env override wins so a +# spawn can be forced on or off without touching the file; otherwise the +# presence of config/trace-context decides, and its absence is the default-off. +fm_trace_context_enabled() { # + local config_dir=$1 v + # A non-empty value is an explicit override; unset OR empty defers to the file + # (the conventional "empty is like unset" behavior). + if [ -n "${FM_TRACE_CONTEXT:-}" ]; then + v=$(printf '%s' "$FM_TRACE_CONTEXT" | tr '[:upper:]' '[:lower:]') + case "$v" in + 1 | on | true | yes) return 0 ;; + *) return 1 ;; + esac + fi + [ -f "$config_dir/trace-context" ] +} + +# Echo the lock pid that owns the effective-state file's home, or fail when the +# adjacent session lock is absent or malformed. Binding the decision to this +# token makes a prior session's record inactive even if publication cannot +# replace or remove that stale file. +fm_trace_context_session_lock() { # + local effective_file=$1 state_dir lock_pid + state_dir=${effective_file%/*} + [ "$state_dir" = "$effective_file" ] && state_dir=. + IFS= read -r lock_pid < "$state_dir/.lock" 2>/dev/null || return 1 + case "$lock_pid" in + '' | *[!0-9]*) return 1 ;; + esac + [ "$lock_pid" -gt 1 ] || return 1 + printf '%s' "$lock_pid" +} + +fm_trace_context_session_start() { # + local config_dir=$1 effective_file=$2 value=off lock_pid tmp + lock_pid=$(fm_trace_context_session_lock "$effective_file") || { + rm -f "$effective_file" 2>/dev/null || true + return 0 + } + fm_trace_context_enabled "$config_dir" && value=on + tmp=$(mktemp "$effective_file.tmp.XXXXXX" 2>/dev/null) || { + rm -f "$effective_file" 2>/dev/null || true + return 0 + } + if ! printf '%s %s\n' "$lock_pid" "$value" > "$tmp" 2>/dev/null \ + || ! mv -f "$tmp" "$effective_file" 2>/dev/null; then + rm -f "$tmp" 2>/dev/null || true + rm -f "$effective_file" 2>/dev/null || true + fi + return 0 +} + +fm_trace_context_session_effective() { # + local effective_file=$1 current_lock recorded_lock='' value='' extra='' + current_lock=$(fm_trace_context_session_lock "$effective_file") || { + printf '%s' off + return 0 + } + if [ -f "$effective_file" ] && [ ! -L "$effective_file" ]; then + IFS=' ' read -r recorded_lock value extra < "$effective_file" 2>/dev/null || true + fi + if [ "$recorded_lock" = "$current_lock" ] && [ "$value" = on ] && [ -z "$extra" ]; then + printf '%s' on + else + printf '%s' off + fi +} + +# Echo any traceparent already recorded in , else nothing. Used for +# the recovery path so a relaunch reuses the first spawn's identity. +fm_trace_context_recorded() { # + local meta=$1 line + [ -f "$meta" ] || return 0 + line=$(grep '^traceparent=' "$meta" 2>/dev/null | head -n1) || return 0 + printf '%s' "${line#traceparent=}" +} + +# Mint a traceparent, adopting a valid parent's trace id and flags when present, +# otherwise starting a fresh root. Echo nothing and return 1 on entropy or +# validation failure so the caller can omit telemetry. +fm_trace_context_mint() { # + local inherited=$1 trace flags span tp + if fm_trace_context_valid "$inherited"; then + trace=${inherited:3:32} + flags=${inherited:53:2} + else + trace=$(fm_trace_context_hex 16) || return 1 + flags=01 + fi + span=$(fm_trace_context_hex 8) || return 1 + tp="00-$trace-$span-$flags" + fm_trace_context_valid "$tp" || return 1 + printf '%s' "$tp" +} + +# Public entry point. Echo the single carrier to inject and record, or nothing. +# Always returns 0 so a spawn is never aborted by a telemetry decision. +fm_trace_context_resolve() { # [] + local config_dir=$1 meta=$2 inherited=${3-${TRACEPARENT:-}} existing + fm_trace_context_enabled "$config_dir" || return 0 + existing=$(fm_trace_context_recorded "$meta") + if fm_trace_context_valid "$existing"; then + printf '%s' "$existing" + return 0 + fi + fm_trace_context_mint "$inherited" || return 0 +} diff --git a/docs/configuration.md b/docs/configuration.md index b226ec6888..0855e63b84 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,6 +116,14 @@ An absent file means `auto`, i.e. default-on on macOS: the alarm exists precisel A missing or failing channel logs and falls through to the next, never crashing the daemon. See [`wedge-alarm.md`](wedge-alarm.md) for the current channel reference, [`verification/supervision.md`](verification/supervision.md#wedge-alarm-channels) for active evidence, and [`examples/wedge-alarm`](examples/wedge-alarm) for a copyable config. +## Trace context propagation (config/trace-context / FM_TRACE_CONTEXT) + +The optional local, gitignored `config/trace-context` presence flag enables default-off native W3C trace-context propagation. +`FM_TRACE_CONTEXT` overrides the file: `1`/`on`/`true`/`yes` enables, any other non-empty value disables, and unset or empty defers to the file. +Each locked home session resolves those inputs once, and all spawns from that home use the frozen decision until a new session starts. +When launching a Secondmate, the primary copies the presence flag into its home and passes the primary session's frozen decision as a non-empty `FM_TRACE_CONTEXT=on|off` override for the Secondmate's own session start. +See [`trace-context.md`](trace-context.md) for carrier semantics, supported routes, the manual fleet-restart requirement, the session boundary, and safety limits; `bin/fm-trace-context-lib.sh`'s header owns the exact mechanics, and [`verification/trace-context.md`](verification/trace-context.md) records repeatable evidence. + ## Gate defaults (.no-mistakes.yaml) The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and pins `commands.lint` to `bin/fm-lint.sh` so local lint matches CI. @@ -299,7 +307,7 @@ When a running home advances and its loaded instruction surface (`AGENTS.md`, `b If that send fails, bootstrap keeps an idempotent retry marker and emits `NUDGE_SECONDMATES:` with the failure reason. The same bootstrap run emits `SECONDMATE_LIVENESS:` only when a registered secondmate is skipped or its relaunch fails; already-live and successfully relaunched secondmates are handled silently. For a mid-session inherited local-material edit where tracked-file sync is not needed, run `bin/fm-config-push.sh`. -It uses the same live secondmate discovery and propagation helper as bootstrap, prints each live home's `crew-dispatch.json`, `crew-harness`, `backlog-backend`, `backend`, `herdr-presentation-spaces`, `startup-memory-budget`, and `data/captain-shared.md` result as `pushed`, `unchanged`, `skipped`, or `error`, and exits non-zero for real propagation errors or config-reread send failures. +It uses the same live secondmate discovery and propagation helper as bootstrap, prints each live home's `crew-dispatch.json`, `crew-harness`, `backlog-backend`, `backend`, `herdr-presentation-spaces`, `startup-memory-budget`, `trace-context`, and `data/captain-shared.md` result as `pushed`, `unchanged`, `skipped`, or `error`, and exits non-zero for real propagation errors or config-reread send failures. When an allowlisted config item changes for an already-running home, it sends the literal-content reread pointer described in [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md); unchanged allowlisted config sends no pointer unless a previous delivery is pending. The locked bootstrap inheritance pass uses the same per-home changed-set and reread path for already-running homes; see `secondmate-provisioning` for the single contract owner. That live discovery starts from `state/*.meta` records with `kind=secondmate`; `data/secondmates.md` only backfills `home=` for older or incomplete meta records. @@ -392,6 +400,7 @@ FM_PROJECTS_OVERRIDE= # alternate projects dir, mainly for tests FM_CONFIG_OVERRIDE= # alternate config dir, mainly for tests FM_PROC_ROOT_OVERRIDE= # alternate /proc root for the Linux process-identity read in fm-wake-lib.sh, mainly for tests FM_BACKEND= # optional runtime backend override for new spawns; tmux/herdr/zellij/orca/cmux support ship/scout spawns, codex-app is not accepted +FM_TRACE_CONTEXT= # optional trace-context override; see "Trace context propagation" HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md) FM_BACKEND_HERDR_COMPOSER_LINES=20 # herdr-only: tail lines scanned by composer-state guard/fallback paths; idle-baseline submit confirmation uses agent-state FM_BACKEND_HERDR_IDLE_RE='^Type a message\.\.\.$' # herdr-only: empty-composer placeholder regex after shared ghost extraction plus border and prompt stripping diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 8f70cd346c..5caeb8bf4f 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -100,6 +100,10 @@ { "source": "docs/codex-app-backend.md", "target": "docs/verification/runtime-backends.md" + }, + { + "source": "docs/trace-context.md", + "target": "docs/verification/trace-context.md" } ], "surfaces": [ @@ -299,6 +303,10 @@ "path": "docs/tmux-backend.md", "audience": "operator-current" }, + { + "path": "docs/trace-context.md", + "audience": "maintainer-architecture" + }, { "path": "docs/turnend-guard.md", "audience": "operator-current" @@ -315,6 +323,10 @@ "path": "docs/verification/supervision.md", "audience": "maintainer-verification" }, + { + "path": "docs/verification/trace-context.md", + "audience": "maintainer-verification" + }, { "path": "docs/watcher-continuity.md", "audience": "operator-current" diff --git a/docs/trace-context.md b/docs/trace-context.md new file mode 100644 index 0000000000..6985f69619 --- /dev/null +++ b/docs/trace-context.md @@ -0,0 +1,102 @@ +# Native W3C trace-context propagation + +Firstmate can propagate a W3C [`traceparent`](https://www.w3.org/TR/trace-context/) to every agent it spawns so an external observer can correlate a worker, a Secondmate, and their nested children into one trace. +The capability is default-off, source-owned, vendor-neutral, and deliberately narrow. +This document is the rationale and current-behavior guide; `docs/configuration.md` owns the configuration schema, `bin/fm-trace-context-lib.sh`'s header owns the exact mechanics, and [`verification/trace-context.md`](verification/trace-context.md) records the repeatable test evidence. + +## Why this is a source change at all + +Firstmate's durable operational artifacts already let a downstream observer derive logical task identity and lifecycle. +The source capability an observer cannot reconstruct after launch is a task-scoped trace id delivered in the agent's environment before launch and recorded under the same identity in task metadata. +This feature adds only that carrier seam. + +## What it does + +When enabled, for each spawn Firstmate resolves one W3C `traceparent` carrier for the task - minted on the first spawn, inherited as a child (fresh span, same trace) for a nested spawn, and reused verbatim from the meta on relaunch - and: + +- forms it as `00-<32 hex trace id>-<16 hex span id>-<2 hex flags>`, with random ids for a new root; +- injects it into the agent's pane shell as the `TRACEPARENT` environment variable immediately before launch, through the same `spawn_send_text_line` channel that already ships `GOTMPDIR`; and +- records the identical value as `traceparent=` in `state/.meta`. + +`TRACEPARENT` as an environment variable is a Firstmate convention carrying a W3C-formatted value: W3C Trace Context standardizes the `traceparent` HTTP header, not an env var, and OpenTelemetry SDKs do not read it from the environment automatically, so a downstream observer must explicitly read this env value or the `traceparent=` meta field. +This feature parents no SDK span by itself. + +Because the injected carrier and the recorded carrier are the same string, an observer that reads the metadata reconstructs exactly the identity the child received. +The injection sits at the unconditional pre-launch export site, so it covers ship, scout, and Secondmate spawns and is identical across every harness (`claude`, `codex`, `opencode`, `pi`, `grok`, `kimi`) - the same coverage `GOTMPDIR` already has, with no `launch_template()` change. +Ship and scout spawns reach that site on every spawn backend (`tmux`, `herdr`, `zellij`, `orca`, `cmux`); a Secondmate reaches it on every backend that accepts a Secondmate spawn (`tmux`, `herdr`, `zellij`), because `bin/fm-spawn.sh` rejects a Secondmate on `orca` and `cmux`. + +## Root, child, and recovery semantics + +The point of these rules is to never mint an unrelated root by accident. + +- **Root** - a spawn with no valid inherited `TRACEPARENT` mints a fresh trace id, a fresh span id, and sampled flags (`01`). + This begins a new trace, one per top-level task. +- **Child** - a spawn whose own Firstmate process already holds a valid `TRACEPARENT` in its environment keeps that trace id and its flags while minting a fresh span id. + This is what makes a nested Firstmate -> Secondmate -> worker chain share one trace: a Secondmate is itself a spawned agent, so when it was launched with trace context enabled it received a `TRACEPARENT`, and it passes that trace on to its own workers. +- **Recovery** - a valid `traceparent=` already recorded in the task's meta is reused verbatim, so a relaunched or recovered task keeps one stable identity across restarts rather than starting a second trace. + +A malformed or all-zero inherited value is treated as absent, so garbage never propagates and the spawn roots a clean trace instead (it is not omitted). +Disabling propagation is an intentional trace boundary: a disabled home injects no carrier even when its process inherited a valid `TRACEPARENT` or the task meta already contains a valid `traceparent=`. +A disabled relaunch regenerates the task meta without `traceparent=`, so a later enabled relaunch derives a new child or root carrier instead of resuming the identity from before the boundary. + +### Enablement is home-session-scoped + +Each locked `bin/fm-session-start.sh` run resolves that home's `config/trace-context` plus `FM_TRACE_CONTEXT` exactly once into session-scoped effective state. +The decision is atomically published through a same-directory temporary file and bound to the current session lock, so a failed publication cannot reactivate a stale `on` record from an earlier session. +Every spawn from that home reads only the frozen `on` or `off` decision. +Later config or environment edits are ignored until that home starts a new session. +Missing, stale, unreadable, invalid, or unsuccessfully published effective state defaults safely to `off`. + +When the primary launches a Secondmate, it propagates `config/trace-context` into the Secondmate home and passes the primary session's frozen decision as a non-empty `FM_TRACE_CONTEXT=on|off` launch override. +The Secondmate resolves that inherited override when its own home session starts. +A Secondmate launched while enabled receives the carrier resolved by the primary, continues the primary trace with a fresh span id, and keeps spawning enabled workers from that trace. +A Secondmate launched while disabled keeps its workers untraced even if `config/trace-context` is present in its home. +When enabled, a relaunch reuses the task's valid recorded carrier; only a task without one derives a child carrier from the primary's current context. +A duplicate Secondmate launch is refused before trace-context inheritance, so duplicate-launch preflight does not mutate the Secondmate home. + +Changing the setting across the whole fleet requires a manual full fleet restart so every home starts a new session and freezes the new decision. +Firstmate does not monitor setting drift, detect mismatches, refuse launches, or automatically stop or restart any home. + +## Sampling + +A new root sets the W3C trace flags to `01` (sampled). +This is a deliberate, source-owned choice: + +- The capability is **opt-in** and default-off, so a home that enables it is asking for its spawns to be traced; an unsampled (`00`) root would produce a trace id that most downstream parent-based samplers drop, yielding nothing for the operator who opted in. +- **Inherited flags are preserved unchanged.** + A child adopts the parent's flags verbatim, so an unsampled (`00`) parent keeps its whole subtree unsampled and an upstream sampler's decision is honored end to end. + Firstmate chooses the flag only when it mints a *root*; it never overrides an inherited decision. +- **Cost and privacy consequence.** + `01` records a sampling *decision*, and a conforming downstream parent-based sampler will honor it - but it does not by itself guarantee that any collector stores a span, and Firstmate emits no spans of its own; it only sets the flag on the carrier. + An operator who enables the capability and points sampling-respecting instrumentation at it should expect on the order of one trace per top-level task plus its nested spans to be recorded, at whatever cardinality and retention that instrumentation is configured for. + An operator who wants unsampled roots or head-sampling owns that downstream or via a later, explicitly-scoped option; Firstmate does not embed a sampler. + +## Safety + +- **Default-off.** + With no `config/trace-context` and no `FM_TRACE_CONTEXT`, nothing is injected and no `traceparent=` line is written, so the generated meta and the launch environment are unchanged. + A locked session start makes the one config-file check, and each spawn sources one extra library and reads the frozen effective-state file, so the process is not literally byte-for-byte identical, but nothing an agent, an observer, or the task meta can see differs. +- **What is and is not exposed.** + A Firstmate-*minted* root uses a random id and reads no prompt, path, task prose, credential, or arbitrary environment key, so Firstmate never *originates* sensitive data in the carrier. + An *inherited* `TRACEPARENT` is opaque caller-controlled data: its 16-byte trace id and 8-byte span id are up to 24 bytes (48 hex chars) that Firstmate passes through after syntax validation without interpreting, so whoever set `TRACEPARENT` in Firstmate's environment (a trusted local operator or observer) controls those bytes. + Exposure is bounded to that fixed-width carrier - it cannot carry a `tracestate`, an `OTEL_*` credential variable, or any arbitrary environment key, and there is no configurable or arbitrary command (only the fixed local `od`/`tr` for entropy) - but the id bytes are not 'structurally impossible to carry data'. +- **Fail-independent.** + Minting is a small local entropy pipeline: it reads a few bytes from `/dev/urandom` through the fixed local `od` and `tr` (resolved from PATH). + There is no configured provider command, no network, and no watchdog. + The normal cost is small, but `od`/`tr` are external processes, so there is no hard latency guarantee - this is not a guaranteed-negligible bound. + Any entropy or self-validation failure that returns omits the carrier for that spawn without aborting source work; a malformed or all-zero inherited value is treated as absent and roots a fresh trace (it is not an omission). + If the pre-launch carrier export fails, Firstmate omits the `traceparent=` metadata claim and still launches the task. + If the backend reports that failed trace input could not be cleared, Firstmate refuses to append the launch command rather than risk launching with an unknown partial carrier. + If recording the carrier fails after export, Firstmate unsets `TRACEPARENT` in the launch command and still launches the task, so the child never receives an identity absent from its metadata. +- **Metadata-only.** + The value lives in the ephemeral pane shell and in `state/.meta`; teardown removes state as before, so there is no new durable surface and no schema migration. + +## Relationship to OpenTelemetry and later increments + +Firstmate learns nothing about OpenTelemetry, any exporter, collector, storage, or UI. +It emits a standard W3C carrier and records the same identity; a downstream observer owns everything else and discovers active propagation from the home session's frozen decision or the `traceparent=` field. +Native lifecycle-event emission, extra stable IDs, intake metadata, and any embedded OTLP are deliberately deferred until a running observer demonstrates a concrete fidelity gap that the derived artifacts cannot cover. + +## Verification + +Repeatable test evidence - the unit and spawn-path suites with exact commands and output - lives in [`verification/trace-context.md`](verification/trace-context.md). diff --git a/docs/verification/trace-context.md b/docs/verification/trace-context.md new file mode 100644 index 0000000000..04d85599ac --- /dev/null +++ b/docs/verification/trace-context.md @@ -0,0 +1,26 @@ +# Trace-context propagation verification + +Repeatable evidence for the default-off native W3C trace-context capability. +Current behavior and rationale are owned by [`../trace-context.md`](../trace-context.md) and the configuration schema by [`../configuration.md`](../configuration.md) ("Trace context propagation"); this page records evidence only. + +Date: 2026-07-26. +Shell: GNU bash 3.2.57 (macOS). +Comparison base: `main` at `a5fe1bc`. + +The colocated unit suite `tests/fm-trace-context-lib.test.sh` (30 assertions) exercises validation (valid accepted; malformed, wrong-length, uppercase, all-zero, `ff` version, and shell-metacharacter values rejected), root and child minting, sampled/unsampled flag inheritance, malformed and all-zero inheritance falling back to a root, the recovery reuse path, default-off omission, the enable precedence of `FM_TRACE_CONTEXT` over `config/trace-context` with unset or empty deferring to the file, normalized home-session state, atomic replacement of a read-only prior record, stale-session rejection after failed publication, missing or invalid state defaulting off, the Secondmate home-session boundary with later file state, forced entropy failure omitting safely, the minted-root fixed-shape check, and the opaque-inheritance trust-boundary assertion (an inherited id passes through verbatim as caller-controlled data). + +The spawn-path integration suite `tests/fm-trace-context-spawn.test.sh` (11 assertions), hermetic against an ambient `FM_TRACE_CONTEXT`, drives `bin/fm-spawn.sh` end to end with a fake tmux pane and a real isolated git worktree: enabled, one resolved carrier is recorded as `traceparent=` in the meta only after the identical `TRACEPARENT` export is sent before the launch literal; disabled, neither is written nor sent (only `GOTMPDIR` is); a failed carrier delivery leaves no `traceparent=` claim while the source task still launches; an unsafe delivery whose partial input cannot be cleared stops before appending the launch command; a failed metadata append removes the carrier from the launched task without aborting it; duplicate Secondmate preflight leaves inherited trace configuration unchanged; a relaunch reuses the recorded carrier verbatim; and spawns ignore later config and environment edits in favor of the frozen home-session decision. +Two further assertions drive a genuine two-level primary -> Secondmate -> worker chain, running `bin/fm-spawn.sh` twice with the exact environment the primary injects into the Secondmate, and prove the primary's effective override governs the nested worker both ways: env-on with no config file keeps the nested worker in the primary's trace, and env-off with the file present keeps the nested worker disabled even though the `config/trace-context` file was copied into the Secondmate home. +A final assertion drives the file-decided path (`FM_TRACE_CONTEXT` unset) and proves the Secondmate's recorded/injected carrier and its delivered `FM_TRACE_CONTEXT=on|off` snapshot are always derived from one frozen decision, so a carrier is never paired with the opposite enable state. +The suite touches no real harness or live fleet. +`tests/fm-session-start.test.sh` additionally proves only a lock-owning session start writes the effective state and a lock-refused read-only start leaves it unchanged. + +```console +$ bash tests/fm-trace-context-lib.test.sh | tail -1 +# fm-trace-context-lib.test.sh: all assertions passed +$ bash tests/fm-trace-context-spawn.test.sh | tail -1 +# all fm-trace-context-spawn tests passed +``` + +Run both trace-context suites from the repo root; each prints one `ok - ...` per assertion. +A single live-backend end-to-end check - a real spawn confirming the pane received the `TRACEPARENT` export before the launch line, with nothing left after teardown - is a bounded manual step, deferred here because a live agent spawn disrupts a running fleet. diff --git a/tests/fm-backend-cmux.test.sh b/tests/fm-backend-cmux.test.sh index 152f78eb7c..046504a14c 100755 --- a/tests/fm-backend-cmux.test.sh +++ b/tests/fm-backend-cmux.test.sh @@ -637,6 +637,47 @@ test_send_literal_uses_separator_for_option_shaped_text() { pass "fm_backend_cmux_send_literal: calls send with an explicit workspace/surface and a -- separator" } +test_send_text_line_clears_partial_input_when_enter_fails() { + local dir fb status log + dir="$TMP_ROOT/sendline-enter-failure"; mkdir -p "$dir/responses" + cmux_panes_response "$dir" 1 "bbbbbbbb-1111-1111-1111-111111111111" + cmux_panes_response "$dir" 3 "bbbbbbbb-1111-1111-1111-111111111111" + printf '1\n' > "$dir/responses/4.exit" + cmux_panes_response "$dir" 5 "bbbbbbbb-1111-1111-1111-111111111111" + fb=$(make_cmux_fakebin "$dir") + + PATH="$fb:$PATH" FM_CMUX_LOG="$dir/log" FM_CMUX_RESPONSES="$dir/responses" \ + bash -c '. "$0/bin/backends/cmux.sh"; fm_backend_cmux_send_text_line "aaaaaaaa-0000-0000-0000-000000000000:bbbbbbbb-1111-1111-1111-111111111111" "export TRACEPARENT=carrier"' "$ROOT" + status=$? + [ "$status" -ne 0 ] || fail "send_text_line should report a failed Enter" + log=$(cat "$dir/log") + assert_contains "$log" $'\x1f''send'$'\x1f''--workspace'$'\x1f''aaaaaaaa-0000-0000-0000-000000000000'$'\x1f''--surface'$'\x1f''bbbbbbbb-1111-1111-1111-111111111111'$'\x1f''--'$'\x1f''export TRACEPARENT=carrier' \ + "send_text_line did not send the trace export before the simulated Enter failure" + assert_contains "$log" $'\x1f''send-key'$'\x1f''--workspace'$'\x1f''aaaaaaaa-0000-0000-0000-000000000000'$'\x1f''--surface'$'\x1f''bbbbbbbb-1111-1111-1111-111111111111'$'\x1f''ctrl-c' \ + "send_text_line did not clear the partial input after Enter failed" + pass "fm_backend_cmux_send_text_line: clears partial input when Enter fails" +} + +test_send_text_line_reports_unsafe_input_when_cleanup_fails() { + local dir fb status log + dir="$TMP_ROOT/sendline-cleanup-failure"; mkdir -p "$dir/responses" + cmux_panes_response "$dir" 1 "bbbbbbbb-1111-1111-1111-111111111111" + cmux_panes_response "$dir" 3 "bbbbbbbb-1111-1111-1111-111111111111" + printf '1\n' > "$dir/responses/4.exit" + cmux_panes_response "$dir" 5 "bbbbbbbb-1111-1111-1111-111111111111" + printf '1\n' > "$dir/responses/6.exit" + fb=$(make_cmux_fakebin "$dir") + + PATH="$fb:$PATH" FM_CMUX_LOG="$dir/log" FM_CMUX_RESPONSES="$dir/responses" \ + bash -c '. "$0/bin/backends/cmux.sh"; fm_backend_cmux_send_text_line "aaaaaaaa-0000-0000-0000-000000000000:bbbbbbbb-1111-1111-1111-111111111111" "export TRACEPARENT=carrier"' "$ROOT" + status=$? + expect_code 2 "$status" "send_text_line should distinguish uncleared input" + log=$(cat "$dir/log") + assert_contains "$log" $'\x1f''send-key'$'\x1f''--workspace'$'\x1f''aaaaaaaa-0000-0000-0000-000000000000'$'\x1f''--surface'$'\x1f''bbbbbbbb-1111-1111-1111-111111111111'$'\x1f''ctrl-c' \ + "send_text_line did not attempt cleanup after Enter failed" + pass "fm_backend_cmux_send_text_line: reports unsafe input when cleanup also fails" +} + # --- current_path: pwd-marker-probe (zellij-shape) --------------------------- test_current_path_probes_with_marker() { @@ -1042,6 +1083,8 @@ test_capture_fails_when_target_not_ready test_send_key_normalizes_and_targets test_send_key_recovers_stale_target_by_label test_send_literal_uses_separator_for_option_shaped_text +test_send_text_line_clears_partial_input_when_enter_fails +test_send_text_line_reports_unsafe_input_when_cleanup_fails test_current_path_probes_with_marker test_composer_state_bare_prompt_is_empty test_composer_state_ghost_placeholder_is_empty diff --git a/tests/fm-backend-zellij.test.sh b/tests/fm-backend-zellij.test.sh index ae4be257bd..5039379f8b 100755 --- a/tests/fm-backend-zellij.test.sh +++ b/tests/fm-backend-zellij.test.sh @@ -616,6 +616,48 @@ test_send_literal_uses_paste_separator_for_option_shaped_text() { pass "fm_backend_zellij_send_literal: calls paste with an explicit pane id and a -- separator" } +test_send_text_line_clears_partial_input_when_enter_fails() { + local dir fb status log + dir="$TMP_ROOT/sendline-enter-failure"; mkdir -p "$dir/responses" + zellij_pane_response "$dir" 1 7 3 + zellij_pane_response "$dir" 3 7 3 + printf '1\n' > "$dir/responses/4.exit" + zellij_pane_response "$dir" 5 7 3 + fb=$(make_zellij_fakebin "$dir") + + PATH="$fb:$PATH" FM_ZELLIJ_LOG="$dir/log" FM_ZELLIJ_RESPONSES="$dir/responses" \ + FM_ZELLIJ_SESSION_LIST="firstmate" bash -c \ + '. "$0/bin/backends/zellij.sh"; fm_backend_zellij_send_text_line "firstmate:7" "export TRACEPARENT=carrier"' "$ROOT" + status=$? + [ "$status" -ne 0 ] || fail "send_text_line should report a failed Enter" + log=$(cat "$dir/log") + assert_contains "$log" $'\x1f''paste'$'\x1f''--pane-id'$'\x1f''7'$'\x1f''--'$'\x1f''export TRACEPARENT=carrier' \ + "send_text_line did not paste the trace export before the simulated Enter failure" + zellij_assert_call_order "$dir/log" $'\x1f''Enter' $'\x1f''Ctrl c' \ + "send_text_line did not clear the partial input after Enter failed" + pass "fm_backend_zellij_send_text_line: clears partial input when Enter fails" +} + +test_send_text_line_reports_unsafe_input_when_cleanup_fails() { + local dir fb status + dir="$TMP_ROOT/sendline-cleanup-failure"; mkdir -p "$dir/responses" + zellij_pane_response "$dir" 1 7 3 + zellij_pane_response "$dir" 3 7 3 + printf '1\n' > "$dir/responses/4.exit" + zellij_pane_response "$dir" 5 7 3 + printf '1\n' > "$dir/responses/6.exit" + fb=$(make_zellij_fakebin "$dir") + + PATH="$fb:$PATH" FM_ZELLIJ_LOG="$dir/log" FM_ZELLIJ_RESPONSES="$dir/responses" \ + FM_ZELLIJ_SESSION_LIST="firstmate" bash -c \ + '. "$0/bin/backends/zellij.sh"; fm_backend_zellij_send_text_line "firstmate:7" "export TRACEPARENT=carrier"' "$ROOT" + status=$? + expect_code 2 "$status" "send_text_line should distinguish uncleared input" + zellij_assert_call_order "$dir/log" $'\x1f''Enter' $'\x1f''Ctrl c' \ + "send_text_line did not attempt cleanup after Enter failed" + pass "fm_backend_zellij_send_text_line: reports unsafe input when cleanup also fails" +} + test_expected_label_allows_matching_task_tab() { local dir fb dir="$TMP_ROOT/label-match"; mkdir -p "$dir/responses" @@ -1056,6 +1098,8 @@ test_capture_fails_when_pane_absent test_capture_fails_when_session_absent test_send_key_normalizes_and_targets_pane test_send_literal_uses_paste_separator_for_option_shaped_text +test_send_text_line_clears_partial_input_when_enter_fails +test_send_text_line_reports_unsafe_input_when_cleanup_fails test_expected_label_allows_matching_task_tab test_expected_label_rejects_reused_pane_id test_current_path_probes_with_marker_and_ignores_prompt_paths diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 446dcd93a1..8271f75023 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -14,11 +14,12 @@ # explicit per-spawn harness arg still wins. # B) Inheritance. The primary pushes a declared, extensible set of LOCAL # (gitignored) config items - config/crew-dispatch.json, config/crew-harness, -# config/backlog-backend, config/backend, config/herdr-presentation-spaces, and -# config/startup-memory-budget - +# config/backlog-backend, config/backend, config/herdr-presentation-spaces, +# config/startup-memory-budget, and config/trace-context - # down into each secondmate home's config/, so the secondmate's OWN crewmates, -# dispatch profiles, backlog backend, runtime-backend default, and Herdr -# presentation opt-in inherit the primary's settings. It is primary-authoritative +# dispatch profiles, backlog backend, runtime-backend default, Herdr +# presentation opt-in, and trace context inherit the primary's settings. +# It is primary-authoritative # (re-pushed at secondmate spawn, on the bootstrap secondmate sweep, and by # config push). # config/secondmate-harness is deliberately NOT inherited (secondmates do @@ -257,6 +258,7 @@ test_propagate_lib() { printf 'manual\n' > "$src/backlog-backend" printf 'tmux\n' > "$src/backend" : > "$src/herdr-presentation-spaces" + : > "$src/trace-context" stdout="$d/clean-copy.out" stderr="$d/clean-copy.err" propagate_inheritable_config "$src" "$dest" >"$stdout" 2>"$stderr" || fail "propagate returned non-zero" @@ -270,6 +272,7 @@ test_propagate_lib() { printf 'herdr\n' > "$dest/backend" propagate_inheritable_config "$src" "$dest" [ "$(cat "$dest/backend")" = tmux ] || fail "primary backend did not overwrite a divergent destination" + [ -f "$dest/trace-context" ] || fail "trace-context not propagated by the default inheritable set" # 2. idempotent: an unchanged re-run does not churn the mtime m1=$(date -r "$dest/crew-harness" +%s 2>/dev/null || stat -c %Y "$dest/crew-harness") @@ -306,13 +309,14 @@ test_propagate_lib() { # 4. removing the source mirrors absence downstream (primary-authoritative) printf 'herdr\n' > "$dest/backend" rm -f "$src/crew-dispatch.json" "$src/crew-harness" "$src/backlog-backend" \ - "$src/backend" "$src/herdr-presentation-spaces" + "$src/backend" "$src/herdr-presentation-spaces" "$src/trace-context" propagate_inheritable_config "$src" "$dest" [ -e "$dest/crew-dispatch.json" ] && fail "dispatch profile absence not mirrored downstream" [ -e "$dest/crew-harness" ] && fail "absence not mirrored downstream" [ -e "$dest/backlog-backend" ] && fail "backlog-backend absence not mirrored downstream" [ -e "$dest/backend" ] && fail "backend absence not mirrored downstream" [ -e "$dest/herdr-presentation-spaces" ] && fail "herdr-presentation-spaces absence not mirrored downstream" + [ -e "$dest/trace-context" ] && fail "trace-context absence not mirrored downstream" rm -f "$dest/crew-harness" ln -s "$d/missing-target" "$dest/crew-harness" @@ -1081,6 +1085,7 @@ test_bootstrap_sweep_propagates_and_reconverges() { printf 'codex\n' > "$w/home/config/crew-harness" printf 'manual\n' > "$w/home/config/backlog-backend" printf 'tmux\n' > "$w/home/config/backend" + : > "$w/home/config/trace-context" printf 'grok\n' > "$w/home/config/secondmate-harness" run_bootstrap "$w" >/dev/null [ "$(cat "$w/sm/config/crew-harness" 2>/dev/null)" = codex ] \ @@ -1091,6 +1096,8 @@ test_bootstrap_sweep_propagates_and_reconverges() { || fail "sweep: backlog-backend not pushed into the live home" [ "$(cat "$w/sm/config/backend" 2>/dev/null)" = tmux ] \ || fail "sweep: backend not pushed into the live home" + [ ! -e "$w/sm/config/trace-context" ] \ + || fail "sweep: trace-context changed a legacy live home before relaunch" [ -e "$w/sm/config/secondmate-harness" ] \ && fail "sweep: secondmate-harness was inherited (must not be)" @@ -1299,6 +1306,7 @@ test_config_push_propagates_reports_without_ff_or_nudge() { printf 'codex\n' > "$w/home/config/crew-harness" printf 'manual\n' > "$w/home/config/backlog-backend" printf 'tmux\n' > "$w/home/config/backend" + : > "$w/home/config/trace-context" err="$w/config-push-basic.err" log="$w/config-push-basic.tmux.log" out=$(run_config_push "$w" "$log" 2>"$err"); status=$? @@ -1316,6 +1324,10 @@ test_config_push_propagates_reports_without_ff_or_nudge() { "config push did not report backlog-backend as pushed" assert_contains "$out" "backend: pushed" \ "config push did not report backend as pushed" + assert_contains "$out" "trace-context: unchanged" \ + "live config push must report trace-context as session-scoped and unchanged" + [ ! -e "$w/sm/config/trace-context" ] \ + || fail "live config push retroactively enabled trace context in a legacy secondmate home" assert_contains "$out" "config-reread: sent" \ "config push with changed config must send a literal reread instruction" assert_not_contains "$out" "NUDGE_SECONDMATES" \ @@ -1341,6 +1353,8 @@ test_config_push_propagates_reports_without_ff_or_nudge() { "idempotent config push did not report backlog-backend as unchanged" assert_contains "$out2" "backend: unchanged" \ "idempotent config push did not report backend as unchanged" + assert_contains "$out2" "trace-context: unchanged" \ + "idempotent config push did not preserve session-scoped trace context" assert_not_contains "$out2" "config-reread: sent" \ "unchanged config must not send a reread message" [ ! -s "$log" ] || fail "unchanged config push still invoked tmux send: $(cat "$log")" diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 9bcf04eddb..799770e9ce 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -678,6 +678,39 @@ EOF pass "session start stays read-only when lock ownership cannot be published" } +test_trace_context_effective_state_is_frozen_after_lock() { + local rec root home fakebin out frozen + rec=$(new_world trace-context-session-state) + IFS='|' read -r root home fakebin < "$home/config/trace-context" + + FM_TRACE_CONTEXT=off run_session_start "$home" "$root" "$fakebin:$BASE_PATH" >/dev/null + [ "$(awk '{print $2}' "$home/state/.trace-context-effective")" = off ] \ + || fail "session start must freeze an env-off override over a present config flag" + + rm "$home/config/trace-context" + FM_TRACE_CONTEXT=on run_session_start "$home" "$root" "$fakebin:$BASE_PATH" >/dev/null + [ "$(awk '{print $2}' "$home/state/.trace-context-effective")" = on ] \ + || fail "a new session start must freeze an env-on override over an absent config flag" + frozen=$(cat "$home/state/.trace-context-effective") + + sleep 300 & + holder_pid=$! + printf '%s\n' "$holder_pid" > "$home/state/.lock" + out=$(FM_TRACE_CONTEXT=off run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + kill "$holder_pid" 2>/dev/null || true + wait "$holder_pid" 2>/dev/null || true + assert_contains "$out" "READ-ONLY SESSION" "trace-context refusal fixture did not enter read-only mode" + [ "$(cat "$home/state/.trace-context-effective")" = "$frozen" ] \ + || fail "a lock-refused session must not mutate the frozen trace-context state" + + pass "locked session start freezes trace context and lock refusal leaves it unchanged" +} + test_session_lock_concurrent_single_winner() { local rec root home fakebin ready completed winners pids i pid count rec=$(new_world lock-concurrency) @@ -723,7 +756,7 @@ SH i=1 while [ "$i" -le 40 ]; do ( - harness_pid=$BASHPID + harness_pid=$(sh -c 'printf "%s\n" "$PPID"') : > "$home/state/harness-$harness_pid" : > "$ready/$i" while [ "$(find "$ready" -type f | wc -l | tr -d ' ')" -lt 40 ]; do @@ -1388,6 +1421,7 @@ EOF test_context_digest_absent_empty_present test_lock_refusal_read_only_path test_lock_write_failure_read_only_path +test_trace_context_effective_state_is_frozen_after_lock test_session_lock_concurrent_single_winner test_output_ordering_diagnostics_lead test_herdr_backend_diagnostics_follow_real_session_start @@ -1413,3 +1447,5 @@ test_pi_diagnostic_rejects_stale_loaded_marker test_pi_diagnostic_accepts_prelock_loaded_marker test_pi_diagnostic_rejects_missing_turnend_guard_marker test_pi_diagnostic_rejects_previous_session_loaded_marker + +echo "# fm-session-start.test.sh: all assertions passed" diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index cfb1578fc4..b680a87b72 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -432,22 +432,25 @@ exit 1 SH cat >"$repo/$a" <<'SH' #!/usr/bin/env bash -sleep 0.5 -touch "$SCHED_EVIDENCE/slow-done" -echo "ok - slow fixture" +attempts=0 +while [ ! -e "$SCHED_EVIDENCE/replacement-started" ]; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 200 ]; then + echo "not ok - replacement fixture never started" + exit 1 + fi + sleep 0.05 +done +echo "ok - blocked fixture released" SH cat >"$repo/$b" <<'SH' #!/usr/bin/env bash -sleep 0.05 echo "ok - fast fixture" SH cat >"$repo/$c" <<'SH' #!/usr/bin/env bash -if [ -e "$SCHED_EVIDENCE/slow-done" ]; then - echo "not ok - scheduler waited for oldest worker" - exit 1 -fi -echo "ok - replacement fixture started before slow fixture finished" +touch "$SCHED_EVIDENCE/replacement-started" +echo "ok - replacement fixture released blocked fixture" SH chmod +x "$runner" "$repo/$a" "$repo/$b" "$repo/$c" "$fake_bin/stat" set +e @@ -491,7 +494,6 @@ echo "not ok - deliberate proven-set fail" exit 1 SH chmod +x "$repo/$b" - rm -f "$evidence/slow-done" set +e SCHED_EVIDENCE="$evidence" "$runner" --jobs 2 "$a" "$b" >"$tmp/out4" 2>"$tmp/err4" rc=$? diff --git a/tests/fm-trace-context-lib.test.sh b/tests/fm-trace-context-lib.test.sh new file mode 100755 index 0000000000..50b46f8063 --- /dev/null +++ b/tests/fm-trace-context-lib.test.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# tests/fm-trace-context-lib.test.sh - unit tests for the native, default-off +# W3C trace-context library (bin/fm-trace-context-lib.sh) plus structural checks +# that bin/fm-spawn.sh wires it in at the pre-launch injection seam and that the +# capability is inherited into secondmate homes. Pure functions, no backend and +# no live spawn required. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# shellcheck source=/dev/null +. "$ROOT/bin/fm-trace-context-lib.sh" + +VALID='00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' + +# --- strict W3C validation --------------------------------------------------- + +fm_trace_context_valid "$VALID" || fail "a conformant traceparent must validate" +pass "fm_trace_context_valid accepts a conformant W3C traceparent" + +for bad in \ + '00-00000000000000000000000000000000-00f067aa0ba902b7-01' \ + '00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01' \ + 'ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' \ + '00-4BF92F3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01' \ + '00-4bf92f3577b34da6a3ce929d0e0e473-00f067aa0ba902b7-01' \ + '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7' \ + '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01; rm -rf /' \ + '' ; do + if fm_trace_context_valid "$bad"; then + fail "invalid traceparent wrongly accepted: '$bad'" + fi +done +pass "fm_trace_context_valid rejects all-zero ids, ff version, uppercase, wrong length, missing field, shell metacharacters, and empty" + +# A value shaped like a command substitution must be rejected as inert data and +# never executed. Assemble it so the test itself never runs it. +dollar='$' +fm_trace_context_valid "${dollar}(touch pwned-$$)" && fail "command-substitution-shaped value wrongly accepted" +[ ! -e "pwned-$$" ] || fail "validation must never execute an injected value" +pass "a command-substitution-shaped value is rejected as inert data, never executed" + +# --- entropy source: exact length, hex-only, fresh each call ----------------- + +t=$(fm_trace_context_hex 16) +[ "${#t}" -eq 32 ] || fail "16-byte hex must be 32 chars, got ${#t}" +case "$t" in *[!0-9a-f]*) fail "trace hex is not lowercase hex: $t" ;; esac +s=$(fm_trace_context_hex 8) +[ "${#s}" -eq 16 ] || fail "8-byte hex must be 16 chars, got ${#s}" +[ "$(fm_trace_context_hex 8)" != "$(fm_trace_context_hex 8)" ] || fail "hex must be fresh per call" +pass "fm_trace_context_hex yields exact-length lowercase hex, distinct per call" + +# --- root mint --------------------------------------------------------------- + +ROOT_TP=$(fm_trace_context_mint "") +fm_trace_context_valid "$ROOT_TP" || fail "root mint must be a valid traceparent: $ROOT_TP" +[ "${ROOT_TP:53:2}" = "01" ] || fail "root mint must default to sampled flags 01: $ROOT_TP" +[ "${ROOT_TP:3:32}" != "00000000000000000000000000000000" ] || fail "root trace id must be non-zero" +pass "fm_trace_context_mint with no parent starts a valid sampled root trace" + +# --- child mint inherits trace id + flags, mints a fresh span ---------------- + +PARENT='00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00' +CHILD=$(fm_trace_context_mint "$PARENT") +fm_trace_context_valid "$CHILD" || fail "child mint must be valid: $CHILD" +[ "${CHILD:3:32}" = "${PARENT:3:32}" ] || fail "child must inherit the parent trace id" +[ "${CHILD:53:2}" = "${PARENT:53:2}" ] || fail "child must inherit the parent flags (00)" +[ "${CHILD:36:16}" != "${PARENT:36:16}" ] || fail "child must mint a fresh span id" +pass "fm_trace_context_mint adopts a valid parent's trace id and flags with a fresh span id" + +# --- sampling flags: root is sampled 01; a child preserves the parent flag ----- + +sroot=$(fm_trace_context_mint "") +[ "${sroot:53:2}" = "01" ] || fail "a minted root must be sampled (01): $sroot" +sampled_child=$(fm_trace_context_mint '00-33333333333333333333333333333333-4444444444444444-01') +[ "${sampled_child:53:2}" = "01" ] || fail "a child of a sampled (01) parent must stay sampled: $sampled_child" +unsampled_child=$(fm_trace_context_mint '00-33333333333333333333333333333333-4444444444444444-00') +[ "${unsampled_child:53:2}" = "00" ] || fail "a child of an unsampled (00) parent must stay unsampled: $unsampled_child" +[ "${sampled_child:3:32}" = "33333333333333333333333333333333" ] || fail "a child must keep the parent trace id regardless of flags" +pass "a root is sampled (01) by decision; a child preserves the parent's sampled/unsampled flag verbatim, never overriding it" + +# --- nested Firstmate -> Secondmate -> worker share one trace id ------------- + +L1=$(fm_trace_context_mint "") # primary spawns a secondmate +L2=$(fm_trace_context_mint "$L1") # secondmate spawns a worker +L3=$(fm_trace_context_mint "$L2") # a further nested spawn +[ "${L1:3:32}" = "${L2:3:32}" ] && [ "${L2:3:32}" = "${L3:3:32}" ] \ + || fail "nested chain must share one trace id" +[ "${L1:36:16}" != "${L2:36:16}" ] && [ "${L2:36:16}" != "${L3:36:16}" ] && [ "${L1:36:16}" != "${L3:36:16}" ] \ + || fail "nested chain must have distinct span ids" +pass "nested Firstmate -> Secondmate -> worker mints share one trace id with distinct span ids" + +# --- malformed / all-zero inherited context roots a clean trace -------------- + +for garbage in 'not-a-traceparent' '00-00000000000000000000000000000000-0000000000000000-01' '' ; do + fresh=$(fm_trace_context_mint "$garbage") + fm_trace_context_valid "$fresh" || fail "malformed/all-zero inherited must fall back to a valid root: '$garbage' -> '$fresh'" + [ "${fresh:3:32}" != "00000000000000000000000000000000" ] || fail "fallback root trace id must be non-zero" +done +pass "malformed or all-zero inherited context is treated as absent and roots a clean trace" + +# --- minted-root shape and the opaque-inheritance boundary ------------------- +# The honest guarantee is NOT "hex cannot carry data" - an inherited traceparent's +# 24 id bytes are opaque caller-controlled data that firstmate passes through. It +# is that a firstmate-MINTED root is exactly the fixed 55-char W3C form with random +# ids and no free-form field where firstmate could originate a prompt, path, or +# secret (that the lib reads no task prose is asserted separately below). +case "$ROOT_TP" in + *[!0-9a-f-]*) fail "a minted traceparent must contain only hex and hyphens: $ROOT_TP" ;; +esac +[ "${#ROOT_TP}" -eq 55 ] || fail "a minted traceparent is exactly 55 chars, got ${#ROOT_TP}" +pass "a minted root is the fixed 55-char W3C form (hex and hyphens only), so firstmate originates no free-form content in the carrier" + +# The trust boundary, stated as a test: an inherited id is preserved verbatim, so +# whoever set TRACEPARENT controls those bytes (opaque caller data, not firstmate- +# originated). +passthrough=$(fm_trace_context_mint '00-deadbeefdeadbeefdeadbeefdeadbeef-1234567812345678-00') +[ "${passthrough:3:32}" = "deadbeefdeadbeefdeadbeefdeadbeef" ] \ + || fail "an inherited trace id must pass through verbatim (caller-controlled): $passthrough" +pass "an inherited traceparent's id bytes pass through verbatim - opaque caller-controlled data, a bounded fixed-width channel, not a firstmate-originated no-content guarantee" + +# --- enablement precedence --------------------------------------------------- + +WORK=$(fm_test_tmproot fm-trace-context) +CFG_ON="$WORK/cfg-on"; CFG_OFF="$WORK/cfg-off" +mkdir -p "$CFG_ON" "$CFG_OFF" +: > "$CFG_ON/trace-context" + +unset FM_TRACE_CONTEXT +fm_trace_context_enabled "$CFG_OFF" && fail "absent config/trace-context must be off by default" +fm_trace_context_enabled "$CFG_ON" || fail "present config/trace-context must enable" +FM_TRACE_CONTEXT=off fm_trace_context_enabled "$CFG_ON" && fail "FM_TRACE_CONTEXT=off must override a present file" +FM_TRACE_CONTEXT=on fm_trace_context_enabled "$CFG_OFF" || fail "FM_TRACE_CONTEXT=on must override an absent file" +FM_TRACE_CONTEXT=1 fm_trace_context_enabled "$CFG_OFF" || fail "FM_TRACE_CONTEXT=1 must enable" +FM_TRACE_CONTEXT=maybe fm_trace_context_enabled "$CFG_ON" && fail "a non-truthy FM_TRACE_CONTEXT must disable" +FM_TRACE_CONTEXT='' fm_trace_context_enabled "$CFG_ON" || fail "empty FM_TRACE_CONTEXT must defer to a present file (enabled)" +FM_TRACE_CONTEXT='' fm_trace_context_enabled "$CFG_OFF" && fail "empty FM_TRACE_CONTEXT must defer to an absent file (disabled)" +pass "enablement is default-off; FM_TRACE_CONTEXT overrides with truthy/other precedence, and unset or empty defers to config/trace-context" + +SESSION_DIR="$WORK/session-state" +SESSION_STATE="$SESSION_DIR/.trace-context-effective" +mkdir -p "$SESSION_DIR" +printf '101\n' > "$SESSION_DIR/.lock" +FM_TRACE_CONTEXT=off fm_trace_context_session_start "$CFG_ON" "$SESSION_STATE" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = off ] \ + || fail "session state must freeze an env-off override over a present config file" +FM_TRACE_CONTEXT=on fm_trace_context_session_start "$CFG_OFF" "$SESSION_STATE" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = on ] \ + || fail "a new session state must freeze an env-on override over an absent config file" +pass "session start normalizes config and environment precedence into frozen on/off state" + +printf '100 on\n' > "$SESSION_STATE" +chmod 0400 "$SESSION_STATE" +FM_TRACE_CONTEXT=off fm_trace_context_session_start "$CFG_ON" "$SESSION_STATE" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = off ] \ + || fail "atomic publication must replace a read-only stale on record with the current off decision" +[ "$(cat "$SESSION_STATE")" = "101 off" ] \ + || fail "session publication must bind the normalized decision to the current lock (got '$(cat "$SESSION_STATE")')" +pass "session state is atomically published through a same-directory replacement" + +FM_TRACE_CONTEXT=on fm_trace_context_session_start "$CFG_OFF" "$SESSION_STATE" +printf '202\n' > "$SESSION_DIR/.lock" +chmod 0500 "$SESSION_DIR" +FM_TRACE_CONTEXT=off fm_trace_context_session_start "$CFG_ON" "$SESSION_STATE" +chmod 0700 "$SESSION_DIR" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = off ] \ + || fail "a failed publication must not reactivate the prior session's on decision" +pass "a stale on record is inactive when publication fails in a new locked session" + +printf '202 invalid\n' > "$SESSION_STATE" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = off ] \ + || fail "invalid session state must fail independent and default off" +rm "$SESSION_STATE" +[ "$(fm_trace_context_session_effective "$SESSION_STATE")" = off ] \ + || fail "missing session state must fail independent and default off" +pass "missing or invalid frozen session state defaults off" + +# --- resolve: default-off omits; enabled mints ------------------------------ + +NOMETA="$WORK/none.meta" +out=$(fm_trace_context_resolve "$CFG_OFF" "$NOMETA"); rc=$? +[ -z "$out" ] && [ "$rc" -eq 0 ] || fail "default-off resolve must omit and return 0 (got rc=$rc out='$out')" +pass "resolve omits the carrier and returns success when the capability is off (byte-identical default)" + +out=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_OFF" "$NOMETA") +fm_trace_context_valid "$out" || fail "enabled resolve must mint a valid traceparent: $out" +pass "resolve mints a valid traceparent when enabled" + +# --- secondmate home-session boundary --------------------------------------- +# fm-spawn launches every Secondmate with the primary session's non-empty frozen +# FM_TRACE_CONTEXT decision. The Secondmate resolves it at its own session start. +# When the decision is on, ambient TRACEPARENT decides whether workers join the +# primary trace or start a new root. +PRIMARY_TP='00-abcabcabcabcabcabcabcabcabcabcab-1212121212121212-01' +saved_tp=${TRACEPARENT-__unset__} +unset TRACEPARENT +frozen_off=$(FM_TRACE_CONTEXT=off fm_trace_context_resolve "$CFG_ON" "$WORK/sm-frozen-off.meta") +[ -z "$frozen_off" ] || fail "a Secondmate launched off must stay disabled even after the config file appears: $frozen_off" +frozen_on=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_OFF" "$WORK/sm-frozen-on.meta") +fm_trace_context_valid "$frozen_on" || fail "a Secondmate launched on must stay enabled even while the config file is absent: $frozen_on" +[ "${frozen_on:3:32}" != "${PRIMARY_TP:3:32}" ] || fail "an enabled Secondmate without an ambient carrier must start a new root" +relaunched=$(TRACEPARENT="$PRIMARY_TP" FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_ON" "$WORK/sm-relaunched.meta") +[ "${relaunched:3:32}" = "${PRIMARY_TP:3:32}" ] || fail "a relaunched secondmate (ambient TRACEPARENT set) must continue the primary trace id: $relaunched" +[ "$saved_tp" = "__unset__" ] || export TRACEPARENT="$saved_tp" +pass "Secondmate home-session state stays off or on despite later file state; a relaunched enabled Secondmate continues the primary trace" + +# --- recovery: a recorded value is reused verbatim, disabled still omits ----- + +REC_META="$WORK/rec.meta" +printf 'kind=ship\ntraceparent=%s\nmode=no-mistakes\n' "$VALID" > "$REC_META" +out=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_ON" "$REC_META" '00-ffffffffffffffffffffffffffffffff-1111111111111111-01') +[ "$out" = "$VALID" ] || fail "recovery must reuse the recorded traceparent verbatim, ignoring inherited (got '$out')" +pass "resolve reuses a valid recorded traceparent verbatim on relaunch (stable identity across restarts)" + +out=$(fm_trace_context_resolve "$CFG_OFF" "$REC_META") +[ -z "$out" ] || fail "a disabled home must omit even when a traceparent is already recorded (got '$out')" +pass "disabling the capability omits the carrier even for a task with a recorded identity" + +CORRUPT_META="$WORK/corrupt.meta" +printf 'traceparent=not-a-valid-traceparent\n' > "$CORRUPT_META" +out=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_ON" "$CORRUPT_META") +fm_trace_context_valid "$out" || fail "a corrupt recorded value must be re-minted to a valid one" +[ "$out" != "not-a-valid-traceparent" ] || fail "a corrupt recorded value must not be reused" +pass "a corrupt recorded traceparent is re-minted rather than propagated" + +# --- durable metadata consistency: one value for record and injection -------- + +out=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_ON" "$NOMETA") +fm_trace_context_valid "$out" || fail "resolve must yield a single valid carrier per call" +pass "resolve yields exactly one carrier per logical task, so the recorded and injected values are identical by construction" + +# --- entropy failure omits telemetry safely (never aborts) ------------------- + +fm_trace_context_hex() { return 1; } +ef_mint=$(fm_trace_context_mint ""); ef_mint_rc=$? +ef_res=$(FM_TRACE_CONTEXT=on fm_trace_context_resolve "$CFG_ON" "$NOMETA"); ef_res_rc=$? +# Restore the real entropy source for any later use. +# shellcheck source=/dev/null +. "$ROOT/bin/fm-trace-context-lib.sh" +[ -z "$ef_mint" ] && [ "$ef_mint_rc" -ne 0 ] || fail "mint must omit and report failure on entropy failure (rc=$ef_mint_rc out='$ef_mint')" +[ -z "$ef_res" ] && [ "$ef_res_rc" -eq 0 ] || fail "resolve must omit and STILL return 0 on entropy failure (rc=$ef_res_rc out='$ef_res')" +pass "entropy failure omits telemetry safely: mint reports failure, resolve returns success with no carrier" + +# --- fail-independent timing: no hang source, always returns 0 --------------- + +assert_no_grep 'sleep' "$ROOT/bin/fm-trace-context-lib.sh" "trace-context lib must not sleep on the spawn path" +assert_no_grep 'timeout' "$ROOT/bin/fm-trace-context-lib.sh" "trace-context lib must not depend on an external timeout" +assert_no_grep 'command:' "$ROOT/bin/fm-trace-context-lib.sh" "trace-context lib must not run an arbitrary command provider" +fm_trace_context_resolve "$CFG_OFF" "$NOMETA" >/dev/null || fail "resolve must return 0 when off" +pass "the resolver has no sleep/timeout/command hang source and always returns success" + +# --- harness/backend/kind independence (code only, comments stripped) --------- + +LIB_CODE=$(sed 's/#.*$//' "$ROOT/bin/fm-trace-context-lib.sh") +for tok in harness backend tmux herdr zellij orca cmux claude codex opencode grok kind ship scout secondmate ; do + case "$LIB_CODE" in + *"$tok"*) fail "trace-context lib code must be harness/backend/kind agnostic, but references '$tok'" ;; + esac +done +pass "the carrier is minted identically for every harness, backend, and spawn kind (no such branching in the lib code)" + +# --- no prompt / task-prose reads (code only, comments stripped) -------------- + +for tok in brief prompt report status ; do + case "$LIB_CODE" in + *"$tok"*) fail "trace-context lib code must never read task prose, but references '$tok'" ;; + esac +done +pass "the lib code never reads a brief, prompt, report, or status - it cannot leak content" + +# --- structural wiring in bin/fm-spawn.sh ------------------------------------ + +SPAWN="$ROOT/bin/fm-spawn.sh" +# Patterns deliberately start after any leading '$' so the fixed-string grep needs +# no shell metacharacters while still pinning the exact wiring. +assert_grep 'fm-trace-context-lib.sh' "$SPAWN" "fm-spawn.sh must source the trace-context lib" +assert_grep 'SPAWN_TRACEPARENT=' "$SPAWN" "fm-spawn.sh must assign the resolved carrier" +assert_grep 'fm_trace_context_resolve' "$SPAWN" "fm-spawn.sh must resolve the carrier through the lib entry point" +# shellcheck disable=SC2016 # Dollar signs are literal source text in this fixed-string assertion. +assert_grep 'if spawn_send_text_line "$T" "export TRACEPARENT=$SPAWN_TRACEPARENT"; then' "$SPAWN" \ + "fm-spawn.sh must condition metadata publication on successful carrier delivery" +# shellcheck disable=SC2016 # Dollar signs are literal source text in this fixed-string assertion. +assert_grep 'echo "traceparent=$SPAWN_TRACEPARENT" >> "$STATE/$ID.meta"' "$SPAWN" \ + "fm-spawn.sh must record the delivered carrier in metadata" +assert_grep 'export TRACEPARENT=' "$SPAWN" "fm-spawn.sh must inject the W3C TRACEPARENT env var" +pass "fm-spawn.sh sources the lib and records one shared SPAWN_TRACEPARENT only after successful injection" + +# The injection must ride the same channel and site as GOTMPDIR (before launch, +# unconditional across kinds): the TRACEPARENT export follows the GOTMPDIR export. +gotmp_line=$(grep -n 'export GOTMPDIR=' "$SPAWN" | tail -1 | cut -d: -f1) +tp_line=$(grep -n 'export TRACEPARENT=' "$SPAWN" | tail -1 | cut -d: -f1) +# shellcheck disable=SC2016 # Dollar signs are literal source text in this grep pattern. +meta_line=$(grep -n 'echo "traceparent=$SPAWN_TRACEPARENT" >>' "$SPAWN" | tail -1 | cut -d: -f1) +[ -n "$gotmp_line" ] && [ -n "$tp_line" ] && [ -n "$meta_line" ] \ + && [ "$tp_line" -gt "$gotmp_line" ] && [ "$((tp_line - gotmp_line))" -le 5 ] \ + && [ "$meta_line" -gt "$tp_line" ] \ + || fail "TRACEPARENT must be exported before metadata publication at the pre-launch GOTMPDIR site (gotmp=$gotmp_line tp=$tp_line meta=$meta_line)" +pass "TRACEPARENT is injected at the unconditional pre-launch GOTMPDIR site and recorded only after successful delivery" + +# --- secondmate inheritance wires the nested chain --------------------------- + +# shellcheck source=/dev/null +. "$ROOT/bin/fm-config-inherit-lib.sh" +case " $FM_INHERITABLE_CONFIG " in + *" trace-context "*) : ;; + *) fail "config/trace-context must be in FM_INHERITABLE_CONFIG so secondmate homes stay traced" ;; +esac +pass "config/trace-context is inherited into secondmate homes, keeping the nested chain enabled end to end" + +echo "# fm-trace-context-lib.test.sh: all assertions passed" diff --git a/tests/fm-trace-context-spawn.test.sh b/tests/fm-trace-context-spawn.test.sh new file mode 100755 index 0000000000..79d9548ad0 --- /dev/null +++ b/tests/fm-trace-context-spawn.test.sh @@ -0,0 +1,501 @@ +#!/usr/bin/env bash +# tests/fm-trace-context-spawn.test.sh - spawn-path integration regressions for +# native W3C trace context using fake tmux panes and real isolated git worktrees. +# See docs/verification/trace-context.md for the maintained coverage inventory. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-trace-context-lib.sh" + +SPAWN="$ROOT/bin/fm-spawn.sh" +TMP_ROOT=$(fm_test_tmproot fm-trace-context-spawn) + +# Fake tmux: answers the pane-path query and logs every literal `send-keys -l` +# argument (the GOTMPDIR export, the TRACEPARENT export, and the launch command) +# one per line, in send order, so ordering is observable. +make_spawn_fakebin() { + local dir=$1 fakebin + fakebin=$(fm_fakebin "$dir") + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "$*" in + *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}"; exit 0 ;; +esac +case "${1:-}" in + display-message) printf 'firstmate\n'; exit 0 ;; + list-windows) + [ -z "${FM_FAKE_DUPLICATE_WINDOW:-}" ] || printf '%s\n' "$FM_FAKE_DUPLICATE_WINDOW" + exit 0 + ;; + has-session|new-session|new-window|kill-window) exit 0 ;; + send-keys) + if [ "${FM_FAKE_TRACEPARENT_SEND_FAIL:-0}" = 1 ]; then + for a in "$@"; do + case "$a" in + "export TRACEPARENT="*) exit 1 ;; + esac + done + fi + if [ "${FM_FAKE_TRACEPARENT_SEND_UNSAFE:-0}" = 1 ]; then + for a in "$@"; do + case "$a" in + "export TRACEPARENT="*) exit 2 ;; + esac + done + fi + if [ "${FM_FAKE_TRACE_METADATA_APPEND_FAIL:-0}" = 1 ]; then + for a in "$@"; do + case "$a" in + "export TRACEPARENT="*) + chmod a-w "$FM_FAKE_META_PATH" + ;; + esac + done + fi + # Capture the text payload of both send forms: the literal launch + # (`send-keys -t -l `) and a text line + # (`send-keys -t Enter`). Skip the flags, the target, and + # the trailing key so only the payload is logged, one per line, in order. + if [ -n "${FM_FAKE_LAUNCH_LOG:-}" ]; then + shift + skip_next= + for a in "$@"; do + if [ -n "$skip_next" ]; then skip_next=; continue; fi + case "$a" in + -t) skip_next=1; continue ;; + -l) continue ;; + Enter|C-m) continue ;; + *) printf '%s\n' "$a" >> "$FM_FAKE_LAUNCH_LOG" ;; + esac + done + fi + exit 0 + ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + fm_fake_exit0 "$fakebin" treehouse + printf '%s\n' "$fakebin" +} + +make_spawn_case() { + local name=$1 case_dir home proj wt fakebin launchlog id + case_dir="$TMP_ROOT/$name" + home="$case_dir/home" + proj="$case_dir/project" + wt="$case_dir/wt" + launchlog="$case_dir/launch.log" + fakebin=$(make_spawn_fakebin "$case_dir/fake") + mkdir -p "$home/data" "$home/projects" "$home/state" "$home/config" + printf 'claude\n' > "$home/config/crew-harness" + printf '%s\n' "$$" > "$home/state/.lock" + printf '%s off\n' "$$" > "$home/state/.trace-context-effective" + fm_git_worktree "$proj" "$wt" "wt-$name" + touch "$home/state/.last-watcher-beat" + id=$name-z1 + mkdir -p "$home/data/$id" + printf 'brief for %s\n' "$id" > "$home/data/$id/brief.md" + printf '%s\n' "$home|$proj|$wt|$fakebin|$launchlog|$id" +} + +# Hermetic against an ambient FM_TRACE_CONTEXT: `env -u` unsets it so enablement +# is decided ONLY by the home's config/trace-context, whether the runner's own +# environment enables or disables trace context. +run_spawn() { + local home=$1 wt=$2 fakebin=$3 launchlog=$4 + shift 4 + : > "$launchlog" + env -u FM_TRACE_CONTEXT \ + FM_ROOT_OVERRIDE='' FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FM_DATA_OVERRIDE="$home/data" \ + FM_PROJECTS_OVERRIDE="$home/projects" FM_CONFIG_OVERRIDE="$home/config" \ + FM_SPAWN_NO_GUARD=1 FM_FAKE_PANE_PATH="$wt" TMUX="fake,1,0" \ + FM_FAKE_TRACEPARENT_SEND_FAIL="${FM_FAKE_TRACEPARENT_SEND_FAIL:-0}" \ + FM_FAKE_TRACEPARENT_SEND_UNSAFE="${FM_FAKE_TRACEPARENT_SEND_UNSAFE:-0}" \ + FM_FAKE_TRACE_METADATA_APPEND_FAIL="${FM_FAKE_TRACE_METADATA_APPEND_FAIL:-0}" \ + FM_FAKE_META_PATH="$home/state/$1.meta" \ + FM_FAKE_LAUNCH_LOG="$launchlog" PATH="$fakebin:$PATH" \ + "$SPAWN" "$@" 2>&1 +} + +# Same, but with an explicit FM_TRACE_CONTEXT override, to prove the env decides. +run_spawn_tc() { + local tc=$1 home=$2 wt=$3 fakebin=$4 launchlog=$5 + shift 5 + : > "$launchlog" + env FM_TRACE_CONTEXT="$tc" \ + FM_ROOT_OVERRIDE='' FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FM_DATA_OVERRIDE="$home/data" \ + FM_PROJECTS_OVERRIDE="$home/projects" FM_CONFIG_OVERRIDE="$home/config" \ + FM_SPAWN_NO_GUARD=1 FM_FAKE_PANE_PATH="$wt" TMUX="fake,1,0" \ + FM_FAKE_LAUNCH_LOG="$launchlog" PATH="$fakebin:$PATH" \ + "$SPAWN" "$@" 2>&1 +} + +start_trace_session() { + local home=$1 tc=${2-} + printf '%s\n' "$$" > "$home/state/.lock" + if [ -n "$tc" ]; then + FM_TRACE_CONTEXT="$tc" fm_trace_context_session_start \ + "$home/config" "$home/state/.trace-context-effective" + else + ( + unset FM_TRACE_CONTEXT + fm_trace_context_session_start \ + "$home/config" "$home/state/.trace-context-effective" + ) + fi +} + +read_case_record() { + IFS='|' read -r HOME_DIR PROJ_DIR WT_DIR FAKEBIN_DIR LAUNCH_LOG CASE_ID < secondmate -> worker regression for the FM_TRACE_CONTEXT +# effective override. Drives bin/fm-spawn.sh TWICE against real homes and a real +# worktree: first the primary launches a secondmate (capturing the exact env the +# primary injects into it), then that secondmate launches its OWN worker with +# exactly that inherited env, reading the secondmate home's own inherited config. +# This is what proves the primary's effective on/off decision - not only the +# copied config/trace-context file - governs the nested worker, which a +# single-home spawn test cannot reach. Sets TL_ENV_TC, TL_CARRIER, TL_WORKER_TP, +# and TL_SM_FILE for the caller. +# run_two_level +run_two_level() { + local name=$1 pfile=$2 penv=$3 + local base prim sm sm_id smlog smfake worker_id wproj wwt wlog wfake + base="$TMP_ROOT/2level-$name" + prim="$base/primary" + sm="$base/sm" + mkdir -p "$prim/config" "$prim/data" "$prim/state" "$prim/projects" + printf 'claude\n' > "$prim/config/crew-harness" + [ "$pfile" = present ] && : > "$prim/config/trace-context" + touch "$prim/state/.last-watcher-beat" + start_trace_session "$prim" "$penv" + + # Seed the secondmate home so validate_firstmate_home_for_spawn accepts it. + mkdir -p "$sm/bin" "$sm/data" + printf '# Firstmate\n' > "$sm/AGENTS.md" + printf 'sm-%s\n' "$name" > "$sm/.fm-secondmate-home" + printf 'charter\n' > "$sm/data/charter.md" + + # Spawn 1: the primary launches the secondmate; capture what it injects. + sm_id="sm-$name" + mkdir -p "$prim/data/$sm_id" + printf 'charter brief\n' > "$prim/data/$sm_id/brief.md" + smlog="$base/sm-launch.log" + smfake=$(make_spawn_fakebin "$base/sm-fake") + : > "$smlog" + env FM_TRACE_CONTEXT="$penv" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$prim" \ + FM_STATE_OVERRIDE="$prim/state" FM_DATA_OVERRIDE="$prim/data" \ + FM_PROJECTS_OVERRIDE="$prim/projects" FM_CONFIG_OVERRIDE="$prim/config" \ + FM_SPAWN_NO_GUARD=1 CLAUDECODE=1 TMUX="fake,1,0" \ + FM_FAKE_LAUNCH_LOG="$smlog" PATH="$smfake:$PATH" \ + "$SPAWN" "$sm_id" "$sm" --secondmate >/dev/null 2>&1 || true + + # Extract the EXACT env the primary put on the secondmate: the normalized + # FM_TRACE_CONTEXT in the launch prefix, and the TRACEPARENT carrier (if any). + TL_ENV_TC=$(grep -o 'FM_TRACE_CONTEXT=[a-z]*' "$smlog" | head -1 | cut -d= -f2) + TL_CARRIER=$(injected_traceparent "$smlog" | head -1) + + # Spawn 2: the secondmate launches its own worker with exactly that inherited + # env, reading the secondmate home's own (inherited) config. + worker_id="w-$name" + wproj="$base/wproj" + wwt="$base/wwt" + fm_git_worktree "$wproj" "$wwt" "wt-$name" + mkdir -p "$sm/state" "$sm/projects" "$sm/data/$worker_id" + printf 'worker brief\n' > "$sm/data/$worker_id/brief.md" + touch "$sm/state/.last-watcher-beat" + start_trace_session "$sm" "$TL_ENV_TC" + wlog="$base/worker-launch.log" + wfake=$(make_spawn_fakebin "$base/w-fake") + : > "$wlog" + env FM_TRACE_CONTEXT="$TL_ENV_TC" TRACEPARENT="$TL_CARRIER" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$sm" \ + FM_STATE_OVERRIDE="$sm/state" FM_DATA_OVERRIDE="$sm/data" \ + FM_PROJECTS_OVERRIDE="$sm/projects" FM_CONFIG_OVERRIDE="$sm/config" \ + FM_SPAWN_NO_GUARD=1 FM_FAKE_PANE_PATH="$wwt" TMUX="fake,1,0" \ + FM_FAKE_LAUNCH_LOG="$wlog" PATH="$wfake:$PATH" \ + "$SPAWN" "$worker_id" "$wproj" >/dev/null 2>&1 || true + + TL_WORKER_TP=$(meta_traceparent "$sm/state/$worker_id.meta") + TL_SM_FILE=absent + [ -f "$sm/config/trace-context" ] && TL_SM_FILE=present +} + +test_enabled_records_and_injects_identical_carrier_before_launch() { + local rec out status meta mtp itp gl tl ll + rec=$(make_spawn_case tc-on) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" # enable via the real config path + start_trace_session "$HOME_DIR" + + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "enabled trace-context spawn should succeed" + assert_contains "$out" "spawned $CASE_ID" "enabled spawn should report success" + meta="$HOME_DIR/state/$CASE_ID.meta" + + mtp=$(meta_traceparent "$meta") + fm_trace_context_valid "$mtp" || fail "enabled spawn must record a valid traceparent= in meta (got '$mtp')" + itp=$(injected_traceparent "$LAUNCH_LOG") + fm_trace_context_valid "$itp" || fail "enabled spawn must inject a valid TRACEPARENT export (got '$itp')" + [ "$mtp" = "$itp" ] || fail "the recorded and injected carriers must be identical (meta='$mtp' injected='$itp')" + + gl=$(grep -n '^export GOTMPDIR=' "$LAUNCH_LOG" | tail -1 | cut -d: -f1) + tl=$(grep -n '^export TRACEPARENT=' "$LAUNCH_LOG" | tail -1 | cut -d: -f1) + ll=$(grep -n 'claude' "$LAUNCH_LOG" | tail -1 | cut -d: -f1) + [ -n "$gl" ] && [ -n "$tl" ] && [ -n "$ll" ] || fail "launch log missing GOTMPDIR/TRACEPARENT/launch lines" + [ "$tl" -gt "$gl" ] || fail "TRACEPARENT export must ride the GOTMPDIR pre-launch site (gotmp=$gl tp=$tl)" + [ "$tl" -lt "$ll" ] || fail "TRACEPARENT export must be sent before the launch literal (tp=$tl launch=$ll)" + pass "enabled: one resolved carrier is recorded in meta and the identical TRACEPARENT is exported before launch" +} + +test_disabled_writes_and_injects_neither() { + local rec out status meta + rec=$(make_spawn_case tc-off) + read_case_record "$rec" + # No config/trace-context and no FM_TRACE_CONTEXT: default-off. + + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "default-off spawn should succeed" + assert_contains "$out" "spawned $CASE_ID" "default-off spawn should report success" + meta="$HOME_DIR/state/$CASE_ID.meta" + + # Anchored regex checks (the assert_grep helpers are fixed-string). + ! grep -q '^traceparent=' "$meta" || fail "default-off spawn must not write a traceparent= line to meta" + ! grep -q '^export TRACEPARENT=' "$LAUNCH_LOG" || fail "default-off spawn must not inject a TRACEPARENT export" + grep -q '^export GOTMPDIR=' "$LAUNCH_LOG" || fail "the spawn should still run (GOTMPDIR is always injected)" + pass "disabled: neither traceparent= in meta nor a TRACEPARENT export is produced" +} + +test_failed_delivery_omits_metadata_and_still_launches() { + local rec out status meta + rec=$(make_spawn_case tc-send-failure) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" + start_trace_session "$HOME_DIR" + + out=$(FM_FAKE_TRACEPARENT_SEND_FAIL=1 \ + run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "failed traceparent delivery must not abort spawn" + assert_contains "$out" "spawned $CASE_ID" "spawn should report success after failed traceparent delivery" + meta="$HOME_DIR/state/$CASE_ID.meta" + + ! grep -q '^traceparent=' "$meta" \ + || fail "failed traceparent delivery must not leave a traceparent= claim in meta" + ! grep -q '^export TRACEPARENT=' "$LAUNCH_LOG" \ + || fail "the failed TRACEPARENT export must not be recorded as delivered" + grep -q 'claude' "$LAUNCH_LOG" || fail "the source task must still launch" + pass "failed TRACEPARENT delivery omits metadata while the source task still launches" +} + +test_unsafe_delivery_refuses_to_append_launch() { + local rec out status + rec=$(make_spawn_case tc-send-unsafe) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" + start_trace_session "$HOME_DIR" + + out=$(FM_FAKE_TRACEPARENT_SEND_UNSAFE=1 \ + run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + [ "$status" -ne 0 ] || fail "uncleared traceparent input must stop spawn" + assert_contains "$out" "refusing to append the launch command" \ + "unsafe traceparent delivery should report why spawn stopped" + ! grep -q 'claude' "$LAUNCH_LOG" \ + || fail "unsafe traceparent delivery must not append the launch command" + pass "uncleared TRACEPARENT input stops before the launch command is appended" +} + +test_failed_metadata_append_unsets_carrier_and_still_launches() { + local rec out status meta + rec=$(make_spawn_case tc-metadata-failure) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" + start_trace_session "$HOME_DIR" + + out=$(FM_FAKE_TRACE_METADATA_APPEND_FAIL=1 \ + run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "failed traceparent metadata append must not abort spawn" + assert_contains "$out" "spawned $CASE_ID" "spawn should report success after failed metadata append" + meta="$HOME_DIR/state/$CASE_ID.meta" + + ! grep -q '^traceparent=' "$meta" \ + || fail "failed metadata append must not leave a traceparent= claim in meta" + grep -q '^unset TRACEPARENT; .*claude' "$LAUNCH_LOG" \ + || fail "failed metadata append must unset TRACEPARENT in the launch command" + pass "failed traceparent metadata append removes the carrier from the launched task" +} + +test_duplicate_secondmate_spawn_does_not_converge_trace_context() { + local base prim sm id log fake out status + base="$TMP_ROOT/duplicate-secondmate" + prim="$base/primary" + sm="$base/sm" + # shellcheck disable=SC2100 # Literal task id, not arithmetic. + id=sm-duplicate + log="$base/launch.log" + mkdir -p "$prim/config" "$prim/data/$id" "$prim/state" "$prim/projects" + : > "$prim/config/trace-context" + printf 'charter brief\n' > "$prim/data/$id/brief.md" + touch "$prim/state/.last-watcher-beat" + start_trace_session "$prim" + mkdir -p "$sm/bin" "$sm/data" + printf '# Firstmate\n' > "$sm/AGENTS.md" + printf '%s\n' "$id" > "$sm/.fm-secondmate-home" + printf 'charter\n' > "$sm/data/charter.md" + fake=$(make_spawn_fakebin "$base/fake") + + out=$(env -u FM_TRACE_CONTEXT \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$prim" \ + FM_STATE_OVERRIDE="$prim/state" FM_DATA_OVERRIDE="$prim/data" \ + FM_PROJECTS_OVERRIDE="$prim/projects" FM_CONFIG_OVERRIDE="$prim/config" \ + FM_SPAWN_NO_GUARD=1 CLAUDECODE=1 TMUX="fake,1,0" \ + FM_FAKE_DUPLICATE_WINDOW="fm-$id" FM_FAKE_LAUNCH_LOG="$log" \ + PATH="$fake:$PATH" "$SPAWN" "$id" "$sm" --secondmate 2>&1) + status=$? + + [ "$status" -ne 0 ] || fail "duplicate secondmate spawn should be refused" + assert_contains "$out" "already exists" "duplicate secondmate spawn should report the existing endpoint" + [ ! -e "$sm/config/trace-context" ] \ + || fail "duplicate preflight must not converge trace-context into the secondmate home" + pass "duplicate secondmate preflight leaves trace-context unchanged" +} + +test_relaunch_reuses_recorded_carrier() { + local rec out status meta first second injected + rec=$(make_spawn_case tc-relaunch) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" + start_trace_session "$HOME_DIR" + meta="$HOME_DIR/state/$CASE_ID.meta" + + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "first trace-context spawn should succeed" + assert_contains "$out" "spawned $CASE_ID" "first spawn should report success" + first=$(meta_traceparent "$meta") + fm_trace_context_valid "$first" || fail "first spawn must record a valid carrier (got '$first')" + + # Relaunch the same task: the recorded carrier must be reused verbatim for both + # the meta and the injected export, so an observer keeps one identity across + # restarts. + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "relaunch spawn should succeed" + assert_contains "$out" "spawned $CASE_ID" "relaunch spawn should report success" + second=$(meta_traceparent "$meta") + injected=$(injected_traceparent "$LAUNCH_LOG") + [ "$second" = "$first" ] || fail "relaunch must reuse the recorded carrier in meta (first='$first' second='$second')" + [ "$injected" = "$first" ] || fail "relaunch must inject the same recorded carrier (first='$first' injected='$injected')" + pass "relaunch reuses the recorded carrier verbatim for both the meta record and the injected export" +} + +test_session_start_freezes_env_override_and_ignores_later_edits() { + local rec out status meta + rec=$(make_spawn_case tc-envoff) + read_case_record "$rec" + : > "$HOME_DIR/config/trace-context" + start_trace_session "$HOME_DIR" off + out=$(run_spawn_tc on "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "env-off spawn should succeed" + assert_contains "$out" "spawned $CASE_ID" "env-off spawn should report success" + meta="$HOME_DIR/state/$CASE_ID.meta" + ! grep -q '^traceparent=' "$meta" || fail "session-frozen off must ignore a later FM_TRACE_CONTEXT=on" + ! grep -q '^export TRACEPARENT=' "$LAUNCH_LOG" || fail "session-frozen off must remain disabled after launch-time edits" + + rec=$(make_spawn_case tc-envon) + read_case_record "$rec" + start_trace_session "$HOME_DIR" on + : > "$HOME_DIR/config/trace-context" + out=$(run_spawn_tc off "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "env-on spawn should succeed" + meta="$HOME_DIR/state/$CASE_ID.meta" + fm_trace_context_valid "$(meta_traceparent "$meta")" \ + || fail "session-frozen on must ignore a later FM_TRACE_CONTEXT=off" + pass "session start freezes the env override and later config or environment edits do not alter spawns" +} + +# End-to-end two-level enable path: the primary is enabled by the environment +# override with NO config file, and that enablement must reach the newly launched +# secondmate's own worker so the whole chain shares one trace. Before the +# effective-override fix, the secondmate saw only the (absent) inherited file and +# left its worker untraced despite receiving the parent carrier. +test_secondmate_env_on_file_absent_keeps_nested_worker_enabled() { + run_two_level enable absent on + [ "$TL_ENV_TC" = on ] || fail "the primary must deliver FM_TRACE_CONTEXT=on to the secondmate (got '$TL_ENV_TC')" + fm_trace_context_valid "$TL_CARRIER" || fail "an enabled primary must mint a carrier for the secondmate (got '$TL_CARRIER')" + fm_trace_context_valid "$TL_WORKER_TP" \ + || fail "env-on/file-absent must keep the nested worker enabled (got '$TL_WORKER_TP')" + [ "${TL_CARRIER:3:32}" = "${TL_WORKER_TP:3:32}" ] \ + || fail "the nested worker must share the primary trace id (parent='${TL_CARRIER:3:32}' worker='${TL_WORKER_TP:3:32}')" + [ "${TL_CARRIER:36:16}" != "${TL_WORKER_TP:36:16}" ] \ + || fail "the nested worker must mint a fresh span id, not reuse the parent's" + pass "two-level: env-on/file-absent keeps the nested worker enabled and in the same trace as the primary" +} + +# End-to-end two-level disable path: the primary is disabled by the environment +# override while the config file is PRESENT (so it is copied into the secondmate +# home). The override must still disable the secondmate's own worker, or +# FM_TRACE_CONTEXT=off is not a real kill switch. Before the fix the copied file +# re-enabled the nested worker. +test_secondmate_env_off_file_present_keeps_nested_worker_disabled() { + run_two_level disable present off + [ "$TL_ENV_TC" = off ] || fail "the primary must deliver FM_TRACE_CONTEXT=off to the secondmate (got '$TL_ENV_TC')" + [ -z "$TL_CARRIER" ] || fail "a disabled primary must inject no carrier into the secondmate (got '$TL_CARRIER')" + [ "$TL_SM_FILE" = present ] \ + || fail "the disable case must exercise a copied config/trace-context in the secondmate home (got '$TL_SM_FILE')" + [ -z "$TL_WORKER_TP" ] \ + || fail "env-off must keep the nested worker disabled even with the file present (got '$TL_WORKER_TP')" + pass "two-level: env-off/file-present keeps the nested worker disabled even though the config file was copied into the secondmate home" +} + +# Single-frozen-decision guarantee: for a secondmate spawn the recorded/injected +# carrier and the delivered FM_TRACE_CONTEXT snapshot are always derived from ONE +# effective decision, so they cannot disagree (no carrier paired with off, no +# off snapshot paired with a carrier). This drives the file-decided path +# (FM_TRACE_CONTEXT unset), which is exactly where the two-read correction matters +# because the environment override is empty and only the config file decides. +test_secondmate_carrier_and_snapshot_share_one_decision() { + run_two_level fileon present "" + [ "$TL_ENV_TC" = on ] || fail "a file-enabled secondmate must snapshot FM_TRACE_CONTEXT=on (got '$TL_ENV_TC')" + fm_trace_context_valid "$TL_CARRIER" \ + || fail "a file-enabled secondmate's carrier must be present and valid, consistent with the on snapshot (got '$TL_CARRIER')" + + run_two_level fileoff absent "" + [ "$TL_ENV_TC" = off ] || fail "a file-disabled secondmate must snapshot FM_TRACE_CONTEXT=off (got '$TL_ENV_TC')" + [ -z "$TL_CARRIER" ] \ + || fail "a file-disabled secondmate must inject no carrier, consistent with the off snapshot (got '$TL_CARRIER')" + pass "secondmate carrier and FM_TRACE_CONTEXT snapshot always agree, both derived from one frozen decision (file-decided path)" +} + +test_enabled_records_and_injects_identical_carrier_before_launch +test_disabled_writes_and_injects_neither +test_failed_delivery_omits_metadata_and_still_launches +test_unsafe_delivery_refuses_to_append_launch +test_failed_metadata_append_unsets_carrier_and_still_launches +test_duplicate_secondmate_spawn_does_not_converge_trace_context +test_relaunch_reuses_recorded_carrier +test_session_start_freezes_env_override_and_ignores_later_edits +test_secondmate_env_on_file_absent_keeps_nested_worker_enabled +test_secondmate_env_off_file_present_keeps_nested_worker_disabled +test_secondmate_carrier_and_snapshot_share_one_decision + +echo "# all fm-trace-context-spawn tests passed"