diff --git a/changelog.d/added-14746-repeat-failure-copilot-omp-agy.md b/changelog.d/added-14746-repeat-failure-copilot-omp-agy.md new file mode 100644 index 0000000000..dba89178dd --- /dev/null +++ b/changelog.d/added-14746-repeat-failure-copilot-omp-agy.md @@ -0,0 +1 @@ +- **The repeated-failure check now sees commands run by copilot, omp and headless agy agents** ([#14746](https://github.com/hivecommons/hive/issues/14746), [#14747](https://github.com/hivecommons/hive/issues/14747), [#14750](https://github.com/hivecommons/hive/issues/14750), part of [#14550](https://github.com/hivecommons/hive/issues/14550)). These backends run shell commands where the neutral `BASH_ENV` route never reaches them, so an agent retrying the same failing command went unnoticed. With `repeat_failure.enabled` on, Hive now reads Copilot's `events.jsonl` session log (one record per shell `tool.execution_complete`, with the real exit code), loads a Hive-owned omp extension via its own `--extension` flag (one record per bash `tool_result`, fail-open), and has `hive agy-turn` record each finished `run_command` step. Nothing is written into the backends' own settings; with the check off Hive passes no flag and removes the omp extension at the next launch. copilot and omp are marked covered in the "Repeated-failure check" table of `docs/backend-support-tiers.md` with committed, re-runnable harness evidence; agy stays "not covered" there until a real-run harness can log in. diff --git a/src/crux/agent/agy_launch_test.go b/src/crux/agent/agy_launch_test.go index 4c87c87c73..82cfe6fb9f 100644 --- a/src/crux/agent/agy_launch_test.go +++ b/src/crux/agent/agy_launch_test.go @@ -130,7 +130,7 @@ func TestAgyLaunchCommandLine_DerivesEffortFromModelSuffix(t *testing.T) { } func TestAgyHeadlessTurnCommandRunsTurnRunner(t *testing.T) { - cmd := agyHeadlessTurnShellCommand("/usr/local/bin/hive", "agy", "gemini-pro", "high", "/data/agents/a/.hive-agy-prompt-a.txt", false) + cmd := agyHeadlessTurnShellCommand("/usr/local/bin/hive", "agy", "gemini-pro", "high", "/data/agents/a/.hive-agy-prompt-a.txt", false, "") if strings.Contains(cmd, "do the task") || strings.Contains(cmd, "$(cat") { t.Fatalf("headless turn command must not expand the prompt into argv (E2BIG past 128 KiB); cmd: %q", cmd) @@ -148,10 +148,10 @@ func TestAgyHeadlessTurnCommandRunsTurnRunner(t *testing.T) { if strings.Contains(cmd, "--new-conversation") { t.Errorf("a normal kick must continue the agent's conversation; cmd: %q", cmd) } - if cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "", "/p", true); !strings.Contains(cmd, "--new-conversation") { + if cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "", "/p", true, ""); !strings.Contains(cmd, "--new-conversation") { t.Errorf("ClearOnKick must start a new conversation; cmd: %q", cmd) } - if cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "high", "/p", false); strings.Contains(cmd, "--model") || strings.Contains(cmd, "--effort") { + if cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "high", "/p", false, ""); strings.Contains(cmd, "--model") || strings.Contains(cmd, "--effort") { t.Errorf("no model configured must pass neither --model nor --effort; cmd: %q", cmd) } } diff --git a/src/crux/agent/agy_turn.go b/src/crux/agent/agy_turn.go index eb873ac9f7..3e8020717c 100644 --- a/src/crux/agent/agy_turn.go +++ b/src/crux/agent/agy_turn.go @@ -124,6 +124,9 @@ type agyTurnOptions struct { promptFile string conversationFile string newConversation bool + // recordFile is the repeat-failure raw record file; empty while the + // check is off (repeat_failure_agy.go). + recordFile string } // RunAgyTurn implements `hive agy-turn`. It returns the process exit code. @@ -149,6 +152,7 @@ func parseAgyTurnArgs(args []string, stderr io.Writer) (agyTurnOptions, error) { fs.StringVar(&opts.promptFile, "prompt-file", "", "file holding the prompt for this turn (required)") fs.StringVar(&opts.conversationFile, "conversation-file", "", "file recording the conversation id to resume; empty disables continuity") fs.BoolVar(&opts.newConversation, "new-conversation", false, "ignore the recorded conversation and start a new one") + fs.StringVar(&opts.recordFile, agyRepeatFailureRecordFlag, "", "repeat-failure record file to append one record per run_command to; empty records nothing") if err := fs.Parse(args); err != nil { return opts, err } @@ -205,6 +209,7 @@ func runAgyTurn(opts agyTurnOptions, stdout, stderr io.Writer) int { } r := newAgyTurnRenderer(stdout, time.Now()) + r.recordFile = opts.recordFile r.status() if err := cmd.Start(); err != nil { r.finish() @@ -366,6 +371,13 @@ type agyTurnRenderer struct { pending map[int]string // step index → assistant text not yet ended by a newline lastTool string result *agyTurnResult + // recordFile receives one repeat-failure record per run_command step that + // ends; empty records nothing. commands holds each run_command step's + // CommandLine from its ACTIVE update; recorded holds the steps already + // written. + recordFile string + commands map[int]string + recorded map[int]bool } func newAgyTurnRenderer(w io.Writer, started time.Time) *agyTurnRenderer { @@ -503,6 +515,9 @@ func (r *agyTurnRenderer) handleStep(s *agyStepUpdate) { if name == "" && s.ToolInfo != nil { name = s.ToolInfo.Name } + if name == "run_command" { + r.recordRunCommand(s) + } switch s.State { case "ACTIVE": r.mu.Lock() @@ -525,6 +540,37 @@ func (r *agyTurnRenderer) handleStep(s *agyStepUpdate) { } } +// recordRunCommand appends the repeat-failure record of a run_command step +// once it ends, at most once per step. Fails open: an unwritable record file +// means no record and nothing else. +func (r *agyTurnRenderer) recordRunCommand(s *agyStepUpdate) { + if r.recordFile == "" { + return + } + r.mu.Lock() + if r.commands == nil { + r.commands, r.recorded = map[int]string{}, map[int]bool{} + } + command := agyRunCommandLine(s.ToolInfo) + if command == "" { + command = r.commands[s.StepIndex] + } else { + r.commands[s.StepIndex] = command + } + rec, ok := agyRunCommandRecord(s, command) + if ok && !r.recorded[s.StepIndex] { + r.recorded[s.StepIndex] = true + delete(r.commands, s.StepIndex) + } else { + ok = false + } + r.mu.Unlock() + if ok { + rec.Time = r.now() + _ = AppendRepeatFailureRecord(r.recordFile, rec) + } +} + // summarizeAgyToolParams renders tool parameters as one short line, with // CommandLine (run_command) first since it is the one operators read most. func summarizeAgyToolParams(info *agyToolInfo) string { diff --git a/src/crux/agent/agy_turn_test.go b/src/crux/agent/agy_turn_test.go index 832f2013dc..a13b612205 100644 --- a/src/crux/agent/agy_turn_test.go +++ b/src/crux/agent/agy_turn_test.go @@ -321,7 +321,7 @@ func TestAgyHeadlessLaunchShim_ExportsEnvAndConversationFile(t *testing.T) { launch := agyHeadlessFullLaunchCmd( shellEnvVar("HIVE_AGENT_MODE", "issues-only")+" "+shellEnvVar("HTTPS_PROXY", "http://127.0.0.1:18443")+" ", agyHeadlessLaunchCmd()) - kick := agyHeadlessTurnShellCommand(runner, "agy", "", "", "/tmp/p", false) + kick := agyHeadlessTurnShellCommand(runner, "agy", "", "", "/tmp/p", false, "") line := launch + "\n" + kick + "\n" + kick + "\n" + launch + "\n" + kick + "\n" out, err := exec.Command(bash, "--noprofile", "--norc", "-c", line).CombinedOutput() diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index 299319e1e8..c66e8309ff 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -107,7 +107,8 @@ func newestCopilotTranscriptActivityCached(home string, cache *transcriptSession } } var mod time.Time - if info, err := os.Stat(filepath.Join(root, name, "events.jsonl")); err == nil { + // Lstat: a link planted in the agent's HOME is never followed. + if info, err := os.Lstat(filepath.Join(root, name, "events.jsonl")); err == nil && info.Mode().IsRegular() { mod = info.ModTime() } if cache != nil { diff --git a/src/crux/agent/kick_transcript.go b/src/crux/agent/kick_transcript.go index e5c88cefa5..a91476b930 100644 --- a/src/crux/agent/kick_transcript.go +++ b/src/crux/agent/kick_transcript.go @@ -11,6 +11,7 @@ import ( "path/filepath" "regexp" "strings" + "syscall" "time" "unicode/utf8" @@ -153,7 +154,12 @@ func loadKickTranscript(t kickTranscriptTarget) (string, error) { if !t.since.IsZero() && h.LastActive.Before(t.since) { return "", fmt.Errorf("newest %s transcript %s predates the kick", t.backend, h.Transcript) } - f, err := os.Open(h.Transcript) + // The transcript sits in the agent's HOME: never follow a link planted + // there (the same discipline as the repeat-failure Copilot reader). + if info, err := os.Lstat(h.Transcript); err != nil || !info.Mode().IsRegular() { + return "", fmt.Errorf("%s transcript %s is not a regular file", t.backend, h.Transcript) + } + f, err := os.OpenFile(h.Transcript, os.O_RDONLY|syscall.O_NOFOLLOW, 0) if err != nil { return "", fmt.Errorf("opening %s transcript: %w", t.backend, err) } diff --git a/src/crux/agent/kick_transcript_copilot.go b/src/crux/agent/kick_transcript_copilot.go index 7e99e6e4c3..f8dfa4c157 100644 --- a/src/crux/agent/kick_transcript_copilot.go +++ b/src/crux/agent/kick_transcript_copilot.go @@ -26,6 +26,11 @@ type copilotTranscriptEvent struct { Success *bool `json:"success"` Result json.RawMessage `json:"result"` Error json.RawMessage `json:"error"` + // ShellExecution ({"exitCode":N}) and ToolTelemetry are read by the + // repeat-failure reader (repeat_failure_copilot.go); raw so a shape + // change never breaks the kick log. + ShellExecution json.RawMessage `json:"shellExecution"` + ToolTelemetry json.RawMessage `json:"toolTelemetry"` } `json:"data"` } diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 54089300cb..575c22f634 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -428,7 +428,9 @@ func (m *Manager) deliverAgyHeadlessKickLocked(agent *AgentProcess, message, tri } model = normalizeModelNameForBackend(model, "agy", false) // ClearOnKick maps to a new agy conversation: there is no TUI to /clear. - cmd := agyHeadlessTurnShellCommand(agyTurnRunnerBinary(), "agy", model, agent.Config.ReasoningEffort, promptFile, agent.Config.ClearOnKick) + // The repeat-failure switch is read per kick, so turning it off stops the + // next turn's records without a relaunch. + cmd := agyHeadlessTurnShellCommand(agyTurnRunnerBinary(), "agy", model, agent.Config.ReasoningEffort, promptFile, agent.Config.ClearOnKick, m.repeatFailureAgyRecordFile(agent)) if err := m.tmuxSendLiteralForAgent(agent, cmd); err != nil { return m.kickDeliveryFailedLocked(agent, "send text", err) } diff --git a/src/crux/agent/manager_launch.go b/src/crux/agent/manager_launch.go index 7c39e19d09..29e46afc51 100644 --- a/src/crux/agent/manager_launch.go +++ b/src/crux/agent/manager_launch.go @@ -190,6 +190,9 @@ func (m *Manager) launchInTmux(ctx context.Context, agent *AgentProcess) error { if advisorFlag := m.advisorLaunchFlag(agent, backend, isInference); advisorFlag != "" { launchCmd += advisorFlag } + // Repeat-failure check (#14550): omp loads Hive's record extension with + // its own --extension, whether or not the advisor is on. + launchCmd += m.repeatFailureOmpLaunchFlag(agent, backend) if bootstrapPrompt == "" && isInference { bootstrapPrompt = "You are an AI agent. Await further instructions." @@ -301,6 +304,7 @@ func (m *Manager) launchInTmux(ctx context.Context, agent *AgentProcess) error { // Repeat-failure check (#14550): write the BASH_ENV route the env prefix // below points at, or remove it when the check is now off. m.provisionRepeatFailureBashEnv(agent, backend) + m.provisionRepeatFailureAgyRecordDir(agent, backend) // NOTE on Copilot's "Confirm folder trust" modal (≥1.0.78): #4563 tried to // pre-seed config.json trusted_folders here so it never appears. That does @@ -1181,8 +1185,9 @@ var agyTurnRunnerBinary = func() string { // agyHeadlessTurnShellCommand is the line a kick types into a headless agy // pane: one `hive agy-turn` run, then the ready marker. The prompt stays in a -// file and reaches agy on stdin (see agy_turn.go for why not argv). -func agyHeadlessTurnShellCommand(hiveBinary, agyBinary, model, effort, promptFile string, newConversation bool) string { +// file and reaches agy on stdin (see agy_turn.go for why not argv). A +// non-empty recordFile turns on the repeat-failure route for this turn. +func agyHeadlessTurnShellCommand(hiveBinary, agyBinary, model, effort, promptFile string, newConversation bool, recordFile string) string { args := []string{shellQuote(hiveBinary), agyTurnSubcommand, "--agy", shellQuote(agyBinary)} if model != "" { args = append(args, "--model", shellQuote(model), "--effort", shellQuote(agyLaunchEffort(model, effort))) @@ -1192,6 +1197,9 @@ func agyHeadlessTurnShellCommand(hiveBinary, agyBinary, model, effort, promptFil if newConversation { args = append(args, "--new-conversation") } + if recordFile != "" { + args = append(args, "--"+agyRepeatFailureRecordFlag, shellQuote(recordFile)) + } return fmt.Sprintf("printf '\\n%s\\n'; %s; echo %s", agyHeadlessRunningMarker, strings.Join(args, " "), agyHeadlessReadyMarker) } diff --git a/src/crux/agent/repeat_failure_agy.go b/src/crux/agent/repeat_failure_agy.go new file mode 100644 index 0000000000..7f5a085518 --- /dev/null +++ b/src/crux/agent/repeat_failure_agy.go @@ -0,0 +1,98 @@ +package agent + +import ( + "os" + "regexp" + "strconv" + "strings" +) + +// agy route for the repeat-failure check (hivecommons/hive#14550, C1, C3, +// C14; issue #14750). +// +// agy is a closed binary whose commands never reach the neutral BASH_ENV +// route. In the headless launch every kick runs one `hive agy-turn`, which +// already parses agy's stream-json steps: a `tool` step for `run_command` +// carries the command in Parameters["CommandLine"]. While the check is on the +// kick passes `--repeat-failure-record ` and the runner appends one +// record per run_command step that ends (DONE, or a failed state) to the +// agent's raw record file. agy reports no exit status field; when the tool +// output states one ("exit code: 2") it is recorded, otherwise only the step +// state decides failed/succeeded. The interactive TUI launch bypasses the +// runner and has no route. Nothing is written into agy's own files; with the +// check off the kick passes no flag and the runner writes nothing (C14). + +// RepeatFailureRouteAgyTurn names the agy-turn runner in a record's Route. +const RepeatFailureRouteAgyTurn = "agy_turn" + +// agyRepeatFailureRecordFlag is the agy-turn flag naming the raw record file. +const agyRepeatFailureRecordFlag = "repeat-failure-record" + +// agyRunCommandExitCode matches an exit status stated in run_command output. +var agyRunCommandExitCode = regexp.MustCompile(`(?i)\bexit(?:ed with)?[ _]?(?:code|status)\b[\s:=]*(-?\d+)`) + +// agyRunCommandRecord builds the record of a run_command step that ended, or +// reports false for any other step. command is the step's CommandLine, taken +// from its earlier ACTIVE update when the final one omits it. +func agyRunCommandRecord(s *agyStepUpdate, command string) (RepeatFailureRecord, bool) { + rec := RepeatFailureRecord{Agent: os.Getenv("HIVE_AGENT"), Backend: "agy", RawCommand: command, Route: RepeatFailureRouteAgyTurn} + switch strings.ToUpper(strings.TrimSpace(s.State)) { + case "DONE": + case "ERROR", "FAILED", "CANCELED", "CANCELLED": + rec.Failed = true + default: + return rec, false + } + if strings.TrimSpace(command) == "" { + return rec, false + } + if s.ToolInfo != nil { + if m := agyRunCommandExitCode.FindAllStringSubmatch(s.ToolInfo.Output, -1); len(m) > 0 { + if code, err := strconv.Atoi(m[len(m)-1][1]); err == nil { + rec.ExitCode = &code + rec.Failed = code != 0 + } + } + } + return rec, true +} + +// agyRunCommandLine returns a run_command step's CommandLine, or "". +func agyRunCommandLine(info *agyToolInfo) string { + if info == nil { + return "" + } + cmd, _ := info.Parameters["CommandLine"].(string) + return cmd +} + +// repeatFailureAgyRecordFile is the record file a headless agy kick passes to +// the runner, or "" when the check is off (C14). Lock-free. +func (m *Manager) repeatFailureAgyRecordFile(agent *AgentProcess) string { + if agent == nil || !m.repeatFailureEnabled() { + return "" + } + home := AgentHome(agent.Name, agent.UID, "agy") + if home == "" { + return "" + } + return RepeatFailureRecordPath(home) +} + +// provisionRepeatFailureAgyRecordDir creates the agent's 0700 record +// directory at a headless agy launch while the check is on, so the runner, +// which runs as the agent, can append to the record file. Best-effort: a +// failure is logged and the runner's appends then fail open. +func (m *Manager) provisionRepeatFailureAgyRecordDir(agent *AgentProcess, backend string) { + if !strings.EqualFold(strings.TrimSpace(backend), "agy") || !agyHeadlessEnabled() { + return + } + recordPath := m.repeatFailureAgyRecordFile(agent) + if recordPath == "" { + return + } + if err := ensureRepeatFailureRecordDirAs(m.agentExecUserSpec(agent), recordPath); err != nil { + m.logger.Warn("repeat-failure: creating the agy record dir failed; agent launches without the route", + "agent", agent.Name, "path", recordPath, "error", err) + } +} diff --git a/src/crux/agent/repeat_failure_agy_test.go b/src/crux/agent/repeat_failure_agy_test.go new file mode 100644 index 0000000000..3ff3917a8e --- /dev/null +++ b/src/crux/agent/repeat_failure_agy_test.go @@ -0,0 +1,128 @@ +package agent + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +func rfReadRecords(t *testing.T, path string) []RepeatFailureRecord { + t.Helper() + f, err := os.Open(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + t.Fatal(err) + } + defer f.Close() + recs, invalid, err := readRepeatFailureRecords(f) + if err != nil || invalid != 0 { + t.Fatalf("reading %s: %d invalid lines, %v", path, invalid, err) + } + return recs +} + +// TestRepeatFailureAgyTurnRecordsRunCommand: with --repeat-failure-record the +// agy-turn runner appends one record for the run_command step agy streamed +// (no exit status in agy's output: DONE means succeeded), and without the +// flag it writes nothing (C14). +func TestRepeatFailureAgyTurnRecordsRunCommand(t *testing.T) { + fake := newFakeAgy(t) + dir := t.TempDir() + prompt := filepath.Join(dir, "prompt.txt") + if err := os.WriteFile(prompt, []byte("go"), 0o644); err != nil { + t.Fatal(err) + } + record := filepath.Join(dir, "records.jsonl") + + if code, _, stderr := runAgyTurnForTest(t, "--agy", fake.binary, "--prompt-file", prompt); code != 0 { + t.Fatalf("turn without the flag: exit %d, stderr %s", code, stderr) + } + if _, err := os.Stat(record); !os.IsNotExist(err) { + t.Fatalf("check off: the runner wrote a record file (err %v)", err) + } + + t.Setenv("HIVE_AGENT", "scanner") + if code, _, stderr := runAgyTurnForTest(t, "--agy", fake.binary, "--prompt-file", prompt, "--repeat-failure-record", record); code != 0 { + t.Fatalf("turn with the flag: exit %d, stderr %s", code, stderr) + } + recs := rfReadRecords(t, record) + if len(recs) != 1 { + t.Fatalf("records = %+v, want one for the run_command step", recs) + } + rec := recs[0] + if rec.RawCommand != "gh pr list" || rec.Failed || rec.ExitCode != nil || rec.Route != RepeatFailureRouteAgyTurn || + rec.Backend != "agy" || rec.Agent != "scanner" || rec.Time.IsZero() { + t.Fatalf("record = %+v", rec) + } +} + +// TestRepeatFailureAgyRunCommandSteps covers the step rules: the command of a +// final update that omits it comes from the step's ACTIVE update, an exit +// status stated in the output wins, a failed state is a failure, other tools +// and repeated final updates add nothing. +func TestRepeatFailureAgyRunCommandSteps(t *testing.T) { + record := filepath.Join(t.TempDir(), "records.jsonl") + r := newAgyTurnRenderer(io.Discard, time.Now()) + r.recordFile = record + for _, line := range []string{ + `{"event":"step_update","step_update":{"step_index":1,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","parameters":{"CommandLine":"ls /nonexistent"}}}}`, + `{"event":"step_update","step_update":{"step_index":1,"state":"DONE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","output":"ls: cannot access '/nonexistent'\nExit code: 2\n"}}}`, + `{"event":"step_update","step_update":{"step_index":1,"state":"DONE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","output":"Exit code: 2"}}}`, + `{"event":"step_update","step_update":{"step_index":2,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","parameters":{"CommandLine":"make test"}}}}`, + `{"event":"step_update","step_update":{"step_index":2,"state":"ERROR","step_type":"tool","tool_name":"run_command"}}`, + `{"event":"step_update","step_update":{"step_index":3,"state":"DONE","step_type":"tool","tool_name":"view_file","tool_info":{"name":"view_file","parameters":{"CommandLine":"not a command"}}}}`, + `{"event":"step_update","step_update":{"step_index":4,"state":"ACTIVE","step_type":"tool","tool_name":"run_command","tool_info":{"name":"run_command","parameters":{"CommandLine":"sleep 600"}}}}`, + } { + r.handleLine([]byte(line)) + } + recs := rfReadRecords(t, record) + if len(recs) != 2 { + t.Fatalf("records = %+v, want ls and make only", recs) + } + if rec := recs[0]; rec.RawCommand != "ls /nonexistent" || rec.ExitCode == nil || *rec.ExitCode != 2 || !rec.Failed { + t.Errorf("ls: %+v, want exit 2 from the output", rec) + } + if rec := recs[1]; rec.RawCommand != "make test" || rec.ExitCode != nil || !rec.Failed { + t.Errorf("make: %+v, want failed without an exit code", rec) + } +} + +// TestRepeatFailureAgyKickPassesRecordFile: a headless kick passes the record +// file only while the check is on, read at each kick. +func TestRepeatFailureAgyKickPassesRecordFile(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "agy", Model: "m"}}, discardLogger(), ProjectContext{}) + ap := &AgentProcess{Name: "scanner", Config: config.AgentConfig{Backend: "agy", Model: "m"}} + on := false + m.SetRepeatFailureEnabledResolver(func() bool { return on }) + if got := m.repeatFailureAgyRecordFile(ap); got != "" { + t.Fatalf("check off: record file = %q, want none", got) + } + if cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "", "/p", false, m.repeatFailureAgyRecordFile(ap)); strings.Contains(cmd, "--repeat-failure-record") { + t.Fatalf("check off: kick passes the record flag: %q", cmd) + } + on = true + want := RepeatFailureRecordPath(home) + if got := m.repeatFailureAgyRecordFile(ap); got != want { + t.Fatalf("check on: record file = %q, want %q", got, want) + } + cmd := agyHeadlessTurnShellCommand("hive", "agy", "", "", "/p", false, m.repeatFailureAgyRecordFile(ap)) + if !strings.Contains(cmd, "--repeat-failure-record "+shellQuote(want)) { + t.Fatalf("check on: kick lacks the record flag: %q", cmd) + } + + t.Setenv(agyLaunchModeEnv, agyLaunchModeHeadless) + m.provisionRepeatFailureAgyRecordDir(ap, "agy") + info, err := os.Stat(filepath.Dir(want)) + if err != nil || info.Mode().Perm() != repeatFailureRecordDirMode { + t.Fatalf("record dir = %v, %v; want mode %o", info, err, repeatFailureRecordDirMode) + } +} diff --git a/src/crux/agent/repeat_failure_bash_env.go b/src/crux/agent/repeat_failure_bash_env.go index 96af651bb0..620bea5e61 100644 --- a/src/crux/agent/repeat_failure_bash_env.go +++ b/src/crux/agent/repeat_failure_bash_env.go @@ -251,39 +251,50 @@ func (m *Manager) provisionRepeatFailureBashEnv(agent *AgentProcess, backend str } recordPath := RepeatFailureRecordPath(home) content := renderRepeatFailureBashEnv(agent.Name, backend, route, recordPath, chain) - if err := writeRepeatFailureBashEnv(m.agentExecUserSpec(agent), recordPath, script, content); err != nil { + if err := writeRepeatFailureRouteFile(m.agentExecUserSpec(agent), recordPath, script, content); err != nil { m.logger.Warn("repeat-failure: writing the BASH_ENV route failed; agent launches without it", "agent", agent.Name, "path", script, "error", err) } } -// writeRepeatFailureBashEnv creates the 0700 record directory and writes the -// script into it, as the agent's user when there is one (the same su-exec -// discipline as the advisor hook files) and directly otherwise. -func writeRepeatFailureBashEnv(userSpec, recordPath, script string, content []byte) error { +// writeRepeatFailureRouteFile creates the 0700 record directory and writes a +// route file (the BASH_ENV script, the omp extension) into it, as the agent's +// user when there is one (the same su-exec discipline as the advisor hook +// files) and directly otherwise. +func writeRepeatFailureRouteFile(userSpec, recordPath, path string, content []byte) error { + if err := ensureRepeatFailureRecordDirAs(userSpec, recordPath); err != nil { + return err + } if userSpec == "" { - if err := EnsureRepeatFailureRecordDir(recordPath, 0); err != nil { - return err - } - return writeRepeatFailureBashEnvFile(script, content) + return writeRepeatFailureRouteFileNoFollow(path, content) + } + return writeFileAsUser(userSpec, path, content) +} + +// ensureRepeatFailureRecordDirAs creates the 0700 record directory as the +// agent's user, or directly when there is none, so a producer running as the +// agent can append to the record file inside it. +func ensureRepeatFailureRecordDirAs(userSpec, recordPath string) error { + if userSpec == "" { + return EnsureRepeatFailureRecordDir(recordPath, 0) } dir := filepath.Dir(recordPath) if out, err := exec.Command("su-exec", userSpec, "sh", "-c", `umask 077 && mkdir -p "$1" && chmod 700 "$1"`, "sh", dir).CombinedOutput(); err != nil { return outputErr("creating "+dir+" as "+userSpec, err, out) } - return writeFileAsUser(userSpec, script, content) + return nil } -// writeRepeatFailureBashEnvFile replaces script atomically, never following a -// planted link. -func writeRepeatFailureBashEnvFile(script string, content []byte) error { - tmp := script + ".new" +// writeRepeatFailureRouteFileNoFollow replaces the route file at path +// atomically, never following a planted link. +func writeRepeatFailureRouteFileNoFollow(path string, content []byte) error { + tmp := path + ".new" _ = os.Remove(tmp) if err := writeFileNoFollow(tmp, content, repeatFailureRecordFileMode); err != nil { _ = os.Remove(tmp) return err } - if err := os.Rename(tmp, script); err != nil { + if err := os.Rename(tmp, path); err != nil { _ = os.Remove(tmp) return err } @@ -306,29 +317,38 @@ func writeFileNoFollow(path string, content []byte, mode os.FileMode) error { // header) and drops the BASH_ENV Hive put into an existing tmux session, so a // switched-off check leaves nothing behind at the next launch. func (m *Manager) removeRepeatFailureBashEnv(agent *AgentProcess, script string) { + if m.removeRepeatFailureFile(agent, script, repeatFailureBashEnvHeader) { + m.clearRepeatFailureSessionEnv(agent, script) + } +} + +// removeRepeatFailureFile deletes a route file Hive wrote, recognised by its +// header line, and reports whether there was one. A file Hive did not write +// is left alone. +func (m *Manager) removeRepeatFailureFile(agent *AgentProcess, path, header string) bool { userSpec := m.agentExecUserSpec(agent) var existing []byte var err error if userSpec == "" { - existing, err = os.ReadFile(script) + existing, err = os.ReadFile(path) } else { - existing, err = readFileAsUser(userSpec, script) + existing, err = readFileAsUser(userSpec, path) } - if err != nil || !strings.HasPrefix(string(existing), repeatFailureBashEnvHeader) { - return + if err != nil || !strings.HasPrefix(string(existing), header) { + return false } if userSpec == "" { - err = os.Remove(script) + err = os.Remove(path) if errors.Is(err, os.ErrNotExist) { err = nil } } else { - err = exec.Command("su-exec", userSpec, "rm", "-f", script).Run() + err = exec.Command("su-exec", userSpec, "rm", "-f", path).Run() } if err != nil { - m.logger.Warn("repeat-failure: removing the stale BASH_ENV route failed", "agent", agent.Name, "path", script, "error", err) + m.logger.Warn("repeat-failure: removing a stale route file failed", "agent", agent.Name, "path", path, "error", err) } - m.clearRepeatFailureSessionEnv(agent, script) + return true } // clearRepeatFailureSessionEnv removes BASH_ENV from the agent's tmux session diff --git a/src/crux/agent/repeat_failure_copilot.go b/src/crux/agent/repeat_failure_copilot.go new file mode 100644 index 0000000000..4911f517e6 --- /dev/null +++ b/src/crux/agent/repeat_failure_copilot.go @@ -0,0 +1,343 @@ +package agent + +import ( + "bytes" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +// Copilot route for the repeat-failure check (hivecommons/hive#14550, C1, C3, +// C12, C14; issue #14746). +// +// Copilot runs every shell tool call in one long-lived Bash session per shell +// id, so the neutral BASH_ENV route never sees its commands. Copilot already +// logs each call in its session log +// (~/.copilot/session-state//events.jsonl under the agent's COPILOT_HOME): +// a `tool.execution_start` with the command, then a `tool.execution_complete` +// whose `shellExecution.exitCode` is the command's exit status. Copilot sets +// `success: true` for a command that exited non-zero, so the exit code wins +// over `success`. +// +// Hive reads those logs itself on every collection while the check is on and +// appends one shared record per completed shell call to the agent's raw +// record file, which the collector then drains like any other route. Nothing +// is written into Copilot's own files, so with the check off Hive adds +// nothing (C14). Each session log is read from where the previous collection +// stopped; a log that already existed when the reader started is read from its +// end, so a Hive restart does not replay history, and a toolCallId is recorded +// at most once even when a log is rewritten and read again. +// +// The logs live under the agent's HOME and Hive reads them as itself (root in +// multi-uid mode) while the extracted commands land in a file the agent can +// read, so the reader never follows a link the agent could plant: the +// session-state tree, every session directory and every events.jsonl are +// Lstat-checked, the log is opened O_NOFOLLOW, and in multi-uid mode a log +// not owned by the agent's uid is skipped (audit F12 discipline, as on the +// record-file side). + +// RepeatFailureRouteCopilotEvents names the Copilot session-log reader in a +// record's Route. +const RepeatFailureRouteCopilotEvents = "copilot_events" + +const ( + // copilotRecordReadMax bounds how much of one session log a collection + // reads; the rest is read on the next collection. + copilotRecordReadMax = 32 << 20 + // copilotRecordSeenMax bounds the toolCallIds remembered per session. + copilotRecordSeenMax = 8192 +) + +// copilotShellTools are the Copilot tools that run a shell command. The +// read_/write_/stop_ companions only talk to a running shell. +var copilotShellTools = map[string]bool{"bash": true, "powershell": true} + +// copilotRecordSession is the reader's position in one session log. +type copilotRecordSession struct { + offset int64 + // commands maps the toolCallId of a shell call that has started but not + // completed to its command. + commands map[string]string + // seen holds the toolCallIds already recorded. + seen map[string]bool +} + +// copilotRecordReader follows one agent's Copilot session logs. +type copilotRecordReader struct { + started bool + sessions map[string]*copilotRecordSession +} + +func newCopilotRecordReader() *copilotRecordReader { + return &copilotRecordReader{sessions: map[string]*copilotRecordSession{}} +} + +// scan reads what was appended to every session log under home since the last +// scan. emit receives one record per completed shell call and observe the +// shell activity (finished false at start, true at completion). When uid > 0 +// only logs owned by uid are read. +func (r *copilotRecordReader) scan(home string, uid int, emit func(RepeatFailureRecord), observe func(finished bool, at time.Time)) { + root := filepath.Join(home, ".copilot", "session-state") + if !copilotRecordRealDir(filepath.Join(home, ".copilot")) || !copilotRecordRealDir(root) { + r.started = true + return + } + entries, err := os.ReadDir(root) + if err != nil { + r.started = true + return + } + present := make(map[string]bool, len(entries)) + for _, entry := range entries { + // A symlinked session directory is not a directory to ReadDir. + if !entry.IsDir() { + continue + } + name := entry.Name() + present[name] = true + info, err := os.Lstat(filepath.Join(root, name, "events.jsonl")) + if err != nil || !info.Mode().IsRegular() || !copilotRecordOwnedBy(info, uid) { + continue + } + sess := r.sessions[name] + if sess == nil { + sess = &copilotRecordSession{commands: map[string]string{}, seen: map[string]bool{}} + if !r.started { + sess.offset = info.Size() + } + r.sessions[name] = sess + } + if info.Size() < sess.offset { + // Rewritten: read it again; seen keeps each call to one record. + sess.offset = 0 + } + if info.Size() > sess.offset { + r.readSession(filepath.Join(root, name, "events.jsonl"), sess, info.Size(), emit, observe) + } + } + for name := range r.sessions { + if !present[name] { + delete(r.sessions, name) + } + } + r.started = true +} + +// copilotRecordRealDir reports whether path is a directory and not a link. +func copilotRecordRealDir(path string) bool { + info, err := os.Lstat(path) + return err == nil && info.IsDir() +} + +// copilotRecordOwnedBy reports whether info's file belongs to uid. Any uid +// passes with uid <= 0 (single-uid mode, where Hive and the agents share one +// user) and nothing passes on a platform without Stat_t. +func copilotRecordOwnedBy(info os.FileInfo, uid int) bool { + if uid <= 0 { + return true + } + st, ok := info.Sys().(*syscall.Stat_t) + return ok && int(st.Uid) == uid +} + +func (r *copilotRecordReader) readSession(path string, sess *copilotRecordSession, size int64, emit func(RepeatFailureRecord), observe func(bool, time.Time)) { + f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return + } + defer f.Close() + // The log was Lstat-checked a moment ago; make sure the descriptor still + // holds a regular file and not something swapped in since. + if info, err := f.Stat(); err != nil || !info.Mode().IsRegular() { + return + } + n := min(size-sess.offset, copilotRecordReadMax) + buf := make([]byte, n) + read, err := f.ReadAt(buf, sess.offset) + if err != nil && err != io.EOF { + return + } + buf = buf[:read] + end := bytes.LastIndexByte(buf, '\n') + if end < 0 { + if int64(read) == copilotRecordReadMax { + // One line larger than a whole read: skip it. + sess.offset += int64(read) + } + return + } + sess.offset += int64(end + 1) + for _, line := range bytes.Split(buf[:end], []byte{'\n'}) { + sess.handleLine(line, emit, observe) + } +} + +// copilotShellArgs is the argument shape of Copilot's shell tools. +type copilotShellArgs struct { + Command string `json:"command"` +} + +// copilotShellTelemetry is the part of tool.execution_complete's +// toolTelemetry that says whether the call returned before the command ended. +type copilotShellTelemetry struct { + Properties struct { + ExecutionMode any `json:"executionMode"` + Detached any `json:"detached"` + } `json:"properties"` +} + +func copilotShellCommand(args json.RawMessage) string { + var a copilotShellArgs + if json.Unmarshal(args, &a) != nil { + return "" + } + return a.Command +} + +func (sess *copilotRecordSession) start(id, tool string, args json.RawMessage) bool { + if id == "" || !copilotShellTools[tool] || sess.seen[id] { + return false + } + if cmd := copilotShellCommand(args); strings.TrimSpace(cmd) != "" { + sess.commands[id] = cmd + return true + } + return false +} + +func (sess *copilotRecordSession) handleLine(line []byte, emit func(RepeatFailureRecord), observe func(bool, time.Time)) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return + } + var ev copilotTranscriptEvent + if json.Unmarshal(line, &ev) != nil { + return + } + ts := parseTranscriptTime(ev.Timestamp) + switch ev.Type { + case "assistant.message": + for _, req := range ev.Data.ToolRequests { + sess.start(req.ToolCallID, req.Name, req.Arguments) + } + case "tool.execution_start": + if sess.start(ev.Data.ToolCallID, ev.Data.ToolName, ev.Data.Arguments) { + observe(false, ts) + } + case "tool.execution_complete": + id := ev.Data.ToolCallID + cmd, ok := sess.commands[id] + if !ok { + return + } + delete(sess.commands, id) + if len(sess.seen) >= copilotRecordSeenMax { + sess.seen = map[string]bool{} + } + sess.seen[id] = true + observe(true, ts) + rec, ok := copilotShellRecord(ev, cmd, ts) + if ok { + emit(rec) + } + } +} + +// copilotShellRecord builds the record of one completed shell call. A call +// that returned while its command keeps running (async or detached) has no +// result yet and is not recorded. +func copilotShellRecord(ev copilotTranscriptEvent, cmd string, ts time.Time) (RepeatFailureRecord, bool) { + rec := RepeatFailureRecord{Backend: "copilot", Time: ts, RawCommand: cmd, Route: RepeatFailureRouteCopilotEvents} + var shell struct { + ExitCode *float64 `json:"exitCode"` + } + if len(ev.Data.ShellExecution) > 0 && json.Unmarshal(ev.Data.ShellExecution, &shell) == nil && shell.ExitCode != nil { + code := int(*shell.ExitCode) + rec.ExitCode = &code + rec.Failed = code != 0 + return rec, true + } + var tel copilotShellTelemetry + if len(ev.Data.ToolTelemetry) > 0 && json.Unmarshal(ev.Data.ToolTelemetry, &tel) == nil { + if mode, _ := tel.Properties.ExecutionMode.(string); strings.EqualFold(mode, "async") { + return rec, false + } + switch d := tel.Properties.Detached.(type) { + case bool: + if d { + return rec, false + } + case string: + if strings.EqualFold(d, "true") { + return rec, false + } + } + } + if ev.Data.Success == nil { + return rec, false + } + rec.Failed = !*ev.Data.Success + return rec, true +} + +// readCopilotRepeatFailureRecords runs every Copilot agent's session-log +// reader and appends the records to the agent's raw record file, so the +// collection that follows picks them up. Only called while the check is on. +func (m *Manager) readCopilotRepeatFailureRecords() { + paths := m.repeatFailureRecordPaths() + type copilotAgent struct { + name string + uid int + home string + } + var agents []copilotAgent + m.mu.RLock() + for name, a := range m.agents { + if effectiveBackend(a) != "copilot" || paths[name] == "" { + continue + } + agents = append(agents, copilotAgent{name: name, uid: a.UID, home: AgentHome(name, a.UID, "copilot")}) + } + m.mu.RUnlock() + + mon := &m.repeatFailure + mon.mu.Lock() + readers := make(map[string]*copilotRecordReader, len(agents)) + for _, a := range agents { + r := mon.copilot[a.name] + if r == nil { + r = newCopilotRecordReader() + } + readers[a.name] = r + } + mon.copilot = readers + mon.mu.Unlock() + + for _, a := range agents { + var records []RepeatFailureRecord + readers[a.name].scan(a.home, a.uid, func(rec RepeatFailureRecord) { + rec.Agent = a.name + records = append(records, rec) + }, func(finished bool, at time.Time) { + m.ObserveRepeatFailureTool(a.name, RepeatFailureToolShell, finished, at) + }) + if len(records) == 0 { + continue + } + path := paths[a.name] + if err := EnsureRepeatFailureRecordDir(path, a.uid); err != nil { + m.logger.Warn("repeat-failure: cannot create the record dir for Copilot records", "agent", a.name, "error", err) + continue + } + for _, rec := range records { + if err := AppendRepeatFailureRecord(path, rec); err != nil { + m.logger.Warn("repeat-failure: writing a Copilot command record failed", "agent", a.name, "error", err) + break + } + } + } +} diff --git a/src/crux/agent/repeat_failure_copilot_test.go b/src/crux/agent/repeat_failure_copilot_test.go new file mode 100644 index 0000000000..ea608a24f4 --- /dev/null +++ b/src/crux/agent/repeat_failure_copilot_test.go @@ -0,0 +1,304 @@ +package agent + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +const rfCopilotHarnessEvents = "testdata/repeat_failure/copilot/events.jsonl" + +func rfCopilotFixture(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(rfCopilotHarnessEvents) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func rfCopilotWriteSession(t *testing.T, home, session, content string) string { + t.Helper() + dir := filepath.Join(home, ".copilot", "session-state", session) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "events.jsonl") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +type rfCopilotScan struct { + uid int + records []RepeatFailureRecord + starts int + finishes int +} + +func (s *rfCopilotScan) run(r *copilotRecordReader, home string) { + r.scan(home, s.uid, func(rec RepeatFailureRecord) { s.records = append(s.records, rec) }, func(finished bool, _ time.Time) { + if finished { + s.finishes++ + } else { + s.starts++ + } + }) +} + +// TestRepeatFailureCopilotHarness is C-AC1 for copilot: the session log the +// real Copilot CLI wrote for `ls /nonexistent` (recorded by +// testdata/repeat_failure/copilot/record.sh) yields exactly one failing record +// with the exit code, although Copilot marks the call success=true, and the +// shell call is reported to the safeguard once started and once finished. +func TestRepeatFailureCopilotHarness(t *testing.T) { + version := rfCopilotFixtureVersion(t) + if !strings.Contains(version, "Copilot CLI") { + t.Fatalf("harness evidence lacks `copilot --version`: %q", version) + } + if row := rfCoverageRow(t, "copilot"); !strings.Contains(row.Evidence, version) { + t.Fatalf("coverage evidence %q does not carry the harness version %q; update repeat_failure_coverage.go and the docs table", row.Evidence, version) + } + home := t.TempDir() + r := newCopilotRecordReader() + var s rfCopilotScan + s.run(r, home) + rfCopilotWriteSession(t, home, "harness", rfCopilotFixture(t)) + s.run(r, home) + + if len(s.records) != 1 { + t.Fatalf("records = %+v, want exactly one", s.records) + } + rec := s.records[0] + if rec.RawCommand != "ls /nonexistent" || rec.ExitCode == nil || *rec.ExitCode != 2 || !rec.Failed || + rec.Route != RepeatFailureRouteCopilotEvents || rec.Backend != "copilot" || rec.Time.IsZero() { + t.Fatalf("record = %+v, want `ls /nonexistent` failed with exit 2 on route %s", rec, RepeatFailureRouteCopilotEvents) + } + if s.starts != 1 || s.finishes != 1 { + t.Fatalf("shell activity: %d starts, %d finishes; want 1 and 1", s.starts, s.finishes) + } +} + +// rfCopilotFixtureVersion is the `copilot --version` line the harness +// recorded, without its trailing period. +func rfCopilotFixtureVersion(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("testdata/repeat_failure/copilot/version.txt") + if err != nil { + t.Fatal(err) + } + return strings.TrimSuffix(strings.TrimSpace(string(data)), ".") +} + +// TestRepeatFailureCopilotNeverFollowsLinks: the reader runs as Hive over +// files the agent controls, so a session-state tree, a session directory or +// an events.jsonl that is a link (here, to another agent's log) is skipped, +// and in multi-uid mode so is a log owned by anyone but the agent. +func TestRepeatFailureCopilotNeverFollowsLinks(t *testing.T) { + fixture := rfCopilotFixture(t) + other := t.TempDir() + otherLog := rfCopilotWriteSession(t, other, "victim", "") + otherState := filepath.Dir(filepath.Dir(otherLog)) + + home := t.TempDir() + state := filepath.Join(home, ".copilot", "session-state") + if err := os.MkdirAll(state, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Dir(otherLog), filepath.Join(state, "linked-dir")); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(state, "linked-file"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(otherLog, filepath.Join(state, "linked-file", "events.jsonl")); err != nil { + t.Fatal(err) + } + r := newCopilotRecordReader() + var s rfCopilotScan + s.run(r, home) + if err := os.WriteFile(otherLog, []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + s.run(r, home) + if len(s.records) != 0 || s.starts != 0 { + t.Fatalf("a linked session dir or log was read: %+v", s.records) + } + + // The whole session-state tree as a link. + linkedHome := t.TempDir() + if err := os.MkdirAll(filepath.Join(linkedHome, ".copilot"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(otherState, filepath.Join(linkedHome, ".copilot", "session-state")); err != nil { + t.Fatal(err) + } + r2 := newCopilotRecordReader() + var s2 rfCopilotScan + s2.run(r2, linkedHome) + if err := os.WriteFile(otherLog, []byte(fixture+fixture), 0o600); err != nil { + t.Fatal(err) + } + s2.run(r2, linkedHome) + if len(s2.records) != 0 { + t.Fatalf("a linked session-state tree was read: %+v", s2.records) + } + + // Owner check: the agent's own uid reads, any other uid is skipped. + ownHome := t.TempDir() + path := rfCopilotWriteSession(t, ownHome, "s", "") + r3 := newCopilotRecordReader() + wrong := rfCopilotScan{uid: os.Getuid() + 1} + wrong.run(r3, ownHome) + if err := os.WriteFile(path, []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + wrong.run(r3, ownHome) + if len(wrong.records) != 0 { + t.Fatalf("a log owned by another uid was read: %+v", wrong.records) + } + right := rfCopilotScan{uid: os.Getuid()} + right.run(r3, ownHome) + if len(right.records) != 1 { + t.Fatalf("the agent's own log was not read as uid %d: %+v", os.Getuid(), right.records) + } +} + +// TestRepeatFailureCopilotOneRecordPerToolCall: re-reading a log, appending a +// duplicate of a recorded call or rewriting the log never records a +// toolCallId twice, and a log that existed when the reader started is not +// replayed. +func TestRepeatFailureCopilotOneRecordPerToolCall(t *testing.T) { + home := t.TempDir() + fixture := rfCopilotFixture(t) + rfCopilotWriteSession(t, home, "old", fixture) + r := newCopilotRecordReader() + var s rfCopilotScan + s.run(r, home) + if len(s.records) != 0 { + t.Fatalf("history present at start was replayed: %+v", s.records) + } + + path := rfCopilotWriteSession(t, home, "new", fixture) + s.run(r, home) + s.run(r, home) + if err := os.WriteFile(path, []byte(fixture+fixture), 0o600); err != nil { + t.Fatal(err) + } + s.run(r, home) + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + s.run(r, home) + if err := os.WriteFile(path, []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + s.run(r, home) + if len(s.records) != 1 { + t.Fatalf("records = %d, want one per toolCallId across re-reads: %+v", len(s.records), s.records) + } + + // A partial last line waits for its newline. + second := strings.ReplaceAll(fixture, "toolu_", "toolu_second_") + cut := strings.Index(second, "\n") + 20 + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + _, _ = f.WriteString(second[:cut]) + s.run(r, home) + _, _ = f.WriteString(second[cut:]) + _ = f.Close() + s.run(r, home) + if len(s.records) != 2 { + t.Fatalf("records = %d after a line written in two parts, want 2", len(s.records)) + } +} + +// TestRepeatFailureCopilotResult covers the result rules: the exit code wins +// over success, a call without one falls back to success, and a call that +// returned while its command keeps running (async, detached) or that is not a +// shell call is not recorded. +func TestRepeatFailureCopilotResult(t *testing.T) { + start := func(id, tool string) string { + return `{"type":"tool.execution_start","data":{"toolCallId":"` + id + `","toolName":"` + tool + `","arguments":{"command":"make ` + id + `"}},"timestamp":"2026-10-28T01:00:00Z"}` + "\n" + } + complete := func(id, data string) string { + return `{"type":"tool.execution_complete","data":{"toolCallId":"` + id + `",` + data + `},"timestamp":"2026-10-28T01:00:01Z"}` + "\n" + } + log := start("ok", "bash") + complete("ok", `"shellExecution":{"exitCode":0},"success":true`) + + start("nocode", "bash") + complete("nocode", `"success":false,"toolTelemetry":{"properties":{"executionMode":"sync"}}`) + + start("async", "bash") + complete("async", `"success":true,"toolTelemetry":{"properties":{"executionMode":"async"}}`) + + start("detached", "bash") + complete("detached", `"success":true,"toolTelemetry":{"properties":{"detached":true}}`) + + start("view", "view") + complete("view", `"success":false`) + + `{"type":"assistant.message","data":{"toolRequests":[{"toolCallId":"req","name":"bash","arguments":{"command":"make req"}}]}}` + "\n" + + complete("req", `"shellExecution":{"exitCode":1},"success":true`) + + "not json\n" + + home := t.TempDir() + r := newCopilotRecordReader() + var s rfCopilotScan + s.run(r, home) + rfCopilotWriteSession(t, home, "s", log) + s.run(r, home) + + got := map[string]RepeatFailureRecord{} + for _, rec := range s.records { + got[strings.TrimPrefix(rec.RawCommand, "make ")] = rec + } + if len(got) != 3 { + t.Fatalf("records = %+v, want ok, nocode and req only", s.records) + } + if rec := got["ok"]; rec.ExitCode == nil || *rec.ExitCode != 0 || rec.Failed { + t.Errorf("ok: %+v", rec) + } + if rec := got["nocode"]; rec.ExitCode != nil || !rec.Failed { + t.Errorf("nocode: %+v, want failed without an exit code", rec) + } + if rec := got["req"]; rec.ExitCode == nil || *rec.ExitCode != 1 || !rec.Failed { + t.Errorf("req (command from toolRequests): %+v", rec) + } +} + +// TestRepeatFailureCopilotRoute runs the route end to end through the +// collector: repeated failures in a Copilot agent's session log raise the +// note, the covered agent shows the check on, and with the check off the +// reader writes no record file. +func TestRepeatFailureCopilotRoute(t *testing.T) { + m := rfStatusManager(t, "copilot") + home := os.Getenv("HOME") + on := false + m.SetRepeatFailureEnabledResolver(func() bool { return on }) + m.SetRepeatFailureConfigResolver(func() *config.RepeatFailureConfig { return &config.RepeatFailureConfig{Enabled: true} }) + now := time.Now() + fixture := rfCopilotFixture(t) + + var log strings.Builder + for i := range 5 { + log.WriteString(strings.ReplaceAll(fixture, "toolu_", "toolu_"+strconv.Itoa(i)+"_")) + } + m.CollectRepeatFailures(now) + rfCopilotWriteSession(t, home, "off", log.String()) + m.CollectRepeatFailures(now.Add(time.Second)) + if _, err := os.Stat(filepath.Dir(RepeatFailureRecordPath(home))); !os.IsNotExist(err) { + t.Fatalf("check off: the copilot route created the record dir (err %v)", err) + } + + on = true + m.CollectRepeatFailures(now.Add(2 * time.Second)) + rfCopilotWriteSession(t, home, "on", log.String()) + m.CollectRepeatFailures(now.Add(3 * time.Second)) + st := rfStatus(m) + if st.Check != RepeatFailureCheckOn { + t.Fatalf("copilot check = %q, want %q (covered)", st.Check, RepeatFailureCheckOn) + } + if want := "Repeating failure: `ls /nonexistent` failed 5 times in 10 min (exit 2)"; st.Note != want { + t.Fatalf("note = %q, want %q", st.Note, want) + } +} diff --git a/src/crux/agent/repeat_failure_coverage.go b/src/crux/agent/repeat_failure_coverage.go index 64ad571954..304959f6f6 100644 --- a/src/crux/agent/repeat_failure_coverage.go +++ b/src/crux/agent/repeat_failure_coverage.go @@ -60,18 +60,18 @@ const ( // then config.InferenceBackends. var repeatFailureCoverageTable = []RepeatFailureCoverage{ {Backend: "claude", Route: "BASH_ENV (neutral), unwraps `eval '…'`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, - {Backend: "copilot", Route: "none yet; planned: `events.jsonl` `tool.execution_complete`", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one long-lived Bash session per shell id; " + rfOwnRouteNotBuilt}, + {Backend: "copilot", Covered: true, Route: "Hive reads `events.jsonl`: shell `tool.execution_complete` (`shellExecution.exitCode`)", Evidence: "real-run `events.jsonl` (`record.sh`), `copilot --version`: GitHub Copilot CLI 1.0.91", EvidencePath: "crux/agent/testdata/repeat_failure/copilot/events.jsonl", ShellActivity: "shell `tool.execution_start` / `tool.execution_complete`"}, {Backend: "goose", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "codex", Route: "BASH_ENV (neutral), one record across `exec /bin/bash -c`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "pi", Route: "BASH_ENV (neutral)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "bob", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash` (documented shell change: dash → bash)", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "aider", Route: "guarded `~/.bashrc` line (`aider_bashrc`) sourcing the BASH_ENV script, Hive-managed homes only, with `SHELL=/bin/bash`; the no-tty `/bin/sh -c` path reads no rc file and is not covered", FollowUp: repeatFailureBackendRoutesIssue, Reason: "interactive `$SHELL -i -c` ignores BASH_ENV; `aider_bashrc` route installed while the check is on, but no committed real-run harness evidence yet (C4)"}, {Backend: "gemini", Route: "BASH_ENV (neutral), unwraps the `{ …; }; __code=$?` wrapper", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, - {Backend: "agy", Route: "none yet; planned: `agy-turn` `run_command` steps", FollowUp: repeatFailureBackendRoutesIssue, Reason: "closed binary, likely a long-lived terminal; " + rfOwnRouteNotBuilt}, + {Backend: "agy", Route: "`agy-turn` `run_command` steps (headless launch only; the TUI launch has none)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "route built, but no committed real-run harness evidence yet (C4): agy 1.4.0 needs a Google login no harness can provide (tried 2026-10-27: `authentication required`) and takes no stub model provider"}, {Backend: "opencode", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "kilo", Route: "BASH_ENV (neutral) with `SHELL=/bin/bash`", FollowUp: repeatFailureBackendRoutesIssue, Reason: rfNeutralRouteNoEvidence}, {Backend: "muse", Route: "none yet; planned: `PostToolUse` / `PostToolUseFailure` hooks once Hive launches muse with a hook file or flag it owns", FollowUp: repeatFailureBackendRoutesIssue, Reason: "sandboxed shell tools in a closed binary, so commands never reach the BASH_ENV route; Hive has no muse launch or hook wiring and writes no muse settings file, so the hook route is not built"}, - {Backend: "omp", Route: "none yet; planned: `tool_result` extension event", FollowUp: repeatFailureBackendRoutesIssue, Reason: "embedded shell that strips BASH_ENV; " + rfOwnRouteNotBuilt}, + {Backend: "omp", Covered: true, Route: "Hive-owned `tool_result` extension (`--extension`), `details.exitCode`", Evidence: "stub-provider harness `run.sh`, `omp --version`: omp/18.13.1", EvidencePath: "crux/agent/testdata/repeat_failure/omp/evidence.txt", ShellActivity: repeatFailureSafeguardUnavailable}, {Backend: "openhands", Route: "none yet; planned: reader of the saved conversation events (`exit_code`)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "one interactive bash kept open in tmux, so commands never reach the BASH_ENV route; Hive has no openhands launch or hook wiring and does not write `~/.openhands/hooks.json` (so no `PostToolUse` hook), and the saved-events reader is not built"}, {Backend: "omnigent", Route: "none yet; planned: tool-call policy checkpoint (`ALLOW`/`DENY` verdicts) or saved session events", FollowUp: repeatFailureBackendRoutesIssue, Reason: "meta-harness driving Claude Code, Codex or Pi through its own policy engine; " + rfOwnRouteNotBuilt}, {Backend: "vllm", Route: "follows `claude` (drives Claude Code)", FollowUp: repeatFailureBackendRoutesIssue, Reason: "follows `claude`, which is not covered"}, diff --git a/src/crux/agent/repeat_failure_coverage_test.go b/src/crux/agent/repeat_failure_coverage_test.go index 87357f74df..54e55bffdc 100644 --- a/src/crux/agent/repeat_failure_coverage_test.go +++ b/src/crux/agent/repeat_failure_coverage_test.go @@ -10,6 +10,18 @@ import ( "github.com/hivecommons/hive/crux/config" ) +// rfCoverageRow returns backend's row of the coverage table. +func rfCoverageRow(t *testing.T, backend string) RepeatFailureCoverage { + t.Helper() + for _, row := range RepeatFailureCoverageTable() { + if row.Backend == backend { + return row + } + } + t.Fatalf("no coverage row for %q", backend) + return RepeatFailureCoverage{} +} + // TestRepeatFailureCoverageListsEveryBackend is C13/C-AC11: the code table has // exactly one row for every config.CLIBackends and config.InferenceBackends // entry, so removing a backend from it fails here. @@ -110,7 +122,7 @@ func TestRepeatFailureCoverageDocsMatchCode(t *testing.T) { } want := RepeatFailureCoverageMarkdownRows() if !slices.Equal(got, want) { - t.Errorf("docs repeated-failure table differs from the code table (pkg/agent/repeat_failure_coverage.go).\nwant rows:\n%s\n\ngot rows:\n%s", + t.Errorf("docs repeated-failure table differs from the code table (crux/agent/repeat_failure_coverage.go).\nwant rows:\n%s\n\ngot rows:\n%s", strings.Join(want, "\n"), strings.Join(got, "\n")) } } diff --git a/src/crux/agent/repeat_failure_omp.go b/src/crux/agent/repeat_failure_omp.go new file mode 100644 index 0000000000..8ba145210f --- /dev/null +++ b/src/crux/agent/repeat_failure_omp.go @@ -0,0 +1,149 @@ +package agent + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" +) + +// omp route for the repeat-failure check (hivecommons/hive#14550, C1, C3, +// C14; issue #14747). +// +// omp runs bash commands in a shell embedded in its own process and strips +// BASH_ENV, so the neutral route never sees them. While the check is on, Hive +// writes its own omp extension into the agent's 0700 record directory and +// passes it with `--extension`, independently of the advisor's extension. +// The extension listens to omp's `tool_result` event for the bash tool, whose +// `details.exitCode` is the command's exit status (omp 18.13.1), and appends +// one record per command to the agent's raw record file, following the +// record-file protocol: one write on an O_APPEND|O_NOFOLLOW descriptor, a +// shared flock(2) taken through flock(1) when it is installed, and only into +// the file still at the record path. Every failure fails open: no record, the +// tool result is never changed or delayed by an error. omp's own config.yml +// is never touched; with the check off Hive passes no flag and removes the +// file it wrote at the next launch (C14). + +const ( + // RepeatFailureRouteOmpExtension names the omp extension in a record's + // Route. + RepeatFailureRouteOmpExtension = "omp_extension" + repeatFailureOmpExtensionName = "omp_extension.ts" + // repeatFailureOmpExtensionHeader marks the extension as Hive's, so the + // launch-time cleanup never removes a file Hive did not write. + repeatFailureOmpExtensionHeader = "// Hive repeat-failure omp route (hivecommons/hive#14550)." +) + +// repeatFailureOmpExtensionPath is the Hive-owned extension inside an agent's +// HOME, next to its raw record file. +func repeatFailureOmpExtensionPath(home string) string { + return filepath.Join(filepath.Dir(RepeatFailureRecordPath(home)), repeatFailureOmpExtensionName) +} + +// renderRepeatFailureOmpExtension renders the extension for one agent with +// the agent name and record file baked in as JSON literals. +func renderRepeatFailureOmpExtension(agentName, recordPath string) []byte { + agent, _ := json.Marshal(agentName) + record, _ := json.Marshal(recordPath) + route, _ := json.Marshal(RepeatFailureRouteOmpExtension) + return []byte(fmt.Sprintf(repeatFailureOmpExtensionTemplate, repeatFailureOmpExtensionHeader, agent, record, route, RepeatFailureRecordMaxLine)) +} + +const repeatFailureOmpExtensionTemplate = `%s +// Written by Hive at agent launch and removed when the check is off; do not edit. +import { spawnSync } from "node:child_process"; +import { closeSync, constants, fstatSync, lstatSync, openSync, writeSync } from "node:fs"; + +const AGENT = %s; +const RECORD_PATH = %s; +const ROUTE = %s; +const MAX_LINE = %d; + +function recordLine(command, exitCode, failed) { + let truncated = false; + for (;;) { + const rec = { agent: AGENT, backend: "omp", time: new Date().toISOString(), raw_command: command }; + if (exitCode !== undefined) rec.exit_code = exitCode; + rec.failed = failed; + rec.route = ROUTE; + if (truncated) rec.truncated = true; + const line = JSON.stringify(rec) + "\n"; + if (Buffer.byteLength(line) <= MAX_LINE || command.length <= 1) return line; + command = command.slice(0, Math.floor(command.length / 2)); + truncated = true; + } +} + +function appendRecord(line) { + for (let attempt = 0; attempt < 3; attempt++) { + let fd; + try { + fd = openSync(RECORD_PATH, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + } catch { + return; + } + try { + try { + spawnSync("flock", ["-s", "3"], { stdio: ["ignore", "ignore", "ignore", fd], timeout: 5000 }); + } catch {} + const held = fstatSync(fd); + let current; + try { + current = lstatSync(RECORD_PATH); + } catch { + continue; + } + if (held.ino === current.ino && held.dev === current.dev) { + writeSync(fd, line); + return; + } + } catch { + return; + } finally { + try { + closeSync(fd); + } catch {} + } + } +} + +export default function (pi) { + pi.on("tool_result", event => { + try { + if (!event || event.toolName !== "bash" || !event.input || typeof event.input.command !== "string") return undefined; + const command = event.input.command; + if (command.trim() === "") return undefined; + const code = event.details && Number.isInteger(event.details.exitCode) ? event.details.exitCode : undefined; + appendRecord(recordLine(command, code, code !== undefined ? code !== 0 : event.isError === true)); + } catch {} + return undefined; + }); +} +` + +// repeatFailureOmpLaunchFlag writes the record extension for an omp launch +// while the check is on and returns the flag that loads it, or removes a file +// Hive wrote earlier and returns "" when the check is off. Best-effort like +// the other launch provisioning: a failure is logged and the agent launches +// without the route. +func (m *Manager) repeatFailureOmpLaunchFlag(agent *AgentProcess, backend string) string { + if agent == nil || !strings.EqualFold(strings.TrimSpace(backend), "omp") { + return "" + } + home := AgentHome(agent.Name, agent.UID, backend) + if home == "" { + return "" + } + ext := repeatFailureOmpExtensionPath(home) + if !m.repeatFailureEnabled() { + m.removeRepeatFailureFile(agent, ext, repeatFailureOmpExtensionHeader) + return "" + } + recordPath := RepeatFailureRecordPath(home) + if err := writeRepeatFailureRouteFile(m.agentExecUserSpec(agent), recordPath, ext, renderRepeatFailureOmpExtension(agent.Name, recordPath)); err != nil { + m.logger.Warn("repeat-failure: writing the omp extension failed; agent launches without it", + "agent", agent.Name, "path", ext, "error", err) + return "" + } + return " --extension " + shellQuote(ext) +} diff --git a/src/crux/agent/repeat_failure_omp_test.go b/src/crux/agent/repeat_failure_omp_test.go new file mode 100644 index 0000000000..d2a6816fc8 --- /dev/null +++ b/src/crux/agent/repeat_failure_omp_test.go @@ -0,0 +1,114 @@ +package agent + +import ( + "bufio" + "errors" + "os" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +// TestRepeatFailureOmpExtensionGolden pins the extension the omp harness +// (testdata/repeat_failure/omp/run.sh) loads to the one Hive renders, so the +// harness evidence is evidence for this code. +func TestRepeatFailureOmpExtensionGolden(t *testing.T) { + want, err := os.ReadFile("testdata/repeat_failure/omp/omp_extension.ts") + if err != nil { + t.Fatal(err) + } + if got := renderRepeatFailureOmpExtension("rf-harness", "__RF_RECORD_PATH__"); string(got) != string(want) { + t.Fatalf("rendered omp extension differs from the harness copy; regenerate testdata/repeat_failure/omp/omp_extension.ts and rerun run.sh.\ngot:\n%s", got) + } +} + +// TestRepeatFailureOmpHarnessEvidence is C-AC1 for omp: the committed harness +// run recorded `omp --version` and exactly one record, written by the real omp +// through Hive's extension, that the collector parses as `ls /nonexistent` +// failing with exit 2. +func TestRepeatFailureOmpHarnessEvidence(t *testing.T) { + f, err := os.Open("testdata/repeat_failure/omp/evidence.txt") + if err != nil { + t.Fatal(err) + } + defer f.Close() + var version string + var recs []RepeatFailureRecord + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + if strings.HasPrefix(line, "omp/") { + version = strings.TrimSpace(line) + } + if strings.HasPrefix(line, "{") { + rec, ok := parseRepeatFailureRecord([]byte(line + "\n")) + if !ok { + t.Fatalf("collector rejects the extension's record line: %s", line) + } + recs = append(recs, rec) + } + } + if version == "" { + t.Fatal("evidence lacks the `omp --version` output") + } + if row := rfCoverageRow(t, "omp"); !strings.Contains(row.Evidence, version) { + t.Fatalf("coverage evidence %q does not carry the harness version %q; update repeat_failure_coverage.go and the docs table", row.Evidence, version) + } + if len(recs) != 1 { + t.Fatalf("evidence records = %+v, want exactly one", recs) + } + rec := recs[0] + if rec.RawCommand != "ls /nonexistent" || rec.ExitCode == nil || *rec.ExitCode != 2 || !rec.Failed || + rec.Route != RepeatFailureRouteOmpExtension || rec.Backend != "omp" || rec.Time.IsZero() { + t.Fatalf("evidence record = %+v", rec) + } +} + +// TestRepeatFailureOmpLaunchFlag covers C14/C-AC12 for omp: with the check on +// an omp launch writes Hive's extension into the record dir and loads it with +// its own --extension even with the advisor off; other backends get nothing; +// with the check off the flag is gone and the file Hive wrote is removed at +// the next launch, while a file Hive did not write is left alone. +func TestRepeatFailureOmpLaunchFlag(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + m := NewManager(map[string]config.AgentConfig{"scanner": {Backend: "omp", Model: "m"}}, discardLogger(), ProjectContext{}) + ap := &AgentProcess{Name: "scanner", Config: config.AgentConfig{Backend: "omp", Model: "m"}} + on := true + m.SetRepeatFailureEnabledResolver(func() bool { return on }) + ext := repeatFailureOmpExtensionPath(home) + + if m.advisorEnabledFor("scanner") { + t.Fatal("test expects the advisor off") + } + flag := m.repeatFailureOmpLaunchFlag(ap, "omp") + if flag != " --extension "+shellQuote(ext) { + t.Fatalf("flag = %q, want the extension at %s", flag, ext) + } + data, err := os.ReadFile(ext) + if err != nil { + t.Fatalf("extension not written: %v", err) + } + if string(data) != string(renderRepeatFailureOmpExtension("scanner", RepeatFailureRecordPath(home))) { + t.Fatalf("extension content:\n%s", data) + } + if got := m.repeatFailureOmpLaunchFlag(ap, "claude"); got != "" { + t.Fatalf("claude launch got %q", got) + } + + on = false + if got := m.repeatFailureOmpLaunchFlag(ap, "omp"); got != "" { + t.Fatalf("check off: flag = %q, want none", got) + } + if _, err := os.Stat(ext); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("check off must remove the extension at the next launch, stat err = %v", err) + } + if err := os.WriteFile(ext, []byte("// someone else's file\n"), 0o600); err != nil { + t.Fatal(err) + } + m.repeatFailureOmpLaunchFlag(ap, "omp") + if _, err := os.Stat(ext); err != nil { + t.Fatalf("a file Hive did not write must be left alone: %v", err) + } +} diff --git a/src/crux/agent/repeat_failure_status.go b/src/crux/agent/repeat_failure_status.go index 0069576f4d..a59b7056ec 100644 --- a/src/crux/agent/repeat_failure_status.go +++ b/src/crux/agent/repeat_failure_status.go @@ -107,6 +107,9 @@ type repeatFailureMonitor struct { detectorKey string window time.Duration agents map[string]*repeatFailureAgentState + // copilot holds each Copilot agent's session-log reader + // (repeat_failure_copilot.go). + copilot map[string]*copilotRecordReader } // SetRepeatFailureConfigResolver injects the live repeat_failure block so the @@ -165,7 +168,7 @@ func (m *Manager) CollectRepeatFailures(now time.Time) { mon := &m.repeatFailure if !m.repeatFailureEnabled() { mon.mu.Lock() - mon.detector, mon.detectorKey, mon.agents = nil, "", nil + mon.detector, mon.detectorKey, mon.agents, mon.copilot = nil, "", nil, nil mon.mu.Unlock() m.mu.Lock() for _, a := range m.agents { @@ -176,6 +179,7 @@ func (m *Manager) CollectRepeatFailures(now time.Time) { } cfg := m.repeatFailureConfig() mon.configure(cfg) + m.readCopilotRepeatFailureRecords() collector := NewRepeatFailureCollector(cfg, m.repeatFailureRecordPaths, func(rec RepeatFailureRecord) { m.recordRepeatFailure(rec, now) }, m.logger) diff --git a/src/crux/agent/testdata/repeat_failure/copilot/events.jsonl b/src/crux/agent/testdata/repeat_failure/copilot/events.jsonl new file mode 100644 index 0000000000..42d341b519 --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/copilot/events.jsonl @@ -0,0 +1,2 @@ +{"type":"tool.execution_start","data":{"toolCallId":"toolu_01YSgXpoPU6hNfMhwFjPkHPy","toolName":"bash","arguments":{"command":"ls /nonexistent","description":"List contents of /nonexistent directory"},"turnId":"0","model":"claude-haiku-4.5","toolTitle":"Running command","shellToolInfo":{"possiblePaths":[],"hasWriteFileRedirection":false}},"id":"f952821e-1b7b-4a97-a1e4-db8c5cbe81aa","timestamp":"2026-10-28T01:24:32.472Z","parentId":"9c26740d-ce72-4054-bd1f-50c2cee99a7c"} +{"type":"tool.execution_complete","data":{"toolCallId":"toolu_01YSgXpoPU6hNfMhwFjPkHPy","model":"claude-haiku-4.5","interactionId":"64a17b39-81ac-4d1f-aabb-65aa605f1970","turnId":"0","rte":true,"shellExecution":{"exitCode":2},"success":true,"result":{"content":"ls: cannot access '/nonexistent': No such file or directory\n"},"toolTelemetry":{"properties":{"customTimeout":"false","executionMode":"sync","detached":"false","sandboxApplied":"false","sandboxOptOutRequested":"false","shell_error_category":"command_nonzero_exit"},"metrics":{"commandTimeout":30000.0,"exit_code":2.0}}},"id":"dec42fa2-983b-48a6-99d6-6c67dccb0d6f","timestamp":"2026-10-28T01:24:32.557Z","parentId":"f952821e-1b7b-4a97-a1e4-db8c5cbe81aa"} diff --git a/src/crux/agent/testdata/repeat_failure/copilot/record.sh b/src/crux/agent/testdata/repeat_failure/copilot/record.sh new file mode 100755 index 0000000000..aaa46907ab --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/copilot/record.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Repeat-failure harness for copilot (hivecommons/hive#14746). +# +# Makes the real Copilot CLI run one failing shell command and records the two +# session-log events Hive's copilot route reads (tool.execution_start and +# tool.execution_complete of that call) into events.jsonl, plus the CLI's own +# `copilot --version` into version.txt. TestRepeatFailureCopilotHarness replays +# events.jsonl through the route and expects exactly one failing record. +# +# Needs a logged-in copilot (uses ${COPILOT_HOME:-~/.copilot}) and jq. +# RF_COPILOT_MODEL picks the model (default claude-haiku-4.5). +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +command='ls /nonexistent' + +state="${COPILOT_HOME:-$HOME/.copilot}/session-state" +before=$(ls "$state" 2>/dev/null || true) +copilot --version | sed -n 1p > "$here/version.txt.new" +copilot --no-auto-update --allow-all-tools --model "${RF_COPILOT_MODEL:-claude-haiku-4.5}" \ + -p "Run exactly one shell command with your bash tool, then stop: $command" >/dev/null 2>&1 || true + +# The session this run created (not a session another copilot is writing). +session=$(ls "$state" | grep -vxF -e "$before" -e '' | sed -n 1p || true) +log="$state/$session/events.jsonl" +id=$(jq -r --arg c "$command" 'select(.type == "tool.execution_start" and .data.toolName == "bash" and .data.arguments.command == $c) | .data.toolCallId' "$log" | tail -n1) +if [ -z "$id" ]; then + echo "record.sh: copilot did not run '$command' (session: ${session:-none})" >&2 + rm -f "$here/version.txt.new" + exit 1 +fi +jq -c --arg id "$id" 'select((.type == "tool.execution_start" or .type == "tool.execution_complete") and .data.toolCallId == $id)' "$log" > "$here/events.jsonl" +mv "$here/version.txt.new" "$here/version.txt" +echo "recorded $(cat "$here/version.txt"): $(wc -l < "$here/events.jsonl") events for toolCallId $id" diff --git a/src/crux/agent/testdata/repeat_failure/copilot/version.txt b/src/crux/agent/testdata/repeat_failure/copilot/version.txt new file mode 100644 index 0000000000..d3a75b9831 --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/copilot/version.txt @@ -0,0 +1 @@ +GitHub Copilot CLI 1.0.91. diff --git a/src/crux/agent/testdata/repeat_failure/omp/evidence.txt b/src/crux/agent/testdata/repeat_failure/omp/evidence.txt new file mode 100644 index 0000000000..1a74363aa2 --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/omp/evidence.txt @@ -0,0 +1,5 @@ +# omp --version +omp/18.13.1 +# stub provider: one bash call, ls /nonexistent +# records.jsonl written by Hive's omp extension +{"agent":"rf-harness","backend":"omp","time":"2026-10-28T01:28:42.345Z","raw_command":"ls /nonexistent","exit_code":2,"failed":true,"route":"omp_extension"} diff --git a/src/crux/agent/testdata/repeat_failure/omp/omp_extension.ts b/src/crux/agent/testdata/repeat_failure/omp/omp_extension.ts new file mode 100644 index 0000000000..1c52f98e6d --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/omp/omp_extension.ts @@ -0,0 +1,70 @@ +// Hive repeat-failure omp route (hivecommons/hive#14550). +// Written by Hive at agent launch and removed when the check is off; do not edit. +import { spawnSync } from "node:child_process"; +import { closeSync, constants, fstatSync, lstatSync, openSync, writeSync } from "node:fs"; + +const AGENT = "rf-harness"; +const RECORD_PATH = "__RF_RECORD_PATH__"; +const ROUTE = "omp_extension"; +const MAX_LINE = 4096; + +function recordLine(command, exitCode, failed) { + let truncated = false; + for (;;) { + const rec = { agent: AGENT, backend: "omp", time: new Date().toISOString(), raw_command: command }; + if (exitCode !== undefined) rec.exit_code = exitCode; + rec.failed = failed; + rec.route = ROUTE; + if (truncated) rec.truncated = true; + const line = JSON.stringify(rec) + "\n"; + if (Buffer.byteLength(line) <= MAX_LINE || command.length <= 1) return line; + command = command.slice(0, Math.floor(command.length / 2)); + truncated = true; + } +} + +function appendRecord(line) { + for (let attempt = 0; attempt < 3; attempt++) { + let fd; + try { + fd = openSync(RECORD_PATH, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + } catch { + return; + } + try { + try { + spawnSync("flock", ["-s", "3"], { stdio: ["ignore", "ignore", "ignore", fd], timeout: 5000 }); + } catch {} + const held = fstatSync(fd); + let current; + try { + current = lstatSync(RECORD_PATH); + } catch { + continue; + } + if (held.ino === current.ino && held.dev === current.dev) { + writeSync(fd, line); + return; + } + } catch { + return; + } finally { + try { + closeSync(fd); + } catch {} + } + } +} + +export default function (pi) { + pi.on("tool_result", event => { + try { + if (!event || event.toolName !== "bash" || !event.input || typeof event.input.command !== "string") return undefined; + const command = event.input.command; + if (command.trim() === "") return undefined; + const code = event.details && Number.isInteger(event.details.exitCode) ? event.details.exitCode : undefined; + appendRecord(recordLine(command, code, code !== undefined ? code !== 0 : event.isError === true)); + } catch {} + return undefined; + }); +} diff --git a/src/crux/agent/testdata/repeat_failure/omp/run.sh b/src/crux/agent/testdata/repeat_failure/omp/run.sh new file mode 100755 index 0000000000..86e03777f4 --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/omp/run.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Repeat-failure harness for omp (hivecommons/hive#14747). +# +# Runs the real omp CLI against a stub model provider (../stub_provider.py) +# that asks for one bash call, `ls /nonexistent`, with Hive's record extension +# loaded the way Hive loads it (`--extension`). omp_extension.ts is the +# extension exactly as Hive renders it (TestRepeatFailureOmpExtensionGolden); +# this script only fills in the record path. Writes `omp --version` and the +# record file the extension produced to evidence.txt, which +# TestRepeatFailureOmpHarnessEvidence checks. +# +# Needs omp and python3. Runs in a throwaway HOME; touches nothing else. +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +work="$here/.run" +rm -rf "$work" +mkdir -p "$work/home/.omp/agent" "$work/cwd" "$work/records" +timeout 180 python3 "$here/../stub_provider.py" "$work/port" & +stub=$! +trap 'kill "$stub" 2>/dev/null; rm -rf "$work"' EXIT +for _ in $(seq 50); do [ -s "$work/port" ] && break; sleep 0.1; done +port=$(cat "$work/port") +cat > "$work/home/.omp/agent/models.yml" < "$work/omp_extension.ts" + +version=$(HOME="$work/home" omp --version) +(cd "$work/cwd" && HOME="$work/home" OMP_SKIP_SETUP=1 timeout 120 \ + omp -p --model rfstub/rf-stub --approval-mode yolo --no-session \ + --extension "$work/omp_extension.ts" "run the command" >/dev/null) + +{ + echo "# omp --version" + echo "$version" + echo "# stub provider: one bash call, ls /nonexistent" + echo "# records.jsonl written by Hive's omp extension" + cat "$record" +} > "$here/evidence.txt" +cat "$here/evidence.txt" diff --git a/src/crux/agent/testdata/repeat_failure/stub_provider.py b/src/crux/agent/testdata/repeat_failure/stub_provider.py new file mode 100644 index 0000000000..88b80660b6 --- /dev/null +++ b/src/crux/agent/testdata/repeat_failure/stub_provider.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Stub OpenAI-compatible chat-completions provider for the repeat-failure +harnesses. The first request in a conversation answers with one shell tool +call running RF_COMMAND (default `ls /nonexistent`); any request that already +carries a tool result answers with plain text, so the CLI ends its turn. + +Usage: stub_provider.py (binds 127.0.0.1 on a free port and +writes the port to ). Tool name: RF_TOOL (default `bash`). +""" +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +COMMAND = os.environ.get("RF_COMMAND", "ls /nonexistent") +TOOL = os.environ.get("RF_TOOL", "bash") +MODEL = "rf-stub" + + +def chunk(delta, finish=None): + return { + "id": "rf-stub", + "object": "chat.completion.chunk", + "created": 0, + "model": MODEL, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + body = json.dumps({"object": "list", "data": [{"id": MODEL, "object": "model"}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + req = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))) or b"{}") + answered = any(m.get("role") == "tool" for m in req.get("messages", [])) + if answered: + events = [chunk({"role": "assistant", "content": "DONE"}), chunk({}, "stop")] + else: + args = json.dumps({"command": COMMAND}) + call = {"index": 0, "id": "call_rf_1", "type": "function", "function": {"name": TOOL, "arguments": args}} + events = [chunk({"role": "assistant", "tool_calls": [call]}), chunk({}, "tool_calls")] + if not req.get("stream"): + msg = {"role": "assistant", "content": "DONE"} if answered else { + "role": "assistant", "content": None, "tool_calls": [dict(call, index=None)]} + body = json.dumps({"id": "rf-stub", "object": "chat.completion", "model": MODEL, + "choices": [{"index": 0, "message": msg, + "finish_reason": "stop" if answered else "tool_calls"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for ev in events: + self.wfile.write(b"data: " + json.dumps(ev).encode() + b"\n\n") + usage = chunk({}) + usage["choices"] = [] + usage["usage"] = {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + self.wfile.write(b"data: " + json.dumps(usage).encode() + b"\n\n") + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + +def main(): + server = HTTPServer(("127.0.0.1", 0), Handler) + with open(sys.argv[1], "w") as f: + f.write(str(server.server_address[1])) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 9b8c7c5d84..76674ad1e5 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -588,6 +588,14 @@ When it is on, backends that run every shell command in a fresh, non-interactive Turning the check off removes the script at each agent's next launch and drops a Hive-set `BASH_ENV` from the agent's tmux session; a pane shell that still holds the old variable points at a missing file, which bash skips silently. +### Backend-specific routes + +Backends whose shell commands never reach a fresh `bash -c` get their own route instead of the script above; all of them write the same record format into `~/.hive/repeat-failure/records.jsonl`, and all are inactive while the check is off. The per-backend status is the [repeated-failure coverage table](backend-support-tiers.md#repeated-failure-check). + +- **copilot** keeps one long-lived shell per session, so Hive reads Copilot's own session log (`~/.copilot/session-state//events.jsonl`) on each collection and records every completed shell tool call with its exit code. Nothing is written into Copilot's files. A log that already exists when Hive starts, or when the check is turned on, is read from its end: calls logged before that point are not replayed. +- **omp** strips `BASH_ENV`, so at each launch Hive writes its own extension to `~/.hive/repeat-failure/omp_extension.ts` and loads it with `--extension`; the extension records each bash tool result. omp's `config.yml` is never touched. With the check off, the next launch passes no flag and removes the file Hive wrote. +- **agy** (headless launch only) records each `run_command` step from the `agy-turn` output; the TUI launch has no route. + ## Methods: subscription CLIs vs self-hosted inference `backend:` picks one of the CLI or inference families below. Hosted execution diff --git a/src/docs/backend-support-tiers.md b/src/docs/backend-support-tiers.md index 0efd91a5f8..9fec85176c 100644 --- a/src/docs/backend-support-tiers.md +++ b/src/docs/backend-support-tiers.md @@ -565,25 +565,25 @@ signal is what the "not receiving data" safeguard watches for a covered backend, or "safeguard unavailable" where the backend has none. This table is generated from the code table in -`src/pkg/agent/repeat_failure_coverage.go`; +`src/crux/agent/repeat_failure_coverage.go`; `TestRepeatFailureCoverageDocsMatchCode` fails when a backend is missing or a row here differs from the code. | Backend | Route | Status | Evidence | Shell-activity signal | Follow-up issue | | --- | --- | --- | --- | --- | --- | | `claude` | BASH_ENV (neutral), unwraps `eval '…'` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | -| `copilot` | none yet; planned: `events.jsonl` `tool.execution_complete` | Not covered: one long-lived Bash session per shell id; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `copilot` | Hive reads `events.jsonl`: shell `tool.execution_complete` (`shellExecution.exitCode`) | Covered | real-run `events.jsonl` (`record.sh`), `copilot --version`: GitHub Copilot CLI 1.0.91 | shell `tool.execution_start` / `tool.execution_complete` | — | | `goose` | BASH_ENV (neutral) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `codex` | BASH_ENV (neutral), one record across `exec /bin/bash -c` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `pi` | BASH_ENV (neutral) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `bob` | BASH_ENV (neutral) with `SHELL=/bin/bash` (documented shell change: dash → bash) | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `aider` | guarded `~/.bashrc` line (`aider_bashrc`) sourcing the BASH_ENV script, Hive-managed homes only, with `SHELL=/bin/bash`; the no-tty `/bin/sh -c` path reads no rc file and is not covered | Not covered: interactive `$SHELL -i -c` ignores BASH_ENV; `aider_bashrc` route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `gemini` | BASH_ENV (neutral), unwraps the `{ …; }; __code=$?` wrapper | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | -| `agy` | none yet; planned: `agy-turn` `run_command` steps | Not covered: closed binary, likely a long-lived terminal; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `agy` | `agy-turn` `run_command` steps (headless launch only; the TUI launch has none) | Not covered: route built, but no committed real-run harness evidence yet (C4): agy 1.4.0 needs a Google login no harness can provide (tried 2026-10-27: `authentication required`) and takes no stub model provider | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `opencode` | BASH_ENV (neutral) with `SHELL=/bin/bash` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `kilo` | BASH_ENV (neutral) with `SHELL=/bin/bash` | Not covered: BASH_ENV route installed while the check is on, but no committed real-run harness evidence yet (C4) | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `muse` | none yet; planned: `PostToolUse` / `PostToolUseFailure` hooks once Hive launches muse with a hook file or flag it owns | Not covered: sandboxed shell tools in a closed binary, so commands never reach the BASH_ENV route; Hive has no muse launch or hook wiring and writes no muse settings file, so the hook route is not built | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | -| `omp` | none yet; planned: `tool_result` extension event | Not covered: embedded shell that strips BASH_ENV; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | +| `omp` | Hive-owned `tool_result` extension (`--extension`), `details.exitCode` | Covered | stub-provider harness `run.sh`, `omp --version`: omp/18.13.1 | safeguard unavailable | — | | `openhands` | none yet; planned: reader of the saved conversation events (`exit_code`) | Not covered: one interactive bash kept open in tmux, so commands never reach the BASH_ENV route; Hive has no openhands launch or hook wiring and does not write `~/.openhands/hooks.json` (so no `PostToolUse` hook), and the saved-events reader is not built | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `omnigent` | none yet; planned: tool-call policy checkpoint (`ALLOW`/`DENY` verdicts) or saved session events | Not covered: meta-harness driving Claude Code, Codex or Pi through its own policy engine; commands never reach the BASH_ENV route and the backend-specific route is not built yet | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) | | `vllm` | follows `claude` (drives Claude Code) | Not covered: follows `claude`, which is not covered | None | n/a | [#14559](https://github.com/hivecommons/hive/issues/14559) |