diff --git a/AGENTS.md b/AGENTS.md index 39c1fba0..f8e4c63e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,6 +227,15 @@ activeCount atomic.Int64 | `sync.RWMutex` | Structured data, snapshots, iteration | `RuntimeState` | | `atomic` | Simple flags, counters | `paused`, `closed` | +## State Machine Patterns + +| Complexity | Pattern | Example | +|------------|---------|---------| +| ≤5 events | Explicit methods (`UserPause()`, `AgentResume()`) | `pkg/mergequeue/pause_state.go` | +| >5 events | Event-driven (`Transition(ctx, event)`) | `pkg/lifecycle/state.go` | + +**Both patterns require:** mutex-protected state, query methods (`State()`, `IsTerminal()`), `String()` for logging. + ## API Conventions (Go ↔ TypeScript) **JSON field names use snake_case** throughout the codebase. diff --git a/cmd/canopy/ps.go b/cmd/canopy/ps.go index 44965599..00235acd 100644 --- a/cmd/canopy/ps.go +++ b/cmd/canopy/ps.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/jzila/canopy/pkg/daemon" + "github.com/jzila/canopy/pkg/repository" "github.com/jzila/canopy/pkg/runtime" ) @@ -19,6 +20,7 @@ var ( psJSON bool psDaemonOnly bool psWorkersOnly bool + psShowHistory bool ) var psCmd = &cobra.Command{ @@ -41,7 +43,10 @@ EXAMPLES canopy ps --workers-only # Output as JSON for scripting - canopy ps --json`, + canopy ps --json + + # Show state transition history for debugging + canopy ps --show-history`, RunE: runPs, } @@ -49,27 +54,31 @@ func init() { psCmd.Flags().BoolVar(&psJSON, "json", false, "Output as JSON") psCmd.Flags().BoolVar(&psDaemonOnly, "daemon-only", false, "Show only daemon status") psCmd.Flags().BoolVar(&psWorkersOnly, "workers-only", false, "Show only worker agents") + psCmd.Flags().BoolVar(&psShowHistory, "show-history", false, "Display state transition history for debugging") rootCmd.AddCommand(psCmd) } // ProcessInfo contains information about canopy processes type ProcessInfo struct { - Daemon *DaemonInfo `json:"daemon,omitempty"` - Workers []WorkerInfo `json:"workers,omitempty"` + Daemon *DaemonInfo `json:"daemon,omitempty"` + Workers []WorkerInfo `json:"workers,omitempty"` + MergeQueue *MergeQueueInfo `json:"merge_queue,omitempty"` } // DaemonInfo contains information about the daemon process type DaemonInfo struct { - Running bool `json:"running"` - PID int `json:"pid,omitempty"` - Port int `json:"port"` - SocketPath string `json:"socketPath"` - Uptime string `json:"uptime,omitempty"` - StartTime time.Time `json:"startTime,omitempty"` - IsPaused bool `json:"isPaused"` - ActiveRepo string `json:"activeRepo,omitempty"` - Stats *StatsInfo `json:"stats,omitempty"` + Running bool `json:"running"` + PID int `json:"pid,omitempty"` + Port int `json:"port"` + SocketPath string `json:"socketPath"` + Uptime string `json:"uptime,omitempty"` + StartTime time.Time `json:"startTime,omitempty"` + IsPaused bool `json:"isPaused"` + ActiveRepo string `json:"activeRepo,omitempty"` + Stats *StatsInfo `json:"stats,omitempty"` + OrchestratorState string `json:"orchestratorState,omitempty"` + ActiveAgentCount int `json:"activeAgentCount,omitempty"` } // StatsInfo contains aggregate statistics from the daemon @@ -83,13 +92,43 @@ type StatsInfo struct { // WorkerInfo contains information about a worker agent type WorkerInfo struct { - AgentID string `json:"agentId"` - TaskID string `json:"taskId"` - TaskTitle string `json:"taskTitle,omitempty"` - Status string `json:"status"` - MergeStatus string `json:"mergeStatus,omitempty"` - Duration string `json:"duration"` - StartTime string `json:"startTime,omitempty"` + AgentID string `json:"agentId"` + TaskID string `json:"taskId"` + TaskTitle string `json:"taskTitle,omitempty"` + Status string `json:"status"` + MergeStatus string `json:"mergeStatus,omitempty"` + ValidationStatus string `json:"validationStatus,omitempty"` + RepairAttempts int `json:"repairAttempts,omitempty"` + LifecycleState string `json:"lifecycleState,omitempty"` + LifecycleHistory []LifecycleHistoryEntry `json:"lifecycleHistory,omitempty"` + MergeQueuePos int `json:"mergeQueuePos,omitempty"` + Duration string `json:"duration"` + StartTime string `json:"startTime,omitempty"` +} + +// LifecycleHistoryEntry represents a single state transition +type LifecycleHistoryEntry struct { + From string `json:"from"` + To string `json:"to"` + Event string `json:"event"` + Timestamp string `json:"timestamp"` +} + +// MergeQueueInfo contains the merge queue state +type MergeQueueInfo struct { + Entries []MergeQueueEntry `json:"entries"` + QueueLength int `json:"queue_length"` + IsPaused bool `json:"is_paused"` + PauseState string `json:"pause_state"` +} + +// MergeQueueEntry represents an item in the merge queue +type MergeQueueEntry struct { + AgentID string `json:"agent_id"` + TaskID string `json:"task_id"` + TaskTitle string `json:"task_title,omitempty"` + Status string `json:"status"` // merging, waiting + Position int `json:"position"` } func runPs(cmd *cobra.Command, args []string) error { @@ -106,12 +145,13 @@ func runPs(cmd *cobra.Command, args []string) error { // Get worker info unless daemon-only is specified if !psDaemonOnly { - workers, err := getWorkerInfo() + workers, mergeQueue, err := getWorkerInfo() if err != nil && info.Daemon != nil && info.Daemon.Running { // Only report error if daemon is running but we failed to get workers return fmt.Errorf("failed to get worker info: %w", err) } info.Workers = workers + info.MergeQueue = mergeQueue } if psJSON { @@ -153,9 +193,18 @@ func getDaemonInfo() (*DaemonInfo, error) { info.Uptime = formatDuration(time.Since(stateResp.StartTime)) } - // Get active repo ID from response + // Get orchestrator state from response + info.OrchestratorState = stateResp.OrchestratorState + info.ActiveAgentCount = stateResp.ActiveAgentCount + + // Get active repo path from response (look up path from ID) if activeRepoID, ok := stateResp.Extra["active_repo_id"].(string); ok && activeRepoID != "" { - info.ActiveRepo = activeRepoID + if repo, err := repository.FromID(activeRepoID); err == nil && repo != nil { + info.ActiveRepo = repo.Path + } else { + // Fallback to ID if lookup fails + info.ActiveRepo = activeRepoID + } } // Extract stats @@ -172,24 +221,40 @@ func getDaemonInfo() (*DaemonInfo, error) { // StateResponse matches the daemon's state response structure type StateResponse struct { - Agents map[string]AgentInfo `json:"agents"` - Tasks map[string]TaskInfo `json:"tasks"` - Stats StatsResponse `json:"stats"` - IsPaused bool `json:"is_paused"` - StartTime time.Time `json:"start_time"` - Extra map[string]interface{} `json:"-"` // For capturing additional fields like active_repo_id + Agents map[string]AgentInfo `json:"agents"` + Tasks map[string]TaskInfo `json:"tasks"` + Stats StatsResponse `json:"stats"` + IsPaused bool `json:"is_paused"` + StartTime time.Time `json:"start_time"` + OrchestratorState string `json:"orchestrator_state,omitempty"` + ActiveAgentCount int `json:"active_agent_count,omitempty"` + MergeQueue *MergeQueueInfo `json:"merge_queue,omitempty"` + Extra map[string]interface{} `json:"-"` // For capturing additional fields like active_repo_id } // AgentInfo matches the daemon's agent state structure type AgentInfo struct { - ID string `json:"id"` - TaskID string `json:"task_id"` - TaskTitle string `json:"task_title"` - Status string `json:"status"` - MergeStatus string `json:"merge_status"` - StartTime time.Time `json:"start_time"` - Duration float64 `json:"duration"` - Archived bool `json:"archived"` + ID string `json:"id"` + TaskID string `json:"task_id"` + TaskTitle string `json:"task_title"` + Status string `json:"status"` + MergeStatus string `json:"merge_status"` + ValidationStatus string `json:"validation_status"` + RepairAttempts int `json:"repair_attempts"` + LifecycleState string `json:"lifecycle_state"` + LifecycleHistory []AgentLifecycleHistoryEntry `json:"lifecycle_history"` + MergeQueuePos int `json:"merge_queue_pos"` + StartTime time.Time `json:"start_time"` + Duration float64 `json:"duration"` + Archived bool `json:"archived"` +} + +// AgentLifecycleHistoryEntry matches the daemon's lifecycle history structure +type AgentLifecycleHistoryEntry struct { + From string `json:"from"` + To string `json:"to"` + Event string `json:"event"` + Timestamp string `json:"timestamp"` } // TaskInfo matches the daemon's task state structure @@ -246,17 +311,17 @@ func queryDaemonState(port int) (*StateResponse, error) { return &state, nil } -func getWorkerInfo() ([]WorkerInfo, error) { +func getWorkerInfo() ([]WorkerInfo, *MergeQueueInfo, error) { // Check if daemon is running running, _, err := daemon.IsRunning() if err != nil || !running { - return nil, nil + return nil, nil, nil } // Query daemon for agent info state, err := queryDaemonState(8080) if err != nil { - return nil, err + return nil, nil, err } var workers []WorkerInfo @@ -278,18 +343,29 @@ func getWorkerInfo() ([]WorkerInfo, error) { duration = formatDuration(time.Duration(agent.Duration * float64(time.Second))) } + // Convert lifecycle history entries + var history []LifecycleHistoryEntry + for _, h := range agent.LifecycleHistory { + history = append(history, LifecycleHistoryEntry(h)) + } + workers = append(workers, WorkerInfo{ - AgentID: agent.ID, - TaskID: agent.TaskID, - TaskTitle: agent.TaskTitle, - Status: agent.Status, - MergeStatus: agent.MergeStatus, - Duration: duration, - StartTime: agent.StartTime.Format("15:04:05"), + AgentID: agent.ID, + TaskID: agent.TaskID, + TaskTitle: agent.TaskTitle, + Status: agent.Status, + MergeStatus: agent.MergeStatus, + ValidationStatus: agent.ValidationStatus, + RepairAttempts: agent.RepairAttempts, + LifecycleState: agent.LifecycleState, + LifecycleHistory: history, + MergeQueuePos: agent.MergeQueuePos, + Duration: duration, + StartTime: agent.StartTime.Format("15:04:05"), }) } - return workers, nil + return workers, state.MergeQueue, nil } func outputPsJSON(info ProcessInfo) error { @@ -309,6 +385,15 @@ func outputPsTable(info ProcessInfo) error { if info.Daemon.Uptime != "" { fmt.Printf(" Uptime: %s\n", info.Daemon.Uptime) } + // Display orchestrator state + if info.Daemon.OrchestratorState != "" { + orchState := info.Daemon.OrchestratorState + if orchState == "active" && info.Daemon.ActiveAgentCount > 0 { + fmt.Printf(" Orchestrator: %s (%d agents)\n", orchState, info.Daemon.ActiveAgentCount) + } else { + fmt.Printf(" Orchestrator: %s\n", orchState) + } + } if info.Daemon.IsPaused { fmt.Printf(" Paused: yes\n") } @@ -336,7 +421,7 @@ func outputPsTable(info ProcessInfo) error { fmt.Println(" No active workers") } else { // Print header - use full agent IDs since they're needed for canopy kill - fmt.Printf(" %-30s %-16s %-10s %-10s %s\n", + fmt.Printf(" %-30s %-16s %-18s %-10s %s\n", "AGENT ID", "TASK ID", "STATUS", "DURATION", "TITLE") for _, w := range info.Workers { @@ -348,21 +433,95 @@ func outputPsTable(info ProcessInfo) error { title = title[:37] + "..." } - status := w.Status - if w.MergeStatus != "" && w.MergeStatus != "none" { - status = w.MergeStatus - } + // Use lifecycle state directly if available + status := formatLifecycleStatus(w) - fmt.Printf(" %-30s %-16s %-10s %-10s %s\n", + fmt.Printf(" %-30s %-16s %-18s %-10s %s\n", w.AgentID, taskID, status, w.Duration, title) + + // Show state transition history if requested + if psShowHistory && len(w.LifecycleHistory) > 0 { + fmt.Println(" State History:") + for _, h := range w.LifecycleHistory { + // Parse and format timestamp for display + ts := h.Timestamp + if t, err := time.Parse(time.RFC3339, h.Timestamp); err == nil { + ts = t.Format("15:04:05") + } + fmt.Printf(" %s %s -> %s (%s)\n", ts, h.From, h.To, h.Event) + } + } } } fmt.Println() } + // Show merge queue section unless daemon-only + if !psDaemonOnly { + outputMergeQueue(info.MergeQueue) + } + return nil } +// outputMergeQueue displays the merge queue state +func outputMergeQueue(mq *MergeQueueInfo) { + if mq == nil || len(mq.Entries) == 0 { + fmt.Println("Merge Queue: empty") + fmt.Println() + return + } + + fmt.Println("Merge Queue:") + for _, entry := range mq.Entries { + // Format: Position N: agent-id (task-id) - status + fmt.Printf(" Position %d: %s (%s) - %s\n", + entry.Position, + truncateID(entry.AgentID, 20), + truncateID(entry.TaskID, 16), + entry.Status, + ) + } + fmt.Println() +} + +// formatLifecycleStatus returns a display string for the agent's status. +// Prefers lifecycle_state when available, with additional context for specific states. +func formatLifecycleStatus(w WorkerInfo) string { + // Use lifecycle state if available + if w.LifecycleState != "" { + switch w.LifecycleState { + case "queued_for_merge": + if w.MergeQueuePos > 0 { + return fmt.Sprintf("queued (pos %d)", w.MergeQueuePos) + } + return "queued" + case "validating": + return "validating" + case "repairing": + if w.RepairAttempts > 0 { + return fmt.Sprintf("repairing (%d/3)", w.RepairAttempts) + } + return "repairing" + default: + return w.LifecycleState + } + } + + // Fallback to legacy status derivation + status := w.Status + if w.ValidationStatus == "repairing" { + if w.RepairAttempts > 0 { + status = fmt.Sprintf("repairing (%d)", w.RepairAttempts) + } else { + status = "repairing" + } + } else if w.MergeStatus != "" && w.MergeStatus != "none" { + status = w.MergeStatus + } + return status +} + // truncateID truncates an ID to the specified length, showing the first part func truncateID(id string, maxLen int) string { if len(id) <= maxLen { diff --git a/cmd/canopy/run.go b/cmd/canopy/run.go index 1ba9e9cc..78ba8604 100644 --- a/cmd/canopy/run.go +++ b/cmd/canopy/run.go @@ -9,18 +9,14 @@ import ( "os" "os/signal" "path/filepath" - "sync" "syscall" "time" - "github.com/google/uuid" "github.com/spf13/cobra" "github.com/jzila/canopy/pkg/agent" "github.com/jzila/canopy/pkg/beads" - "github.com/jzila/canopy/pkg/config" "github.com/jzila/canopy/pkg/ipc" - "github.com/jzila/canopy/pkg/orchestrator" "github.com/jzila/canopy/pkg/repository" "github.com/jzila/canopy/pkg/runtime" "github.com/jzila/canopy/pkg/sandbox" @@ -40,14 +36,17 @@ var ( filterLabels []string excludeLabels []string filterAssignee string - watchMode bool pollInterval time.Duration ) var runCmd = &cobra.Command{ Use: "run", Short: "Execute ready tasks from beads", - Long: `Executes all ready (unblocked) tasks from beads in parallel. + Long: `Executes ready (unblocked) tasks from beads in parallel, polling continuously. + +The orchestrator runs continuously until cancelled (Ctrl+C): +- When work is available: processes tasks (Active state) +- When no work is available: sleeps for poll interval (Idle state) Each task runs in an isolated sandbox with its own copy of the working directory. Changes are merged back after completion. @@ -130,10 +129,8 @@ Example: # Resume interrupted agents after daemon restart canopy run --resume # Resume agents that were interrupted - # Watch mode: keep running and poll for new tasks - canopy run --watch # Run indefinitely, polling for new tasks - canopy run --watch --poll-interval 10s # Poll every 10 seconds (default: 5s) - canopy run -w # Short form of --watch`, + # Set poll interval for idle state + canopy run --poll-interval 10s # Poll every 10 seconds (default: 5s)`, RunE: runOrchestrator, } @@ -154,9 +151,8 @@ func init() { runCmd.Flags().StringSliceVar(&excludeLabels, "exclude-label", nil, "Exclude tasks with these labels (comma-separated)") runCmd.Flags().StringVar(&filterAssignee, "assignee", "", "Filter by assignee (\"\" = unassigned only, \"*\" = any, name = exact match)") - // Watch mode flags - runCmd.Flags().BoolVarP(&watchMode, "watch", "w", false, "Watch mode: keep running and poll for new tasks instead of exiting when queue is empty") - runCmd.Flags().DurationVar(&pollInterval, "poll-interval", 5*time.Second, "Interval between polling for new tasks in watch mode") + // Poll interval for idle state + runCmd.Flags().DurationVar(&pollInterval, "poll-interval", 5*time.Second, "Interval between polling for new tasks when idle") rootCmd.AddCommand(runCmd) } @@ -182,39 +178,6 @@ func runOrchestrator(cmd *cobra.Command, args []string) error { } } - // Orchestrator will be set once created, for signal cleanup - var orch *orchestrator.Orchestrator - - // Handle interrupt signals - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigCh - fmt.Fprintln(os.Stderr, "\nInterrupted, cleaning up overlays...") - - // Cancel context to stop agents - cancel() - - // Synchronously cleanup all active overlays to prevent orphans - if orch != nil { - sched := orch.GetScheduler() - if sched != nil { - count, err := sched.CleanupAll(5 * time.Second) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: overlay cleanup encountered errors: %v\n", err) - } else if count > 0 { - fmt.Fprintf(os.Stderr, "Cleaned up %d active overlay(s)\n", count) - } - } - } - - // Exit after cleanup completes - // We must call os.Exit here because orch.Run returns immediately when - // context is cancelled, which would cause the process to exit before - // cleanup completes (cleanup goroutine races with process exit) - os.Exit(0) - }() - // Resolve working directory absWorkdir, err := filepath.Abs(workdir) if err != nil { @@ -226,7 +189,7 @@ func runOrchestrator(cmd *cobra.Command, args []string) error { outputDir = absWorkdir } - // Resolve resolver timeout with precedence: CLI flag > env var > config file > default + // Resolve resolver timeout with precedence: CLI flag > env var > default effectiveResolverTimeout := resolverTimeout if effectiveResolverTimeout == 0 { // Try environment variable @@ -238,18 +201,7 @@ func runOrchestrator(cmd *cobra.Command, args []string) error { effectiveResolverTimeout = parsed } } - if effectiveResolverTimeout == 0 { - // Try config file - cfg, err := config.LoadConfig(absWorkdir) - if err != nil { - if verbose { - fmt.Fprintf(os.Stderr, "warning: failed to load config: %v\n", err) - } - } else if cfg != nil { - effectiveResolverTimeout = cfg.GetResolverTimeout() - } - } - // Note: if still 0, the processor will use its default of 10 minutes + // Note: if still 0, the daemon will use its default of 10 minutes // Initialize repository for tracking repo, err := repository.GetOrCreate(absWorkdir) @@ -261,6 +213,12 @@ func runOrchestrator(cmd *cobra.Command, args []string) error { repo = nil } + // Get repo ID for daemon calls + repoID := "" + if repo != nil { + repoID = repo.ID + } + // Connect to daemon (required for canopy run) ipcClient, err := ipc.GetClient() if err != nil { @@ -288,137 +246,261 @@ func runOrchestrator(cmd *cobra.Command, args []string) error { } } - // Build CLI rules overrides (orchestrator loads config.toml and merges these) - rules, rulesOverrides := buildCLIRulesOverrides(cmd) - - // Create and run orchestrator - orch, err = orchestrator.New(&orchestrator.Config{ - WorkDir: absWorkdir, - OutputDir: outputDir, - Concurrency: concurrency, - Verbose: verbose, - DryRun: dryRun, - UseBwrap: useSandbox, - MaxRetries: maxRetries, - MaxPriority: maxPriority, - ResolverTimeout: effectiveResolverTimeout, - Rules: rules, - RulesOverrides: rulesOverrides, - Watch: watchMode, - PollInterval: pollInterval, + // Start the orchestrator run via daemon HTTP API + runID, err := startDaemonRun(ctx, startRunRequest{ + WorkDir: absWorkdir, + OutputDir: outputDir, + Concurrency: concurrency, + Verbose: verbose, + DryRun: dryRun, + UseBwrap: useSandbox, + MaxRetries: maxRetries, + MaxPriority: maxPriority, + ResolverTimeoutMS: effectiveResolverTimeout.Milliseconds(), + RepoID: repoID, + PollIntervalMS: pollInterval.Milliseconds(), + Types: filterTypes, + ExcludeTypes: excludeTypes, + Labels: filterLabels, + ExcludeLabels: excludeLabels, + Assignee: filterAssignee, }) if err != nil { - return fmt.Errorf("failed to create orchestrator: %w", err) + return fmt.Errorf("failed to start orchestrator run: %w", err) } - // Always create stats collector for history tracking - runID := uuid.New().String() - runStats := &runStatsCollector{ - runID: runID, - workDir: absWorkdir, - startTime: time.Now(), + if verbose { + fmt.Printf("Started orchestrator run: %s\n", runID) } - // Get repo ID for IPC calls (empty string if repo is nil) - repoID := "" - if repo != nil { - repoID = repo.ID + // Set up signal handler to stop the run on Ctrl+C + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + fmt.Fprintln(os.Stderr, "\nInterrupted, stopping orchestrator run...") + if err := stopDaemonRun(runID); err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to stop run: %v\n", err) + } + cancel() + }() + + // Stream events via WebSocket until the run completes + return streamRunEvents(ctx, runID, verbose) +} + +// startRunRequest matches the daemon's ExecuteRunRequest +type startRunRequest struct { + WorkDir string `json:"work_dir"` + OutputDir string `json:"output_dir,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + Verbose bool `json:"verbose,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + UseBwrap bool `json:"use_bwrap,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + MaxPriority int `json:"max_priority,omitempty"` + ResolverTimeoutMS int64 `json:"resolver_timeout_ms,omitempty"` + RepoID string `json:"repo_id,omitempty"` + PollIntervalMS int64 `json:"poll_interval_ms,omitempty"` + Types []string `json:"types,omitempty"` + ExcludeTypes []string `json:"exclude_types,omitempty"` + Labels []string `json:"labels,omitempty"` + ExcludeLabels []string `json:"exclude_labels,omitempty"` + Assignee string `json:"assignee,omitempty"` +} + +// startRunResponse matches the daemon's ExecuteRunResponse +type startRunResponse struct { + Success bool `json:"success"` + RunID string `json:"run_id,omitempty"` + Error string `json:"error,omitempty"` +} + +// stopRunRequest matches the daemon's StopRunRequest +type stopRunRequest struct { + RunID string `json:"run_id"` +} + +// stopRunResponse matches the daemon's StopRunResponse +type stopRunResponse struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` +} + +// startDaemonRun calls the daemon's /api/orchestrator/run endpoint to start a run +func startDaemonRun(ctx context.Context, req startRunRequest) (string, error) { + url := fmt.Sprintf("http://localhost:%d/api/orchestrator/run", runtime.DefaultDaemonPort) + + body, err := json.Marshal(req) + if err != nil { + return "", fmt.Errorf("failed to encode request: %w", err) } - // Set up callbacks for history tracking and IPC - // Note: parentAgentID is empty for top-level orchestrated agents - // Child agents (e.g., resolvers) will populate this when spawned - callbacks := &orchestrator.EventCallbacks{ - OnAgentStartFn: func(_ context.Context, taskID string, task *beads.Task) { - runStats.recordTaskStart(taskID, task) - agentID := makeAgentID(runID, taskID) - // Record agent ID for parent-child tracking (resolver agents need this) - orch.SetAgentID(taskID, agentID) - parentAgentID := "" // Top-level agents have no parent - if err := ipcClient.SendAgentStart(agentID, runID, taskID, task.Title, task.Description, parentAgentID, repoID); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send agent start: %v\n", err) - } - }, - OnOutputFn: func(_ context.Context, taskID string, output string, isError bool) { - agentID := makeAgentID(runID, taskID) - if err := ipcClient.SendAgentOutput(agentID, output, isError); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send agent output: %v\n", err) - } - }, - OnLiveFeedFn: func(_ context.Context, taskID string, event *agent.LiveFeedEvent) { - agentID := makeAgentID(runID, taskID) - if err := ipcClient.SendAgentLiveFeed(agentID, string(event.EventType), event.RawData); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send agent live feed: %v\n", err) - } - }, - OnDoneFn: func(_ context.Context, taskID string, result *agent.Result) { - runStats.recordResult(taskID, result, true) - agentID := makeAgentID(runID, taskID) - parentAgentID := "" // Top-level agents have no parent - // Note: Commit events are sent by the merge processor after merge completes, - // using the actual merged commit hashes (not overlay commits which may be destroyed) - ipcResult := convertToIPCResult(result) - if err := ipcClient.SendAgentDone(agentID, parentAgentID, ipcResult); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send agent done: %v\n", err) - } - }, - OnFailFn: func(_ context.Context, taskID string, result *agent.Result) { - runStats.recordResult(taskID, result, false) - agentID := makeAgentID(runID, taskID) - parentAgentID := "" // Top-level agents have no parent - // Note: Commit events are sent by the merge processor after merge completes, - // using the actual merged commit hashes (not overlay commits which may be destroyed) - ipcResult := convertToIPCResult(result) - execErr := fmt.Errorf("%s", result.Error) - if err := ipcClient.SendAgentFail(agentID, parentAgentID, execErr, ipcResult); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send agent fail: %v\n", err) - } - }, + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &bodyReader{data: body}) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.ContentLength = int64(len(body)) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(httpReq) + if err != nil { + return "", fmt.Errorf("failed to call daemon: %w", err) } + defer func() { _ = resp.Body.Close() }() - orch.SetCallbacks(callbacks) + var result startRunResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to decode response: %w", err) + } - // Pass IPC client to orchestrator for resolver agent events - orch.SetIPCClient(ipcClient) + if !result.Success { + return "", fmt.Errorf("daemon returned error: %s", result.Error) + } - // Pass repo ID to orchestrator for resolver agent tracking - if repo != nil { - orch.SetRepoID(repo.ID) + return result.RunID, nil +} + +// bodyReader wraps a byte slice for use as io.ReadCloser +type bodyReader struct { + data []byte + pos int +} + +func (r *bodyReader) Read(p []byte) (n int, err error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + n = copy(p, r.data[r.pos:]) + r.pos += n + return n, nil +} + +func (r *bodyReader) Close() error { + return nil +} + +// stopDaemonRun calls the daemon's /api/orchestrator/run/stop endpoint +func stopDaemonRun(runID string) error { + url := fmt.Sprintf("http://localhost:%d/api/orchestrator/run/stop", runtime.DefaultDaemonPort) + + req := stopRunRequest{RunID: runID} + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to encode request: %w", err) + } + + httpReq, err := http.NewRequest(http.MethodPost, url, &bodyReader{data: body}) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.ContentLength = int64(len(body)) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to call daemon: %w", err) } + defer func() { _ = resp.Body.Close() }() + + var result stopRunResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + + if !result.Success { + return fmt.Errorf("daemon returned error: %s", result.Error) + } + + return nil +} + +// streamRunEvents connects to the WebSocket and streams events until the run completes +func streamRunEvents(ctx context.Context, runID string, verbose bool) error { + // For now, just poll the run status until it completes + // TODO: Implement WebSocket streaming for real-time output + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + status, err := getRunStatus(runID) + if err != nil { + if verbose { + fmt.Fprintf(os.Stderr, "warning: failed to get run status: %v\n", err) + } + continue + } - // Pass run ID to orchestrator for unique agent ID generation - orch.SetRunID(runID) + if verbose { + fmt.Printf("Run status: %s (tasks: %d/%d, failed: %d)\n", + status.Status, status.TasksDone, status.TasksTotal, status.TasksFailed) + } - // Get initial ready tasks to send task count - beadsClient, err := beads.NewClient(absWorkdir) - if err == nil { - tasks, err := beadsClient.Ready(ctx) - if err == nil && len(tasks) > 0 { - if err := ipcClient.SendRunStarted(runID, len(tasks), repo); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send run started: %v\n", err) + // Check if run is terminal + switch status.Status { + case "completed": + fmt.Printf("Run completed: %d/%d tasks succeeded\n", status.TasksDone, status.TasksTotal) + return nil + case "failed": + return fmt.Errorf("run failed: %s", status.Error) + case "cancelled": + return fmt.Errorf("run cancelled") } } } +} - // Send IPC completion after orchestration finishes - // Note: Run persistence is handled by the daemon via the IPC events - // (run_started creates the run, run_completed updates it with final stats) - defer func() { - if dryRun { - return - } +// runStatus matches the daemon's RunStatusWire +type runStatus struct { + ID string `json:"id"` + RepoPath string `json:"repo_path"` + RepoID string `json:"repo_id,omitempty"` + Status string `json:"status"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time,omitempty"` + Error string `json:"error,omitempty"` + TasksTotal int `json:"tasks_total"` + TasksDone int `json:"tasks_done"` + TasksFailed int `json:"tasks_failed"` +} - // Send IPC completion - daemon will persist the run stats - stats := runStats.getStats() - if err := ipcClient.SendRunCompleted(runID, stats); err != nil && verbose { - fmt.Fprintf(os.Stderr, "warning: failed to send run completed: %v\n", err) - } - }() +// runStatusResponse matches the daemon's RunStatusResponse +type runStatusResponse struct { + Success bool `json:"success"` + Run *runStatus `json:"run,omitempty"` + Error string `json:"error,omitempty"` +} + +// getRunStatus queries the daemon for run status +func getRunStatus(runID string) (*runStatus, error) { + url := fmt.Sprintf("http://localhost:%d/api/orchestrator/runs/%s", runtime.DefaultDaemonPort, runID) - return orch.Run(ctx) + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to call daemon: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + var result runStatusResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + if !result.Success { + return nil, fmt.Errorf("daemon returned error: %s", result.Error) + } + + return result.Run, nil } // convertToIPCResult converts agent.Result to ipc.AgentResult +// (kept for resumeInterruptedAgents) func convertToIPCResult(result *agent.Result) *ipc.AgentResult { ipcResult := &ipc.AgentResult{ ExitCode: result.ExitCode, @@ -464,91 +546,6 @@ func convertToIPCResult(result *agent.Result) *ipc.AgentResult { return ipcResult } -// makeAgentID creates a unique agent ID by combining run ID prefix with task ID. -// Format: agent-{runID[:8]}-{taskID} -// This ensures agent IDs are unique per run, even when retrying the same task. -func makeAgentID(runID, taskID string) string { - // Use first 8 characters of run ID as prefix for readability - prefix := runID - if len(prefix) > 8 { - prefix = prefix[:8] - } - return fmt.Sprintf("agent-%s-%s", prefix, taskID) -} - -// runStatsCollector tracks statistics across all agents in a run -type runStatsCollector struct { - runID string - workDir string - startTime time.Time - totalTasks int - succeeded int - failed int - inputTokens int - outputTokens int - cacheCreationTokens int - cacheReadTokens int - costUSD float64 - totalTurns int - filesChanged int - gitCommits int - conflictsRes int - mu sync.Mutex -} - -func (r *runStatsCollector) recordTaskStart(taskID string, task *beads.Task) { - // Task start is tracked via IPC to daemon for real-time UI - // Individual agent records are persisted by the daemon -} - -func (r *runStatsCollector) recordResult(taskID string, result *agent.Result, success bool) { - r.mu.Lock() - defer r.mu.Unlock() - - r.totalTasks++ - if success { - r.succeeded++ - } else { - r.failed++ - } - - if result.Output != nil { - r.inputTokens += result.Output.TotalInputTokens - r.outputTokens += result.Output.TotalOutputTokens - r.cacheCreationTokens += result.Output.CacheCreationInputTokens - r.cacheReadTokens += result.Output.CacheReadInputTokens - r.costUSD += result.Output.CostUSD - r.totalTurns += result.Output.NumTurns - } - - r.filesChanged += len(result.Changes) - - if result.GitState != nil { - r.gitCommits += len(result.GitState.NewCommits) - } -} - -func (r *runStatsCollector) getStats() *ipc.RunStats { - r.mu.Lock() - defer r.mu.Unlock() - - return &ipc.RunStats{ - TotalTasks: r.totalTasks, - SucceededTasks: r.succeeded, - FailedTasks: r.failed, - TotalDuration: time.Since(r.startTime).Seconds(), - TotalInputTokens: r.inputTokens, - TotalOutputTokens: r.outputTokens, - TotalCacheCreationInputToken: r.cacheCreationTokens, - TotalCacheReadInputTokens: r.cacheReadTokens, - TotalCostUSD: r.costUSD, - TotalTurns: r.totalTurns, - FilesChanged: r.filesChanged, - GitCommits: r.gitCommits, - ConflictsResolved: r.conflictsRes, - } -} - // resumableAgentInfo is the wire format for resumable agent data from the daemon type resumableAgentInfo struct { AgentID string `json:"agent_id"` @@ -667,55 +664,3 @@ func resumeInterruptedAgents(ctx context.Context, workDir string, useBwrap, verb return resumed, errors } -// buildCLIRulesOverrides builds rules settings and override tracking from CLI flags. -// Returns (nil, nil) if no CLI flags were set (use config.toml as-is). -func buildCLIRulesOverrides(cmd *cobra.Command) (*config.RulesSettings, *orchestrator.RulesOverrides) { - // Check if any CLI flags were set - hasOverrides := cmd.Flags().Changed("max-priority") || - cmd.Flags().Changed("type") || - cmd.Flags().Changed("exclude-type") || - cmd.Flags().Changed("label") || - cmd.Flags().Changed("exclude-label") || - cmd.Flags().Changed("assignee") - - if !hasOverrides { - return nil, nil // No CLI overrides, orchestrator uses config.toml as-is - } - - // Build overrides from CLI flags only - rules := config.DefaultRulesSettings() - overrides := &orchestrator.RulesOverrides{} - - if cmd.Flags().Changed("max-priority") { - rules.PriorityMax = maxPriority - overrides.PriorityMax = true - } - - if cmd.Flags().Changed("type") { - rules.Types = filterTypes - overrides.Types = true - } - - if cmd.Flags().Changed("exclude-type") { - rules.ExcludeTypes = excludeTypes - overrides.ExcludeTypes = true - } - - if cmd.Flags().Changed("label") { - rules.Labels = filterLabels - overrides.Labels = true - } - - if cmd.Flags().Changed("exclude-label") { - rules.ExcludeLabels = excludeLabels - overrides.ExcludeLabels = true - } - - if cmd.Flags().Changed("assignee") { - rules.Assignee = filterAssignee - overrides.Assignee = true - } - - return &rules, overrides -} - diff --git a/docs/design/active-orchestrator-model.md b/docs/design/active-orchestrator-model.md index eb2ff9e6..9132bac5 100644 --- a/docs/design/active-orchestrator-model.md +++ b/docs/design/active-orchestrator-model.md @@ -1,7 +1,7 @@ # Design: Active Orchestrator Model **Status**: Draft -**Epic**: canopy-i4sm +**Epic**: canopy-2360 **Author**: Design Agent **Date**: 2026-01-23 @@ -40,23 +40,23 @@ Key characteristics: 4. **Dashboard confusion**: Switching between run-centric (filtering by run) and repo-centric (the repo I'm working on) mental models -5. **Session continuity**: No conceptual continuity between runs on the same repo +5. **Run continuity**: No conceptual continuity between runs on the same repo ## Proposed Design -### Core Concept: Always-Running Orchestrator +### Core Concept: Always-Watching Orchestrator Shift from "discrete runs" to "repo has an orchestrator that can be activated/deactivated": ``` Repository - └─ Orchestrator (always exists when repo is registered) - ├─ State: idle | active | paused - ├─ RulesEngine (persistent) - ├─ Config (persistent) - └─ Sessions (historical tracking) - ├─ Session 1 (was: Run) - ├─ Session 2 + └─ OrchestratorLifecycle (created on activation) + ├─ State: off | idle | active | paused + ├─ RulesEngine (persistent while activated) + ├─ Config (persistent in .canopy/config.toml) + └─ Runs (historical tracking) + ├─ Run 1 + ├─ Run 2 └─ ... ``` @@ -66,135 +66,195 @@ Repository type OrchestratorState string const ( - OrchestratorIdle OrchestratorState = "idle" // Ready but not processing + OrchestratorOff OrchestratorState = "off" // Not activated, no instance exists + OrchestratorIdle OrchestratorState = "idle" // Activated, watching for work, none available OrchestratorActive OrchestratorState = "active" // Processing tasks - OrchestratorPaused OrchestratorState = "paused" // Active but temporarily stopped + OrchestratorPaused OrchestratorState = "paused" // Temporarily stopped (resumes on unpause) ) ``` **State transitions:** -- `idle → active`: User clicks "Start" (or CLI `canopy run`) -- `active → paused`: User clicks "Pause" (or CLI signal) -- `paused → active`: User clicks "Resume" -- `active → idle`: Run completes or user clicks "Stop" -- `paused → idle`: User clicks "Stop" + +``` + ┌──────────────────────────────────┐ + │ │ + ▼ │ +┌─────┐ activate ┌──────┐ work available ┌────────┴─┐ +│ Off │ ─────────► │ Idle │ ◄──────────────► │ Active │ +└─────┘ └──────┘ work exhausted └────────┬─┘ + ▲ │ │ │ + │ │ deactivate │ │ + │ ▼ │ │ + │ (destroyed)◄───────────────────┘ │ + │ │ + │ │ pause + │ ▼ + │ ┌──────────┐ + │ │ Paused │ + │ └────┬─────┘ + │ │ + │◄────────────────────────────────────────────────┘ + deactivate +``` + +**Key behaviors:** +- `off → idle`: Activation creates OrchestratorLifecycle, starts watching for work +- `idle ↔ active`: Automatic transitions as work becomes available/exhausted +- `active → paused`: User pauses; orchestrator stops dispatching but keeps watching +- `paused → active`: User resumes; orchestrator continues processing +- `* → off`: Deactivation (context cancellation) destroys the lifecycle + +**Critical invariant:** An activated orchestrator **never exits on its own**. It continuously polls for available work and transitions between idle and active states. Only explicit deactivation (user action or context cancellation) terminates it. ### What's Already Implemented -Looking at `pkg/daemon/orchestrator_manager.go`, much of this is already in place: +Looking at `pkg/daemon/orchestrator_manager.go`, the structural foundation exists: ```go -type RepoOrchestrator struct { +type OrchestratorLifecycle struct { RepoPath string RepoID string - State OrchestratorState // idle | active | paused - RunID string // Current session ID when active + State OrchestratorState // off | idle | active | paused + RunID string // Current run ID when active rulesEngine *rules.Engine // Persistent rules engine // ... } ``` -The `RegisterRepo()` function creates an always-running orchestrator: -```go -func (m *OrchestratorManager) RegisterRepo(repoPath string, repoID string) (*RepoOrchestrator, error) -``` +**canopy-o7ww added:** +- The `OrchestratorLifecycle` wrapper struct +- State enum with `off` state +- Basic state management infrastructure + +**What canopy-o7ww did NOT add:** +- Always-watching behavior (orchestrator still exits when work is exhausted) +- Automatic idle ↔ active transitions based on work availability +- Polling loop for continuous work detection + +The current implementation still treats "idle" as essentially "off" — there is no persistent watching behavior. The orchestrator terminates when its current batch of work completes rather than polling for new work. ### What Needs to Change -#### 1. Mental Model in UI +#### 1. Core Orchestration Loop + +The orchestrator needs a persistent watch loop: + +```go +func (o *OrchestratorLifecycle) Run(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() // Only way to exit + default: + tasks := o.pollForWork() + if len(tasks) == 0 { + o.setState(OrchestratorIdle) + time.Sleep(o.pollInterval) + continue + } + o.setState(OrchestratorActive) + o.dispatchTasks(ctx, tasks) + } + } +} +``` + +#### 2. Mental Model in UI **Current**: "Start Run" / "Stop Run" with run selector -**Proposed**: "Activate" / "Deactivate" with session history +**Proposed**: "Activate" / "Deactivate" with run history The UI should present: -- **Primary state**: Is the orchestrator active? -- **Secondary**: Which session are you viewing? (for history) +- **Primary state**: Is the orchestrator active/idle/off? +- **Secondary**: Which run are you viewing? (for history) ``` ┌─────────────────────────────────────────────────────────────────┐ │ [Repository Selector] │ Orchestrator: Active ● │ [Pause] [Stop]│ -│ │ Session: abc123 (2h ago)│ │ +│ │ Run: abc123 (2h ago) │ │ ├─────────────────────────────────────────────────────────────────┤ │ │ -│ [Agent Cards - filtered by selected session] │ +│ [Agent Cards - filtered by selected run] │ │ │ └─────────────────────────────────────────────────────────────────┘ ``` -#### 2. Terminology Changes +#### 3. Terminology -| Current Term | Proposed Term | Notes | -|--------------|---------------|-------| -| Run | Session | Historical work period | -| Run ID | Session ID | For filtering/tracking | -| Start Run | Activate | State transition | -| Stop Run | Deactivate | State transition | -| Active Run | Current Session | When orchestrator is active | +| Term | Meaning | +|------|---------| +| Run | A period of work within an activated orchestrator | +| OrchestratorLifecycle | Wrapper managing orchestrator state and lifecycle | +| Orchestrator | The execution engine that dispatches agents | +| OrchestratorManager | Daemon component that manages lifecycles across repos | +| Activate | Create lifecycle and start watching | +| Deactivate | Cancel context, destroy lifecycle | -#### 3. API Endpoints +#### 4. API Endpoints **Keep (rename optional)**: - `POST /api/orchestrator/run` → `POST /api/orchestrator/activate` - `POST /api/orchestrator/run/stop` → `POST /api/orchestrator/deactivate` -- `GET /api/daemon/runs` → `GET /api/daemon/sessions` **New**: - `GET /api/repos/:repo_id/orchestrator` - Get orchestrator state - `PATCH /api/repos/:repo_id/orchestrator` - Update config (concurrency, etc.) -#### 4. Persistence Between Sessions +#### 5. Persistence **What persists (tied to repo):** -- Rules configuration (already in `.canopy/config.toml`) -- Rules engine state (enabled/disabled rules) +- Rules configuration (in `.canopy/config.toml`) - Orchestrator configuration (concurrency, max_priority, etc.) - Repository registration -**What resets (per session):** +**What resets (per activation):** - Active agents -- In-progress tasks (returned to "ready" on session end) +- In-progress tasks (returned to "ready" on deactivation) - Pause state - Real-time metrics -**What accumulates (across sessions):** -- Session history (in SQLite) +**What accumulates (across runs):** +- Run history (in SQLite) - Agent history (in SQLite) -- Cost/token aggregates (per session) +- Cost/token aggregates (per run) ### Migration Path -#### Phase 1: Backend Alignment (Mostly Done) +#### Phase 1: Always-Watching Behavior -The backend already has `RepoOrchestrator` with state management. Remaining work: -- [ ] Ensure rules engine persists in idle state (already implemented) -- [ ] Add `GET /api/repos/:repo_id/orchestrator` endpoint -- [ ] Add `PATCH /api/repos/:repo_id/orchestrator` for config updates +Implement the core watch loop so activated orchestrators poll for work: +- [ ] Add polling loop to OrchestratorLifecycle +- [ ] Implement automatic idle ↔ active transitions +- [ ] Ensure context cancellation is the only exit path +- [ ] Add configurable poll interval -#### Phase 2: API Deprecation Layer +#### Phase 2: API Updates Add new endpoints alongside old ones: ```go // New router.POST("/api/orchestrator/activate", h.HandleActivate) router.POST("/api/orchestrator/deactivate", h.HandleDeactivate) -router.GET("/api/repos/:repo_id/sessions", h.HandleListSessions) // Old (deprecated but functional) router.POST("/api/orchestrator/run", h.HandleExecuteRun) // calls HandleActivate internally router.POST("/api/orchestrator/run/stop", h.HandleStopRun) // calls HandleDeactivate internally ``` +- [ ] Add `GET /api/repos/:repo_id/orchestrator` endpoint +- [ ] Add `PATCH /api/repos/:repo_id/orchestrator` for config updates + #### Phase 3: Dashboard Updates 1. Rename UI elements (Start Run → Activate, etc.) -2. Add orchestrator state indicator to header -3. Change "Run Selector" to "Session Selector" with historical context -4. Update state store terminology +2. Add orchestrator state indicator showing off/idle/active/paused +3. Update state store to reflect new state model +4. Show "Watching for work..." in idle state #### Phase 4: CLI Updates 1. `canopy run` becomes `canopy activate` (alias `canopy run` for backward compat) -2. `canopy status` shows orchestrator state +2. `canopy status` shows orchestrator state (off/idle/active/paused) 3. `canopy config set concurrency=8` modifies repo orchestrator ### Dashboard State Store Changes @@ -210,16 +270,16 @@ interface StateStore { // Proposed interface StateStore { - // Orchestrator state (always present when repo selected) + // Orchestrator state (present when repo has activated orchestrator) orchestrator: { - state: 'idle' | 'active' | 'paused'; + state: 'off' | 'idle' | 'active' | 'paused'; config: OrchestratorConfig; - currentSessionId: string | null; // null when idle + currentRunId: string | null; // null when off }; - // Session filtering (for history view) - selectedSessionId: string; // empty = all sessions - sessions: Session[]; // Historical sessions + // Run filtering (for history view) + selectedRunId: string; // empty = all runs + runs: Run[]; // Historical runs } ``` @@ -231,26 +291,26 @@ interface StateStore { 3. **Config changes feel natural**: Adjusting concurrency is tweaking the orchestrator, not a run -4. **Session continuity**: Clear that sessions are work periods within a persistent orchestrator +4. **Run continuity**: Clear that runs are work periods within a persistent orchestrator -5. **Better state representation**: Dashboard shows orchestrator state, not just "is there a run?" +5. **Better state representation**: Dashboard shows orchestrator state (off/idle/active/paused), not just "is there a run?" -## Open Questions +6. **No orphaned work**: Orchestrator watches continuously until explicitly deactivated -1. **Session auto-naming?** Should sessions have human-readable names or descriptions? +## Open Questions -2. **Session grouping?** Should multiple activations in quick succession be grouped? +1. **Run auto-naming?** Should runs have human-readable names or descriptions? -3. **Watch mode sessions?** How do watch mode iterations relate to sessions? +2. **Multi-repo support?** How does this model extend to working on multiple repos? -4. **Multi-repo support?** How does this model extend to working on multiple repos? +3. **Poll interval tuning?** What's the right balance between responsiveness and resource usage? ## Implementation Tasks -See child tasks of canopy-i4sm for implementation breakdown. +See child tasks of canopy-2360 for implementation breakdown. ## References -- `pkg/daemon/orchestrator_manager.go` - Current RepoOrchestrator implementation +- `pkg/daemon/orchestrator_manager.go` - Current OrchestratorLifecycle implementation - `web/dashboard/src/stores/stateStore.ts` - Dashboard state model - `pkg/daemon/rules_handlers.go` - Rules API (already supports standalone engine) diff --git a/docs/design/agent-lifecycle-state-machine.md b/docs/design/agent-lifecycle-state-machine.md new file mode 100644 index 00000000..96d96a55 --- /dev/null +++ b/docs/design/agent-lifecycle-state-machine.md @@ -0,0 +1,454 @@ +# Agent Lifecycle State Machine + +## Status: Approved (Architect Review Complete) + +## Problem Statement + +The current agent status model uses three independent fields updated from different code paths: + +```go +type AgentState struct { + Status AgentStatus // starting, running, completed, failed, timed_out, cancelled + MergeStatus MergeStatus // none, pending, acquiring, merging, resolving, merged, failed, skipped, merged_needs_repair + ValidationStatus string // pending, running, passed, failed, skipped, repairing + RepairAttempts int +} +``` + +This creates several problems: + +1. **Race conditions**: Merge processor updates `MergeStatus` while `OnDoneFn` callback updates `Status` - these are not synchronized, leading to windows where task shows "completed" but agent shows "running" + +2. **Invalid state combinations**: Nothing prevents `Status=completed` with `ValidationStatus=repairing` or other nonsensical combinations + +3. **Scattered update logic**: Status fields are mutated from: + - `pkg/mergequeue/processor.go` (merge status, validation status) + - `pkg/daemon/orchestrator_manager.go` (agent status via callbacks) + - `pkg/daemon/state.go` (event handlers) + - `pkg/repairagent/spawner.go` (repair status) + +4. **UI confusion**: Dashboard must interpret multiple fields to determine what to display. `canopy ps` shows "running" during validation/repair because main `Status` field isn't updated. + +5. **No transition validation**: Any code can set any status at any time with no enforcement of valid transitions. + +## Proposed Solution + +Replace the multi-field model with a single **Agent Lifecycle State Machine** that: +- Has one authoritative state field +- Defines explicit valid transitions +- Rejects invalid transitions with errors +- Emits events atomically with state changes +- Provides computed properties for backwards compatibility + +## State Diagram + +``` + ┌────────────────┐ + │ Starting │ + └───────┬────────┘ + │ AgentSpawned + ▼ + ┌────────────────┐ + ┌───────────────►│ Running │◄──────────────────┐ + │ └───────┬────────┘ │ + │ │ WorkComplete │ + │ Retry ▼ │ Retry + │ (attempts remaining) ┌────────────────┐ │ (attempts remaining) + │ │ QueuedForMerge│ │ + │ └───────┬────────┘ │ + │ │ MergeStarted │ + │ ▼ │ + │ ┌────────────────┐ │ + │ ┌────────│ Merging │────────┐ │ + │ │ └────────────────┘ │ │ + │ │ Conflict Success│ │ + │ ▼ ▼ │ + │ ┌────────────────┐ ┌────────────────┐ + │ │ Resolving │───────────────►│ Validating │ + │ └───────┬────────┘ Resolved └───────┬────────┘ + │ │ │ + │ │ ResolveFailed ┌────────┼────────┐ + │ ▼ │ │ │ + │ ┌────────────────┐ Passed │ Failed │ Skipped│ + │ │ MergeFailed │◄──────────────┘ │ │ + │ └───────┬────────┘ ▼ │ + │ │ ┌────────────────┐│ + └──────────────┤ │ Repairing ││ + │ └───────┬────────┘│ + │ │ │ + │ RepairSuccess │ │ + │ ┌────────────────────────────┘ │ + │ │ │ + │ │ RepairExhausted │ + │ │ │ │ + ▼ ▼ ▼ ▼ + ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ + │ Failed │ │ NeedsAttention │ │ Completed │ + └────────────────┘ └────────────────┘ └────────────────┘ + +Terminal States: Failed, Completed, NeedsAttention, Cancelled, TimedOut +``` + +## State Definitions + +| State | Description | What User Sees | +|-------|-------------|----------------| +| `Starting` | Agent process spawning | "Starting..." | +| `Running` | Agent executing work (Claude API) | "Running" with live output | +| `QueuedForMerge` | Work done, waiting in merge queue | "Queued (position N)" | +| `Merging` | Applying changes to repo (includes pre-commit hook repair if needed) | "Merging..." | +| `Resolving` | Conflict resolver agent running | "Resolving conflicts..." | +| `Validating` | Post-merge validation running | "Validating..." | +| `Repairing` | Repair agent fixing validation failure | "Repairing (attempt N/M)" | +| `MergeFailed` | Merge/pre-commit failed, will retry if attempts remain | "Merge failed (retry N/M)" | +| `Completed` | Successfully done | "Completed" (green) | +| `Failed` | Terminal failure | "Failed" (red) | +| `NeedsAttention` | Repair exhausted, human needed | "Needs attention" (orange) | +| `Cancelled` | User cancelled | "Cancelled" | +| `TimedOut` | Exceeded time limit | "Timed out" | + +## Transition Table + +| From State | Event | To State | Condition | +|------------|-------|----------|-----------| +| Starting | AgentSpawned | Running | - | +| Running | WorkComplete | QueuedForMerge | - | +| Running | WorkFailed | Failed | max_retries exhausted | +| Running | WorkFailed | Running | retry attempt | +| QueuedForMerge | MergeStarted | Merging | - | +| Merging | MergeSuccess | Validating | validation enabled | +| Merging | MergeSuccess | Completed | validation disabled | +| Merging | MergeConflict | Resolving | - | +| Merging | MergeFailed | MergeFailed | - | +| Resolving | ResolveSuccess | Validating | - | +| Resolving | ResolveFailed | MergeFailed | - | +| Validating | ValidationPassed | Completed | - | +| Validating | ValidationFailed | Repairing | repair enabled, attempts remaining | +| Validating | ValidationFailed | NeedsAttention | repair exhausted | +| Validating | ValidationFailed | NeedsAttention | lenient mode, repair disabled | +| Validating | ValidationFailed | Failed | strict mode | +| Validating | ValidationSkipped | Completed | - | +| Repairing | RepairComplete | Validating | re-run validation | +| MergeFailed | Retry | Running | attempts remaining | +| MergeFailed | RetriesExhausted | Failed | - | +| * | Cancel | Cancelled | - | +| * | Timeout | TimedOut | - | + +## Implementation Design + +### Core Types + +```go +// pkg/lifecycle/state.go + +// AgentLifecycleState represents the current state in the agent lifecycle. +type AgentLifecycleState string + +const ( + StateStarting AgentLifecycleState = "starting" + StateRunning AgentLifecycleState = "running" + StateQueuedForMerge AgentLifecycleState = "queued_for_merge" + StateMerging AgentLifecycleState = "merging" + StateResolving AgentLifecycleState = "resolving" + StateValidating AgentLifecycleState = "validating" + StateRepairing AgentLifecycleState = "repairing" + StateMergeFailed AgentLifecycleState = "merge_failed" + StateCompleted AgentLifecycleState = "completed" + StateFailed AgentLifecycleState = "failed" + StateNeedsAttention AgentLifecycleState = "needs_attention" + StateCancelled AgentLifecycleState = "cancelled" + StateTimedOut AgentLifecycleState = "timed_out" +) + +// AgentEvent represents an event that can trigger a state transition. +type AgentEvent string + +const ( + EventAgentSpawned AgentEvent = "agent_spawned" + EventWorkComplete AgentEvent = "work_complete" + EventWorkFailed AgentEvent = "work_failed" + EventMergeStarted AgentEvent = "merge_started" + EventMergeSuccess AgentEvent = "merge_success" + EventMergeConflict AgentEvent = "merge_conflict" + EventMergeFailed AgentEvent = "merge_failed" + EventResolveSuccess AgentEvent = "resolve_success" + EventResolveFailed AgentEvent = "resolve_failed" + EventValidationPassed AgentEvent = "validation_passed" + EventValidationFailed AgentEvent = "validation_failed" + EventValidationSkipped AgentEvent = "validation_skipped" + EventRepairComplete AgentEvent = "repair_complete" + EventRetry AgentEvent = "retry" + EventRetriesExhausted AgentEvent = "retries_exhausted" + EventCancel AgentEvent = "cancel" + EventTimeout AgentEvent = "timeout" +) + +// TransitionContext provides context for state transitions. +type TransitionContext struct { + ValidationEnabled bool + RepairEnabled bool + StrictMode bool + AttemptsRemaining int + RepairAttempt int + MaxRepairAttempts int + QueuePosition int + Error string +} + +// AgentLifecycle manages state transitions for a single agent. +type AgentLifecycle struct { + mu sync.RWMutex + state AgentLifecycleState + stateHistory []StateTransition + context TransitionContext + onTransition func(from, to AgentLifecycleState, event AgentEvent) +} + +// StateTransition records a state change. +type StateTransition struct { + From AgentLifecycleState + To AgentLifecycleState + Event AgentEvent + Timestamp time.Time + Context TransitionContext +} + +// Transition attempts to transition to a new state based on an event. +// Returns an error if the transition is invalid. +func (l *AgentLifecycle) Transition(event AgentEvent, ctx TransitionContext) error { + l.mu.Lock() + defer l.mu.Unlock() + + newState, err := l.nextState(l.state, event, ctx) + if err != nil { + return fmt.Errorf("invalid transition: %s + %s: %w", l.state, event, err) + } + + oldState := l.state + l.state = newState + l.context = ctx + l.stateHistory = append(l.stateHistory, StateTransition{ + From: oldState, + To: newState, + Event: event, + Timestamp: time.Now(), + Context: ctx, + }) + + if l.onTransition != nil { + l.onTransition(oldState, newState, event) + } + + return nil +} + +// State returns the current state (thread-safe). +func (l *AgentLifecycle) State() AgentLifecycleState { + l.mu.RLock() + defer l.mu.RUnlock() + return l.state +} + +// IsTerminal returns true if the current state is terminal. +func (l *AgentLifecycle) IsTerminal() bool { + switch l.State() { + case StateCompleted, StateFailed, StateNeedsAttention, StateCancelled, StateTimedOut: + return true + } + return false +} +``` + +### Backwards Compatibility + +```go +// LegacyStatus returns the legacy AgentStatus for backwards compatibility. +func (l *AgentLifecycle) LegacyStatus() AgentStatus { + switch l.State() { + case StateStarting: + return AgentStatusStarting + case StateRunning, StateQueuedForMerge, StateMerging, StateResolving, StateValidating, StateRepairing: + return AgentStatusRunning + case StateCompleted: + return AgentStatusCompleted + case StateFailed, StateMergeFailed, StateNeedsAttention: + return AgentStatusFailed + case StateCancelled: + return AgentStatusCancelled + case StateTimedOut: + return AgentStatusTimedOut + } + return AgentStatusRunning +} + +// LegacyMergeStatus returns the legacy MergeStatus for backwards compatibility. +func (l *AgentLifecycle) LegacyMergeStatus() MergeStatus { + switch l.State() { + case StateStarting, StateRunning: + return MergeStatusNone + case StateQueuedForMerge: + return MergeStatusPending + case StateMerging: + return MergeStatusMerging + case StateResolving: + return MergeStatusResolving + case StateValidating, StateRepairing: + return MergeStatusMerged // validation/repair happen AFTER successful merge + case StateCompleted: + return MergeStatusMerged + case StateFailed, StateMergeFailed: + return MergeStatusFailed + case StateNeedsAttention: + return MergeStatusMergedNeedsRepair + } + return MergeStatusNone +} + +// LegacyValidationStatus returns the legacy validation status string. +func (l *AgentLifecycle) LegacyValidationStatus() string { + switch l.State() { + case StateValidating: + return "running" + case StateRepairing: + return "repairing" + case StateCompleted: + return "passed" + case StateNeedsAttention: + return "failed" + } + return "" +} +``` + +### Integration Points + +1. **Merge Queue Processor**: Replace direct field mutations with `Transition()` calls +2. **RuntimeState**: Store `AgentLifecycle` instead of separate fields +3. **IPC Events**: Map events to `AgentEvent` constants +4. **Dashboard**: Read `lifecycle_state` field, use legacy fields as fallback +5. **canopy ps**: Display `State()` directly instead of interpreting multiple fields + +### Transition Ownership and Integration + +The state machine integrates with the existing event-driven architecture as follows: + +**Transition Call Sites:** +1. **Merge Queue Processor** (`pkg/mergequeue/processor.go`): + - Calls `lifecycle.Transition()` for all merge/validation/repair events + - Replaces scattered `sendMergeStatus()` calls with atomic transitions + - Highest density of state transitions + +2. **Orchestrator Callbacks** (`pkg/daemon/orchestrator_manager.go`): + - `OnAgentStartFn` → `Transition(EventAgentSpawned)` + - `OnDoneFn` → `Transition(EventWorkComplete)` + - `OnFailFn` → `Transition(EventWorkFailed)` + +3. **Event Handlers** (`pkg/daemon/state.go`): + - Subscribe to EventBus events + - Call `lifecycle.Transition()` instead of direct field assignment + - Handle validation/repair status updates + +**Event Publishing:** +- `onTransition` callback publishes to EventBus when state changes +- Maintains event-driven architecture for dashboard/WebSocket updates +- Events contain both new lifecycle state and legacy fields for backwards compat + +**Concurrency:** +- All transitions protected by `AgentLifecycle.mu` mutex +- `Transition()` is atomic: state update + history append + callback +- No gaps where state is inconsistent + +### Child Agent Lifecycle + +Resolver and repair agents are spawned as independent agents with their own lifecycle instances: + +- **Resolver agents**: Spawned when merge conflicts occur. Parent transitions to `Resolving`, waits for resolver to complete, then transitions to `Validating` (success) or `MergeFailed` (failure). + +- **Repair agents**: Spawned when validation fails. Parent transitions to `Repairing`, waits for repair to complete, then transitions to `Validating` (re-run validation). + +**Key properties:** +- Child agents have their own `AgentLifecycle` instance +- Child agent states are independent and queryable (for UI display) +- Parent transitions based on child's **final result** only +- Child completion doesn't directly trigger parent state changes (processor logic handles this) +- Parent-child relationship tracked via `ParentAgentID` and `ChildAgentIDs[]` fields + +## Migration Strategy + +### Phase 1: Add State Machine (Non-Breaking) +- Add `pkg/lifecycle` package with state machine implementation +- Add `LifecycleState` field to `AgentState` (new field, optional) +- Existing fields remain and continue to work +- State machine runs in parallel, logging any divergence + +### Phase 2: Migrate Writers +- Update merge processor to call `Transition()` instead of sending separate events +- Update orchestrator callbacks to use state machine +- Keep legacy field updates for backwards compatibility + +### Phase 3: Migrate Readers +- Update dashboard to prefer `lifecycle_state` over legacy fields +- Update `canopy ps` to use state machine +- Update persistence to store/restore lifecycle state + +### Phase 4: Remove Legacy (Breaking) +- Remove separate `Status`, `MergeStatus`, `ValidationStatus` fields +- Remove legacy compatibility methods +- Clean up redundant event types + +## Testing Strategy + +1. **Unit tests**: Test all valid transitions, reject invalid ones +2. **Property tests**: Verify state machine invariants (terminal states stay terminal, etc.) +3. **Integration tests**: Verify IPC events map correctly to transitions +4. **Divergence tests**: Run old and new systems in parallel, flag any differences + +## Alternatives Considered + +### Alternative 1: Keep Separate Fields, Add Coordinator +Add a coordinator that serializes all status updates through a single channel. Rejected because it doesn't solve the semantic confusion of multiple fields. + +### Alternative 2: Enum with Substates +Use a primary enum with optional substates (e.g., `Running(MergePhase)`). Rejected because it's more complex and Go doesn't have sum types. + +### Alternative 3: Event Sourcing +Store all events and derive state. Considered for future work, but adds significant complexity. The state machine can be extended to event sourcing later. + +## Design Decisions + +Based on architectural review, the following questions have been resolved: + +### 1. Should `QueuedForMerge` be visible to users? + +**YES**. The merge queue position is explicitly tracked (`MergeQueuePos` field) and users need to know when work is done but waiting. Display as "Queued (position N)" in UI. + +### 2. How to handle daemon restart? + +**Restore from `LifecycleState` as source of truth**. During migration (Phases 1-3), database stores both lifecycle state and legacy fields. After Phase 4, only `lifecycle_state` is stored. Legacy fields are derived via `LegacyStatus()` methods for any remaining compatibility needs. + +### 3. Should state history be persisted? + +**In-memory initially**. State history (`stateHistory[]`) is kept in memory for debugging but not persisted to database. Can add persistence later if debugging needs warrant it. + +### 4. How to handle API versioning for clients expecting legacy fields? + +**Dual-field approach during Phases 1-3**. IPC events and JSON responses include: +- `lifecycle_state` (new field) +- `status`, `merge_status`, `validation_status` (legacy fields, derived from lifecycle) + +Dashboard checks for `lifecycle_state` first, falls back to legacy. Remove legacy fields only in Phase 4 as a documented breaking change requiring major version bump. + +## Implementation Risks and Mitigations + +| Risk | Severity | Impact | Mitigation | +|------|----------|--------|------------| +| Invalid transition logic in `nextState()` | **Critical** | Agents stuck in invalid states, orchestration breaks | Comprehensive unit tests covering all 30+ transitions; property-based tests (terminal states never transition out); parallel tracking in Phase 1 to detect divergence | +| Race between `Transition()` and EventBus | **High** | State updates lost, UI shows stale data | `Transition()` holds mutex during callback; EventBus publish is synchronous within lock; add integration test for concurrent transitions | +| Breaking changes to existing clients | **Medium** | Dashboard/CLI breaks on upgrade | Dual-field approach (lifecycle + legacy fields); phased migration over 4 releases; feature flag to enable/disable lifecycle | +| Performance regression | **Low** | Slower agent throughput | Benchmark `Transition()` overhead; state history bounded (e.g., last 100 transitions); profile in staging with 50+ concurrent agents | +| Debugging complexity | **Medium** | Harder to trace state issues | State history kept in memory; log all transitions at DEBUG level; add `canopy ps --show-history` command | + +## References + +- [canopy-jygc](beads://canopy-jygc): Agent appears 'running' after merge complete due to status update race +- [canopy-vf1u](beads://canopy-vf1u): Repair agents run silently - no UI indication parent agent is repairing diff --git a/pkg/agent/executor.go b/pkg/agent/executor.go index 0134cdec..22e8a7e8 100644 --- a/pkg/agent/executor.go +++ b/pkg/agent/executor.go @@ -25,6 +25,7 @@ const processGroupGracePeriod = 3 * time.Second // Result holds the execution result from an agent type Result struct { TaskID string + BeadID string // Original bead ID for commit messages (set by repair/resolver agents) Success bool Output *ClaudeOutput Stdout string @@ -237,12 +238,11 @@ func (e *Executor) Execute(ctx context.Context, task *beads.Task, overlay *sandb cmd.Env = env } - // Apply resource limits (process groups, death signals) on supported platforms - // This is a no-op on non-Linux platforms - // When using bwrap, it handles process isolation internally - if !useBwrap { - setResourceLimits(cmd) - } + // Apply resource limits (process groups, death signals) on supported platforms. + // This is a no-op on non-Linux platforms. + // Process group isolation (Setpgid) is needed for killProcessGroup to work + // on context cancellation, regardless of whether we use bwrap. + setResourceLimits(cmd) // Set up streaming stdout/stderr capture var stderr bytes.Buffer diff --git a/pkg/beads/client.go b/pkg/beads/client.go index 99d61c6a..f174ddc6 100644 --- a/pkg/beads/client.go +++ b/pkg/beads/client.go @@ -52,9 +52,13 @@ type BeadsClient interface { // Done marks a task as completed Done(ctx context.Context, taskID string) error - // Fail marks a task as failed by closing it with a failure reason + // Fail marks a task as failed by reopening it so it can be retried Fail(ctx context.Context, taskID string, reason string) error + // FailPermanently marks a task as permanently failed by closing it and adding needs-investigation label + // This prevents the task from being retried again + FailPermanently(ctx context.Context, taskID string, reason string) error + // NeedsInput marks a task as needing user input (paused state) // The sessionID is preserved so the task can be resumed later NeedsInput(ctx context.Context, taskID string, sessionID string, reason string) error @@ -194,6 +198,23 @@ func (c *Client) Fail(ctx context.Context, taskID string, reason string) error { return err } +// FailPermanently marks a task as permanently failed by closing it and adding needs-investigation label. +// This prevents the task from being automatically retried by the orchestrator. +// The failure reason is recorded in the close event. +func (c *Client) FailPermanently(ctx context.Context, taskID string, reason string) error { + // Close the task with the failure reason + if _, err := c.run(ctx, "close", taskID, "--reason", "FAILED PERMANENTLY: "+reason); err != nil { + return fmt.Errorf("failed to close task: %w", err) + } + + // Add needs-investigation label to exclude from future runs + if _, err := c.run(ctx, "update", taskID, "--labels", "needs-investigation"); err != nil { + return fmt.Errorf("failed to add needs-investigation label: %w", err) + } + + return nil +} + // NeedsInput marks a task as needing user input by setting its status to needs-input. // The sessionID is stored in the notes field so the task can be resumed later. // This prevents the scheduler from picking up the task again until user provides input. diff --git a/pkg/beads/mock.go b/pkg/beads/mock.go index 3a91213c..a2762108 100644 --- a/pkg/beads/mock.go +++ b/pkg/beads/mock.go @@ -29,6 +29,10 @@ type MockClient struct { TaskID string Reason string } + FailPermanently []struct { + TaskID string + Reason string + } NeedsInput []struct { TaskID string SessionID string @@ -64,6 +68,7 @@ type MockClient struct { Start error Done error Fail error + FailPermanently error NeedsInput error Create error CreateWithDescription error @@ -217,6 +222,39 @@ func (m *MockClient) Fail(_ context.Context, taskID string, reason string) error return nil } +// FailPermanently marks a task as permanently failed by closing it and adding needs-investigation label +func (m *MockClient) FailPermanently(_ context.Context, taskID string, reason string) error { + m.mu.Lock() + m.Calls.FailPermanently = append(m.Calls.FailPermanently, struct { + TaskID string + Reason string + }{taskID, reason}) + m.mu.Unlock() + + if m.Errors.FailPermanently != nil { + return m.Errors.FailPermanently + } + + m.mu.Lock() + defer m.mu.Unlock() + + if task, ok := m.Tasks[taskID]; ok { + task.Status = "closed" + // Add needs-investigation label if not already present + hasLabel := false + for _, label := range task.Labels { + if label == "needs-investigation" { + hasLabel = true + break + } + } + if !hasLabel { + task.Labels = append(task.Labels, "needs-investigation") + } + } + return nil +} + // NeedsInput marks a task as needing user input func (m *MockClient) NeedsInput(_ context.Context, taskID string, sessionID string, reason string) error { m.mu.Lock() @@ -386,6 +424,7 @@ func (m *MockClient) Reset() { m.Calls.Start = nil m.Calls.Done = nil m.Calls.Fail = nil + m.Calls.FailPermanently = nil m.Calls.NeedsInput = nil m.Calls.Create = nil m.Calls.CreateWithDescription = nil @@ -401,6 +440,7 @@ func (m *MockClient) Reset() { m.Errors.Start = nil m.Errors.Done = nil m.Errors.Fail = nil + m.Errors.FailPermanently = nil m.Errors.NeedsInput = nil m.Errors.Create = nil m.Errors.CreateWithDescription = nil diff --git a/pkg/beads/mock_test.go b/pkg/beads/mock_test.go index b89a91e6..e3f30705 100644 --- a/pkg/beads/mock_test.go +++ b/pkg/beads/mock_test.go @@ -124,6 +124,44 @@ func TestMockClient_Fail(t *testing.T) { } } +func TestMockClient_FailPermanently(t *testing.T) { + ctx := context.Background() + mock := NewMockClient() + mock.SetTask(&Task{ID: "task-1", Title: "Test", Status: "in_progress", Labels: []string{"bug"}}) + + err := mock.FailPermanently(ctx, "task-1", "max retries exceeded") + if err != nil { + t.Fatalf("FailPermanently() error = %v", err) + } + + if len(mock.Calls.FailPermanently) != 1 { + t.Fatalf("FailPermanently() calls count = %d, want 1", len(mock.Calls.FailPermanently)) + } + if mock.Calls.FailPermanently[0].TaskID != "task-1" { + t.Errorf("FailPermanently() taskID = %s, want task-1", mock.Calls.FailPermanently[0].TaskID) + } + if mock.Calls.FailPermanently[0].Reason != "max retries exceeded" { + t.Errorf("FailPermanently() reason = %s, want 'max retries exceeded'", mock.Calls.FailPermanently[0].Reason) + } + + // Verify status is closed (not open) + if mock.Tasks["task-1"].Status != "closed" { + t.Errorf("Task status = %s, want closed (permanently failed tasks should be closed)", mock.Tasks["task-1"].Status) + } + + // Verify needs-investigation label was added + hasLabel := false + for _, label := range mock.Tasks["task-1"].Labels { + if label == "needs-investigation" { + hasLabel = true + break + } + } + if !hasLabel { + t.Error("FailPermanently() should add needs-investigation label") + } +} + func TestMockClient_Create(t *testing.T) { ctx := context.Background() mock := NewMockClient() diff --git a/pkg/config/config.go b/pkg/config/config.go index 0efddf1b..3615e3a5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,7 +44,7 @@ type RulesSettings struct { Assignee string `toml:"assignee"` // Behavior - // StopWhenEmpty stops the run when no tasks match (for watch mode) + // StopWhenEmpty stops the orchestrator when no tasks match rules StopWhenEmpty bool `toml:"stop_when_empty"` // Concurrency limits @@ -59,6 +59,18 @@ type RulesSettings struct { Custom []CustomRule `toml:"custom"` } +// RuleSource indicates where a rule comes from in the precedence hierarchy. +type RuleSource string + +const ( + // RuleSourceDefault indicates a built-in default rule. + RuleSourceDefault RuleSource = "default" + // RuleSourceConfig indicates a rule loaded from .canopy/config.toml. + RuleSourceConfig RuleSource = "config" + // RuleSourceOverride indicates a run-time override set via the Configure dialog. + RuleSourceOverride RuleSource = "override" +) + // CustomRule defines a named rule with a condition and action. // This allows for more complex filtering logic beyond simple whitelists/blacklists. type CustomRule struct { diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index d70a2038..cec9652a 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -3,6 +3,7 @@ package daemon import ( "context" "fmt" + "time" "github.com/jzila/canopy/pkg/events" "github.com/jzila/canopy/pkg/logging" @@ -58,6 +59,9 @@ type Daemon struct { ipcServerFactory IPCServerFactory httpServer *Server scheduler SchedulerInterface + + // Subscription cleanup + stateUnsubscribe func() } // NewDaemon creates a new daemon instance with the given configuration. @@ -147,6 +151,14 @@ func (d *Daemon) Init() { d.state = NewRuntimeState() } + // Subscribe RuntimeState to EventBus early, before state restoration. + // This ensures lifecycle callbacks can publish events during restoration. + // Without this, agents restored with non-terminal lifecycles would have + // callbacks that silently drop events because eventBus is nil. + if d.stateUnsubscribe == nil { + d.stateUnsubscribe = d.state.SubscribeToEventBus(d.eventBus) + } + // Initialize managers if nil (for backwards compatibility with tests) if d.repoManager == nil { d.repoManager = NewRepositoryManager(nil) @@ -358,12 +370,16 @@ func (d *Daemon) Start() error { d.Init() logging.Debug("eventbus initialized") logging.Debug("runtime state initialized") - - // Subscribe RuntimeState to EventBus to update from IPC events - unsubscribeState := d.state.SubscribeToEventBus(d.eventBus) - defer unsubscribeState() logging.Debug("runtime state subscribed to eventbus") + // Ensure cleanup of state subscription when Start() returns + defer func() { + if d.stateUnsubscribe != nil { + d.stateUnsubscribe() + d.stateUnsubscribe = nil + } + }() + // Initialize HTTP server d.httpServer = NewServerWithDaemon( d.config.Port, @@ -419,6 +435,17 @@ func (d *Daemon) Stop() error { d.periodicSyncMgr.Stop() } + // Clean up active overlays before stopping orchestrators + // This prevents orphaned FUSE mounts and overlay directories + if d.orchManager != nil { + count, err := d.orchManager.CleanupAllOverlays(30 * time.Second) + if err != nil { + logging.Warn("overlay cleanup encountered errors", "count", count, "error", err) + } else if count > 0 { + logging.Info("cleaned up active overlays", "count", count) + } + } + // Stop lifecycle-managed components (IPC, HTTP, pidfile) if err := d.lifecycle.Stop(); err != nil && firstErr == nil { firstErr = err @@ -451,11 +478,33 @@ func (d *Daemon) GetEventBus() *EventBus { } // GetState returns a snapshot of the current runtime state. +// It includes orchestrator state from the OrchestratorManager if available. func (d *Daemon) GetState() RuntimeStateSnapshot { - if d.state != nil { - return d.state.GetSnapshot() + if d.state == nil { + return RuntimeStateSnapshot{} } - return RuntimeStateSnapshot{} + + snapshot := d.state.GetSnapshot() + + // Populate orchestrator state from OrchestratorManager if available + if d.repoManager != nil && d.orchManager != nil { + if repo := d.GetActiveRepository(); repo != nil { + state := d.orchManager.GetOrchestratorState(repo.Path) + if state != "" { + snapshot.OrchestratorState = string(state) + } + // Count active agents from the snapshot we already have + activeCount := 0 + for _, agent := range snapshot.Agents { + if agent.Status == "running" { + activeCount++ + } + } + snapshot.ActiveAgentCount = activeCount + } + } + + return snapshot } // GetRuntimeState returns the RuntimeState instance for direct access. diff --git a/pkg/daemon/event_types.go b/pkg/daemon/event_types.go index 70dcc81d..17cf1543 100644 --- a/pkg/daemon/event_types.go +++ b/pkg/daemon/event_types.go @@ -16,6 +16,7 @@ const ( EventRunStarted = events.EventRunStarted EventRunCompleted = events.EventRunCompleted EventAgentStarted = events.EventAgentStarted + EventAgentRunning = events.EventAgentRunning EventAgentResumed = events.EventAgentResumed EventAgentOutput = events.EventAgentOutput EventAgentLiveFeed = events.EventAgentLiveFeed @@ -25,10 +26,12 @@ const ( EventAgentDone = events.EventAgentDone EventAgentFailed = events.EventAgentFailed EventTaskUpdated = events.EventTaskUpdated - EventOrchPaused = events.EventOrchPaused - EventOrchResumed = events.EventOrchResumed - EventStatsUpdated = events.EventStatsUpdated - EventRulesChanged = events.EventRulesChanged + EventOrchPaused = events.EventOrchPaused + EventOrchResumed = events.EventOrchResumed + EventStatsUpdated = events.EventStatsUpdated + EventRulesChanged = events.EventRulesChanged + EventLifecycleStateChanged = events.EventLifecycleStateChanged + EventConfigUpdated = events.EventConfigUpdated ) // NewEventBus creates a new EventBus instance. diff --git a/pkg/daemon/handlers.go b/pkg/daemon/handlers.go index 66eb6977..81d840ff 100644 --- a/pkg/daemon/handlers.go +++ b/pkg/daemon/handlers.go @@ -11,11 +11,14 @@ import ( "github.com/jzila/canopy/pkg/beads" "github.com/jzila/canopy/pkg/persistence" + "github.com/jzila/canopy/pkg/repository" ) // DaemonInterface abstracts daemon operations for handlers type DaemonInterface interface { GetActiveRepositoryID() string + GetActiveRepository() *repository.Repository + GetOrchestratorManager() *OrchestratorManager } // MergeQueueInterface abstracts merge queue operations for handlers @@ -53,6 +56,7 @@ type BeadsClientInterface interface { Start(ctx context.Context, taskID string) error Done(ctx context.Context, taskID string) error Fail(ctx context.Context, taskID string, reason string) error + FailPermanently(ctx context.Context, taskID string, reason string) error AddDep(ctx context.Context, child, parent string) error List(ctx context.Context) ([]beads.Task, error) Show(ctx context.Context, taskID string) (*beads.Task, error) @@ -86,7 +90,25 @@ func (h *Handler) SetMergeQueue(mq MergeQueueInterface) { // StateResponse wraps RuntimeStateSnapshot with additional daemon-level information type StateResponse struct { RuntimeStateSnapshot - ActiveRepoID string `json:"active_repo_id,omitempty"` + ActiveRepoID string `json:"active_repo_id,omitempty"` + MergeQueueInfo *MergeQueueSnapshot `json:"merge_queue,omitempty"` +} + +// MergeQueueSnapshot contains information about the merge queue state for display +type MergeQueueSnapshot struct { + Entries []MergeQueueEntry `json:"entries"` + QueueLength int `json:"queue_length"` + IsPaused bool `json:"is_paused"` + PauseState string `json:"pause_state"` +} + +// MergeQueueEntry represents an item in the merge queue +type MergeQueueEntry struct { + AgentID string `json:"agent_id"` + TaskID string `json:"task_id"` + TaskTitle string `json:"task_title,omitempty"` + Status string `json:"status"` // merging, waiting + Position int `json:"position"` } // HandleGetState returns the current RuntimeState as JSON @@ -106,6 +128,29 @@ func (h *Handler) HandleGetState(w http.ResponseWriter, r *http.Request) { // with a different or empty RepoID, and filtering would hide them from the UI) snapshot := h.state.GetSnapshot() + // Add orchestrator state if available + if h.daemon != nil { + orchMgr := h.daemon.GetOrchestratorManager() + if orchMgr != nil { + // Get the active repo and query its orchestrator state + activeRepo := h.daemon.GetActiveRepository() + if activeRepo != nil { + state := orchMgr.GetOrchestratorState(activeRepo.Path) + if state != "" { + snapshot.OrchestratorState = string(state) + } + } + } + // Count active agents (running status) + activeCount := 0 + for _, agent := range snapshot.Agents { + if agent.Status == "running" { + activeCount++ + } + } + snapshot.ActiveAgentCount = activeCount + } + // Wrap snapshot with additional daemon-level state response := StateResponse{ RuntimeStateSnapshot: snapshot, @@ -116,6 +161,9 @@ func (h *Handler) HandleGetState(w http.ResponseWriter, r *http.Request) { response.ActiveRepoID = activeRepoID } + // Build merge queue snapshot from agent states + response.MergeQueueInfo = h.buildMergeQueueSnapshot(snapshot) + w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { http.Error(w, fmt.Sprintf("Failed to encode state: %v", err), http.StatusInternalServerError) @@ -177,12 +225,6 @@ func (h *Handler) HandleKillAgent(w http.ResponseWriter, r *http.Request) { return } - // Check if scheduler is available - if h.scheduler == nil { - http.Error(w, "Scheduler not available", http.StatusNotImplemented) - return - } - // Check if agent exists agent := h.state.GetAgent(agentID) if agent == nil { @@ -190,9 +232,33 @@ func (h *Handler) HandleKillAgent(w http.ResponseWriter, r *http.Request) { return } - // Kill the agent via scheduler - if err := h.scheduler.Kill(agentID); err != nil { - http.Error(w, fmt.Sprintf("Failed to kill agent: %v", err), http.StatusInternalServerError) + // Kill via OrchestratorManager (preferred) or legacy scheduler + var killErr error + if h.daemon != nil { + orchManager := h.daemon.GetOrchestratorManager() + if orchManager != nil { + killErr = orchManager.KillAgent(agentID) + } else if h.scheduler != nil { + killErr = h.scheduler.Kill(agentID) + } else { + http.Error(w, "No kill mechanism available", http.StatusNotImplemented) + return + } + } else if h.scheduler != nil { + killErr = h.scheduler.Kill(agentID) + } else { + http.Error(w, "No kill mechanism available", http.StatusNotImplemented) + return + } + + if killErr != nil { + // Check if error is "agent not found" in scheduler + errMsg := killErr.Error() + if strings.Contains(errMsg, "not found") { + http.Error(w, fmt.Sprintf("Agent %s not found in scheduler", agentID), http.StatusNotFound) + } else { + http.Error(w, fmt.Sprintf("Failed to kill agent: %v", killErr), http.StatusInternalServerError) + } return } @@ -754,3 +820,71 @@ func (h *Handler) HandleGetMergeQueue(w http.ResponseWriter, r *http.Request) { return } } + +// buildMergeQueueSnapshot constructs a merge queue snapshot from agent states. +// This is used to include queue info in the /api/state response for CLI tools. +func (h *Handler) buildMergeQueueSnapshot(snapshot RuntimeStateSnapshot) *MergeQueueSnapshot { + queueSnapshot := &MergeQueueSnapshot{ + Entries: make([]MergeQueueEntry, 0), + PauseState: "running", + } + + // Get detailed pause state from merge queue if available + if h.mergeQueue != nil { + queueSnapshot.IsPaused = h.mergeQueue.IsPaused() + queueSnapshot.PauseState = h.mergeQueue.PauseStateString() + } + + // Collect agents that are in the merge queue (pending or actively merging) + for _, agent := range snapshot.Agents { + var entry *MergeQueueEntry + + switch agent.MergeStatus { + case MergeStatusMerging: + // Currently merging - position 1 + entry = &MergeQueueEntry{ + AgentID: agent.ID, + TaskID: agent.TaskID, + TaskTitle: agent.TaskTitle, + Status: "merging", + Position: 1, + } + case MergeStatusPending, MergeStatusAcquiring: + // Waiting in queue + position := agent.MergeQueuePos + if position == 0 { + // If position not set, estimate from queue length + position = len(queueSnapshot.Entries) + 2 // +2 because merging agent is position 1 + } + entry = &MergeQueueEntry{ + AgentID: agent.ID, + TaskID: agent.TaskID, + TaskTitle: agent.TaskTitle, + Status: "waiting", + Position: position, + } + } + + if entry != nil { + queueSnapshot.Entries = append(queueSnapshot.Entries, *entry) + } + } + + // Sort entries by position (merging first, then waiting by position) + // Simple bubble sort since queue is typically small + for i := 0; i < len(queueSnapshot.Entries)-1; i++ { + for j := 0; j < len(queueSnapshot.Entries)-i-1; j++ { + if queueSnapshot.Entries[j].Position > queueSnapshot.Entries[j+1].Position { + queueSnapshot.Entries[j], queueSnapshot.Entries[j+1] = queueSnapshot.Entries[j+1], queueSnapshot.Entries[j] + } + } + } + + // Renumber positions to be sequential (1, 2, 3, ...) + for i := range queueSnapshot.Entries { + queueSnapshot.Entries[i].Position = i + 1 + } + + queueSnapshot.QueueLength = len(queueSnapshot.Entries) + return queueSnapshot +} diff --git a/pkg/daemon/handlers_test.go b/pkg/daemon/handlers_test.go index 98012f90..95a67b59 100644 --- a/pkg/daemon/handlers_test.go +++ b/pkg/daemon/handlers_test.go @@ -64,6 +64,11 @@ func (m *mockBeadsClient) Fail(_ context.Context, taskID string, reason string) return nil } +func (m *mockBeadsClient) FailPermanently(_ context.Context, taskID string, reason string) error { + m.failCalls = append(m.failCalls, taskID) + return nil +} + func (m *mockBeadsClient) AddDep(_ context.Context, child, parent string) error { return nil } @@ -858,3 +863,133 @@ func TestHandleGetMergeQueue(t *testing.T) { } }) } + +func TestBuildMergeQueueSnapshot(t *testing.T) { + state := NewRuntimeState() + handler := NewHandler(state, nil, nil, nil) + + t.Run("EmptyState", func(t *testing.T) { + snapshot := state.GetSnapshot() + mqSnapshot := handler.buildMergeQueueSnapshot(snapshot) + + if mqSnapshot == nil { + t.Fatal("Expected non-nil snapshot") + } + if len(mqSnapshot.Entries) != 0 { + t.Errorf("Expected 0 entries, got %d", len(mqSnapshot.Entries)) + } + if mqSnapshot.QueueLength != 0 { + t.Errorf("Expected queue length 0, got %d", mqSnapshot.QueueLength) + } + }) + + t.Run("WithMergingAndPendingAgents", func(t *testing.T) { + // Clear state + state = NewRuntimeState() + handler = NewHandler(state, nil, nil, nil) + + // Add agent currently merging + mergingAgent := &AgentState{ + ID: "agent-merging", + TaskID: "task-merging", + TaskTitle: "Merging Task", + Status: AgentStatusRunning, + MergeStatus: MergeStatusMerging, + StartTime: time.Now(), + } + state.AddAgent(mergingAgent) + + // Add two agents pending in queue + pendingAgent1 := &AgentState{ + ID: "agent-pending1", + TaskID: "task-pending1", + TaskTitle: "Pending Task 1", + Status: AgentStatusRunning, + MergeStatus: MergeStatusPending, + MergeQueuePos: 2, + StartTime: time.Now(), + } + state.AddAgent(pendingAgent1) + + pendingAgent2 := &AgentState{ + ID: "agent-pending2", + TaskID: "task-pending2", + TaskTitle: "Pending Task 2", + Status: AgentStatusRunning, + MergeStatus: MergeStatusAcquiring, + MergeQueuePos: 3, + StartTime: time.Now(), + } + state.AddAgent(pendingAgent2) + + snapshot := state.GetSnapshot() + mqSnapshot := handler.buildMergeQueueSnapshot(snapshot) + + if mqSnapshot == nil { + t.Fatal("Expected non-nil snapshot") + } + if len(mqSnapshot.Entries) != 3 { + t.Errorf("Expected 3 entries, got %d", len(mqSnapshot.Entries)) + } + if mqSnapshot.QueueLength != 3 { + t.Errorf("Expected queue length 3, got %d", mqSnapshot.QueueLength) + } + + // Check that entries are sorted by position + for i, entry := range mqSnapshot.Entries { + expectedPos := i + 1 + if entry.Position != expectedPos { + t.Errorf("Entry %d: expected position %d, got %d", i, expectedPos, entry.Position) + } + } + + // First entry should be merging + if mqSnapshot.Entries[0].Status != "merging" { + t.Errorf("First entry should have status 'merging', got %q", mqSnapshot.Entries[0].Status) + } + }) +} + +func TestStateResponseIncludesMergeQueue(t *testing.T) { + state := NewRuntimeState() + handler := NewHandler(state, nil, nil, nil) + + // Add an agent in the merge queue + pendingAgent := &AgentState{ + ID: "agent-pending", + TaskID: "task-pending", + TaskTitle: "Pending Task", + Status: AgentStatusRunning, + MergeStatus: MergeStatusPending, + MergeQueuePos: 1, + StartTime: time.Now(), + } + state.AddAgent(pendingAgent) + + req := httptest.NewRequest(http.MethodGet, "/api/state", nil) + w := httptest.NewRecorder() + + handler.HandleGetState(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Expected status 200, got %d", w.Code) + } + + var result StateResponse + if err := json.NewDecoder(w.Body).Decode(&result); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + + // Check that merge queue is included + if result.MergeQueueInfo == nil { + t.Fatal("Expected MergeQueueInfo in response") + } + + if len(result.MergeQueueInfo.Entries) != 1 { + t.Errorf("Expected 1 merge queue entry, got %d", len(result.MergeQueueInfo.Entries)) + } + + if result.MergeQueueInfo.Entries[0].AgentID != "agent-pending" { + t.Errorf("Expected agent ID 'agent-pending', got %q", result.MergeQueueInfo.Entries[0].AgentID) + } +} diff --git a/pkg/daemon/orchestration_handler.go b/pkg/daemon/orchestration_handler.go index 3ae053a6..ca8c70fb 100644 --- a/pkg/daemon/orchestration_handler.go +++ b/pkg/daemon/orchestration_handler.go @@ -7,12 +7,16 @@ import ( "strings" "time" + "github.com/jzila/canopy/pkg/config" "github.com/jzila/canopy/pkg/logging" + "github.com/jzila/canopy/pkg/persistence" ) // OrchestrationHandler handles HTTP requests for orchestration control. type OrchestrationHandler struct { - manager *OrchestratorManager + manager *OrchestratorManager + daemon *Daemon + eventBus *EventBus } // NewOrchestrationHandler creates a new OrchestrationHandler. @@ -22,18 +26,44 @@ func NewOrchestrationHandler(manager *OrchestratorManager) *OrchestrationHandler } } +// NewOrchestrationHandlerWithDaemon creates a new OrchestrationHandler with daemon access. +// This allows the handler to access persistence store and event bus for config CRUD. +func NewOrchestrationHandlerWithDaemon(manager *OrchestratorManager, daemon *Daemon, eventBus *EventBus) *OrchestrationHandler { + return &OrchestrationHandler{ + manager: manager, + daemon: daemon, + eventBus: eventBus, + } +} + +// RuleOverrideRequest represents a custom rule in API requests. +type RuleOverrideRequest struct { + Name string `json:"name"` + Condition string `json:"condition"` + Action string `json:"action"` + Enabled *bool `json:"enabled,omitempty"` + Reason string `json:"reason,omitempty"` +} + // ExecuteRunRequest is the JSON request body for starting a run. type ExecuteRunRequest struct { - WorkDir string `json:"work_dir"` - OutputDir string `json:"output_dir,omitempty"` - Concurrency int `json:"concurrency,omitempty"` - Verbose bool `json:"verbose,omitempty"` - DryRun bool `json:"dry_run,omitempty"` - UseBwrap bool `json:"use_bwrap,omitempty"` - MaxRetries int `json:"max_retries,omitempty"` - MaxPriority int `json:"max_priority,omitempty"` - ResolverTimeoutMS int64 `json:"resolver_timeout_ms,omitempty"` - RepoID string `json:"repo_id,omitempty"` + WorkDir string `json:"work_dir"` + OutputDir string `json:"output_dir,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + Verbose bool `json:"verbose,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + UseBwrap bool `json:"use_bwrap,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + MaxPriority int `json:"max_priority,omitempty"` + ResolverTimeoutMS int64 `json:"resolver_timeout_ms,omitempty"` + RepoID string `json:"repo_id,omitempty"` + PollIntervalMS int64 `json:"poll_interval_ms,omitempty"` + Types []string `json:"types,omitempty"` + ExcludeTypes []string `json:"exclude_types,omitempty"` + Labels []string `json:"labels,omitempty"` + ExcludeLabels []string `json:"exclude_labels,omitempty"` + Assignee string `json:"assignee,omitempty"` + RuleOverrides []RuleOverrideRequest `json:"rule_overrides,omitempty"` } // ExecuteRunResponse is the JSON response for starting a run. @@ -54,6 +84,22 @@ type StopRunResponse struct { Error string `json:"error,omitempty"` } +// ActivateRequest is the JSON request body for activating an orchestrator. +// This is the preferred API for the always-active model. +type ActivateRequest = ExecuteRunRequest + +// ActivateResponse is the JSON response for activating an orchestrator. +type ActivateResponse = ExecuteRunResponse + +// DeactivateRequest is the JSON request body for deactivating an orchestrator. +type DeactivateRequest struct { + RunID string `json:"run_id,omitempty"` + RepoPath string `json:"repo_path,omitempty"` +} + +// DeactivateResponse is the JSON response for deactivating an orchestrator. +type DeactivateResponse = StopRunResponse + // RunStatusResponse is the JSON response for run status queries. type RunStatusResponse struct { Success bool `json:"success"` @@ -64,10 +110,9 @@ type RunStatusResponse struct { // UpdateRunConfigRequest is the JSON request body for updating run configuration. type UpdateRunConfigRequest struct { - Concurrency *int `json:"concurrency,omitempty"` - MaxPriority *int `json:"max_priority,omitempty"` - Watch *bool `json:"watch,omitempty"` - PollInterval *int `json:"poll_interval_ms,omitempty"` // milliseconds + Concurrency *int `json:"concurrency,omitempty"` + MaxPriority *int `json:"max_priority,omitempty"` + PollInterval *int `json:"poll_interval_ms,omitempty"` // milliseconds } // UpdateRunConfigResponse is the JSON response for updating run configuration. @@ -114,8 +159,20 @@ func (h *OrchestrationHandler) HandleExecuteRun(w http.ResponseWriter, r *http.R return } + // Convert rule overrides to config.CustomRule + var ruleOverrides []config.CustomRule + for _, r := range req.RuleOverrides { + ruleOverrides = append(ruleOverrides, config.CustomRule{ + Name: r.Name, + Condition: r.Condition, + Action: r.Action, + Enabled: r.Enabled, + Reason: r.Reason, + }) + } + // Convert request to RunConfig - config := RunConfig{ + runCfg := RunConfig{ WorkDir: req.WorkDir, OutputDir: req.OutputDir, Concurrency: req.Concurrency, @@ -125,11 +182,18 @@ func (h *OrchestrationHandler) HandleExecuteRun(w http.ResponseWriter, r *http.R MaxRetries: req.MaxRetries, MaxPriority: req.MaxPriority, ResolverTimeout: time.Duration(req.ResolverTimeoutMS) * time.Millisecond, + PollInterval: time.Duration(req.PollIntervalMS) * time.Millisecond, RepoID: req.RepoID, + Types: req.Types, + ExcludeTypes: req.ExcludeTypes, + Labels: req.Labels, + ExcludeLabels: req.ExcludeLabels, + Assignee: req.Assignee, + RuleOverrides: ruleOverrides, } // Start the run with a background context (not tied to request) - runID, err := h.manager.StartRun(context.Background(), config) + runID, err := h.manager.StartRun(context.Background(), runCfg) if err != nil { logging.Warn("failed to start orchestration run", "work_dir", req.WorkDir, @@ -204,6 +268,83 @@ func (h *OrchestrationHandler) HandleStopRun(w http.ResponseWriter, r *http.Requ }) } +// HandleActivate handles POST /api/orchestrator/activate requests. +// This is the preferred endpoint for the always-active orchestrator model. +func (h *OrchestrationHandler) HandleActivate(w http.ResponseWriter, r *http.Request) { + // Delegate to HandleExecuteRun - they use the same request/response format + h.HandleExecuteRun(w, r) +} + +// HandleDeactivate handles POST /api/orchestrator/deactivate requests. +// This is the preferred endpoint for the always-active orchestrator model. +// Accepts either run_id or repo_path. +func (h *OrchestrationHandler) HandleDeactivate(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + if h.manager == nil { + writeJSON(w, http.StatusServiceUnavailable, DeactivateResponse{ + Success: false, + Error: "orchestration not available", + }) + return + } + + var req DeactivateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, DeactivateResponse{ + Success: false, + Error: "invalid request body: " + err.Error(), + }) + return + } + + var err error + if req.RunID != "" { + // Stop by run ID + err = h.manager.StopRun(req.RunID) + } else if req.RepoPath != "" { + // Stop by repo path - find the active run and stop it + runState, getErr := h.manager.GetActiveRunForRepo(req.RepoPath) + if getErr != nil { + err = getErr + } else if runState == nil { + err = nil // No active run, treat as success + } else { + err = h.manager.StopRun(runState.ID) + } + } else { + writeJSON(w, http.StatusBadRequest, DeactivateResponse{ + Success: false, + Error: "either run_id or repo_path is required", + }) + return + } + + if err != nil { + logging.Warn("failed to deactivate orchestrator", + "run_id", req.RunID, + "repo_path", req.RepoPath, + "error", err) + + writeJSON(w, http.StatusInternalServerError, DeactivateResponse{ + Success: false, + Error: err.Error(), + }) + return + } + + logging.Info("deactivated orchestrator via HTTP", + "run_id", req.RunID, + "repo_path", req.RepoPath) + + writeJSON(w, http.StatusOK, DeactivateResponse{ + Success: true, + }) +} + // HandleGetRunStatus handles GET /api/orchestrator/run/:id requests. func (h *OrchestrationHandler) HandleGetRunStatus(w http.ResponseWriter, r *http.Request, runID string) { if r.Method != http.MethodGet { @@ -317,7 +458,7 @@ func (h *OrchestrationHandler) HandleUpdateRunConfig(w http.ResponseWriter, r *h return } - err := h.manager.UpdateRunConfig(runID, req.Concurrency, req.MaxPriority, req.Watch) + err := h.manager.UpdateRunConfig(runID, req.Concurrency, req.MaxPriority) if err != nil { logging.Warn("failed to update run config", "run_id", runID, @@ -362,6 +503,175 @@ func runStateToWire(rs *RunState) *RunStatusWire { return wire } +// RunConfigRequest is the JSON request/response body for run configuration CRUD. +// This uses the persistence.RunConfig fields but with omitempty for optional updates. +type RunConfigRequest struct { + Concurrency int `json:"concurrency"` + MaxPriority int `json:"max_priority"` + UseBwrap bool `json:"use_bwrap"` + MaxRetries int `json:"max_retries"` +} + +// RunConfigResponse is the JSON response for GET /api/config. +type RunConfigResponse struct { + Concurrency int `json:"concurrency"` + MaxPriority int `json:"max_priority"` + UseBwrap bool `json:"use_bwrap"` + MaxRetries int `json:"max_retries"` + Error string `json:"error,omitempty"` +} + +// HandleGetConfig handles GET /api/config requests. +// Returns the saved run configuration for the current repository, or defaults if none saved. +func (h *OrchestrationHandler) HandleGetConfig(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Get the repo ID from the current active repository + repoID := h.getActiveRepoID() + if repoID == "" { + writeJSON(w, http.StatusBadRequest, RunConfigResponse{ + Error: "no active repository", + }) + return + } + + // Get the persistence store + store := h.getPersistenceStore() + if store == nil { + // No persistence - return defaults + config := persistence.DefaultRunConfig(repoID) + writeJSON(w, http.StatusOK, RunConfigResponse{ + Concurrency: config.Concurrency, + MaxPriority: config.MaxPriority, + UseBwrap: config.UseBwrap, + MaxRetries: config.MaxRetries, + }) + return + } + + // Load config from persistence (returns defaults if not found) + config, err := store.GetRunConfig(repoID) + if err != nil { + logging.Error("failed to get run config", "repo_id", repoID, "error", err) + writeJSON(w, http.StatusInternalServerError, RunConfigResponse{ + Error: "failed to load configuration: " + err.Error(), + }) + return + } + + writeJSON(w, http.StatusOK, RunConfigResponse{ + Concurrency: config.Concurrency, + MaxPriority: config.MaxPriority, + UseBwrap: config.UseBwrap, + MaxRetries: config.MaxRetries, + }) +} + +// HandlePutConfig handles PUT /api/config requests. +// Saves run configuration for the current repository, broadcasts config_updated event. +func (h *OrchestrationHandler) HandlePutConfig(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Get the repo ID from the current active repository + repoID := h.getActiveRepoID() + if repoID == "" { + writeJSON(w, http.StatusBadRequest, RunConfigResponse{ + Error: "no active repository", + }) + return + } + + // Parse request body + var req RunConfigRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, RunConfigResponse{ + Error: "invalid request body: " + err.Error(), + }) + return + } + + // Get the persistence store + store := h.getPersistenceStore() + if store == nil { + writeJSON(w, http.StatusServiceUnavailable, RunConfigResponse{ + Error: "persistence not enabled", + }) + return + } + + // Create config from request + config := &persistence.RunConfig{ + RepoID: repoID, + Concurrency: req.Concurrency, + MaxPriority: req.MaxPriority, + UseBwrap: req.UseBwrap, + MaxRetries: req.MaxRetries, + } + + // Save to persistence + if err := store.SaveRunConfig(repoID, config); err != nil { + logging.Error("failed to save run config", "repo_id", repoID, "error", err) + writeJSON(w, http.StatusInternalServerError, RunConfigResponse{ + Error: "failed to save configuration: " + err.Error(), + }) + return + } + + logging.Info("saved run config via HTTP", "repo_id", repoID, + "concurrency", config.Concurrency, + "max_priority", config.MaxPriority, + "use_bwrap", config.UseBwrap, + "max_retries", config.MaxRetries) + + // Broadcast config_updated event via WebSocket + if h.eventBus != nil { + h.eventBus.Publish(Event{ + Type: EventConfigUpdated, + Timestamp: time.Now(), + Payload: RunConfigResponse{ + Concurrency: config.Concurrency, + MaxPriority: config.MaxPriority, + UseBwrap: config.UseBwrap, + MaxRetries: config.MaxRetries, + }, + }) + } + + // Return the saved configuration + writeJSON(w, http.StatusOK, RunConfigResponse{ + Concurrency: config.Concurrency, + MaxPriority: config.MaxPriority, + UseBwrap: config.UseBwrap, + MaxRetries: config.MaxRetries, + }) +} + +// getActiveRepoID returns the repository ID for the current active repository. +func (h *OrchestrationHandler) getActiveRepoID() string { + if h.daemon == nil { + return "" + } + return h.daemon.GetWorkDir() +} + +// getPersistenceStore returns the persistence store, or nil if persistence is disabled. +func (h *OrchestrationHandler) getPersistenceStore() *persistence.Store { + if h.daemon == nil { + return nil + } + pm := h.daemon.GetPersistenceStore() + if pm == nil { + return nil + } + return pm.GetStore() +} + // writeJSON writes a JSON response with the given status code. func writeJSON(w http.ResponseWriter, status int, data interface{}) { w.Header().Set("Content-Type", "application/json") @@ -376,13 +686,25 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) { func (h *OrchestrationHandler) RouteOrchestrator(w http.ResponseWriter, r *http.Request) { path := r.URL.Path - // POST /api/orchestrator/run - start a new run + // POST /api/orchestrator/activate - activate orchestrator (preferred) + if path == "/api/orchestrator/activate" && r.Method == http.MethodPost { + h.HandleActivate(w, r) + return + } + + // POST /api/orchestrator/deactivate - deactivate orchestrator (preferred) + if path == "/api/orchestrator/deactivate" && r.Method == http.MethodPost { + h.HandleDeactivate(w, r) + return + } + + // POST /api/orchestrator/run - start a new run (legacy, calls activate internally) if path == "/api/orchestrator/run" && r.Method == http.MethodPost { h.HandleExecuteRun(w, r) return } - // POST /api/orchestrator/run/stop - stop a run + // POST /api/orchestrator/run/stop - stop a run (legacy, calls deactivate internally) if path == "/api/orchestrator/run/stop" && r.Method == http.MethodPost { h.HandleStopRun(w, r) return diff --git a/pkg/daemon/orchestrator_manager.go b/pkg/daemon/orchestrator_manager.go index 89eb2fe8..aa9b9292 100644 --- a/pkg/daemon/orchestrator_manager.go +++ b/pkg/daemon/orchestrator_manager.go @@ -3,6 +3,7 @@ package daemon import ( "context" "fmt" + "strings" "sync" "time" @@ -10,12 +11,15 @@ import ( "github.com/jzila/canopy/pkg/agent" "github.com/jzila/canopy/pkg/beads" - "github.com/jzila/canopy/pkg/config" + cfgpkg "github.com/jzila/canopy/pkg/config" canopyerrors "github.com/jzila/canopy/pkg/errors" "github.com/jzila/canopy/pkg/events" + "github.com/jzila/canopy/pkg/lifecycle" "github.com/jzila/canopy/pkg/logging" + "github.com/jzila/canopy/pkg/mergequeue" "github.com/jzila/canopy/pkg/orchestrator" "github.com/jzila/canopy/pkg/rules" + "github.com/jzila/canopy/pkg/types" ) // RunConfig holds configuration for starting a new orchestration run. @@ -30,7 +34,17 @@ type RunConfig struct { MaxRetries int `json:"max_retries,omitempty"` MaxPriority int `json:"max_priority,omitempty"` ResolverTimeout time.Duration `json:"resolver_timeout,omitempty"` + PollInterval time.Duration `json:"poll_interval,omitempty"` RepoID string `json:"repo_id,omitempty"` + Types []string `json:"types,omitempty"` + ExcludeTypes []string `json:"exclude_types,omitempty"` + Labels []string `json:"labels,omitempty"` + ExcludeLabels []string `json:"exclude_labels,omitempty"` + Assignee string `json:"assignee,omitempty"` + // RuleOverrides contains custom rules to apply for this run only. + // These rules are applied with the highest precedence (above config.toml rules). + // When the run ends, these overrides are discarded unless persisted. + RuleOverrides []cfgpkg.CustomRule `json:"rule_overrides,omitempty"` } // RunStatus represents the current status of an orchestration run. @@ -48,23 +62,33 @@ const ( type OrchestratorState string const ( - // OrchestratorIdle means the orchestrator is running but not processing tasks. + // OrchestratorOff means no orchestrator instance exists (not activated, not watching). + OrchestratorOff OrchestratorState = "off" + // OrchestratorIdle means the orchestrator is running, polling for work, but none available. OrchestratorIdle OrchestratorState = "idle" // OrchestratorActive means the orchestrator is actively processing tasks. OrchestratorActive OrchestratorState = "active" - // OrchestratorPaused means the orchestrator is paused (tasks queued but not processed). + // OrchestratorPaused means the orchestrator is running but scheduler is paused. OrchestratorPaused OrchestratorState = "paused" ) -// RepoOrchestrator represents an always-running orchestrator for a registered repository. -// The orchestrator goroutine starts when the repo is registered and stops when unregistered. -// "Starting a run" is a state transition (idle → active), not object creation. -type RepoOrchestrator struct { +// OrchestratorLifecycle manages the lifecycle of an always-running orchestrator for a repository. +// It wraps orchestrator instances and manages state transitions between off, idle, active, and paused. +// +// State machine: +// - Off: Not activated, no orchestrator instance exists +// - Idle: Activated and watching for work, but no tasks currently available +// - Active: Processing tasks +// - Paused: Temporarily stopped by user or agent, will resume when unpaused +// +// The orchestrator starts when Activate() is called and stops when Deactivate() is called. +// State transitions between Idle and Active happen automatically based on work availability. +type OrchestratorLifecycle struct { RepoPath string `json:"repo_path"` RepoID string `json:"repo_id"` State OrchestratorState `json:"state"` - // Current run information (only valid when State == OrchestratorActive) + // Current run information (only valid when State != OrchestratorOff) RunID string `json:"run_id,omitempty"` RunConfig RunConfig `json:"run_config,omitempty"` StartTime time.Time `json:"start_time,omitempty"` @@ -116,7 +140,7 @@ type OrchestratorManager struct { // Always-running orchestrators indexed by repo path // These are created when a repo is registered and destroyed when unregistered - orchestrators sync.Map // map[string]*RepoOrchestrator (repo path -> orchestrator) + orchestrators sync.Map // map[string]*OrchestratorLifecycle (repo path -> orchestrator) // Event bus for publishing orchestration events directly eventBus *events.EventBus @@ -136,69 +160,95 @@ func NewOrchestratorManager(eventBus *events.EventBus, state *RuntimeState) *Orc // RegisterRepo creates an always-running orchestrator for a repository. // The orchestrator starts in IDLE state and waits for activation (StartRun). // If repoID is empty, it will be derived from the path. -// Returns the RepoOrchestrator or an error if registration fails. -func (m *OrchestratorManager) RegisterRepo(repoPath string, repoID string) (*RepoOrchestrator, error) { +// Returns the OrchestratorLifecycle or an error if registration fails. +func (m *OrchestratorManager) RegisterRepo(repoPath string, repoID string) (*OrchestratorLifecycle, error) { // Check if already registered if existing, ok := m.orchestrators.Load(repoPath); ok { - return existing.(*RepoOrchestrator), nil + return existing.(*OrchestratorLifecycle), nil } // Load config to create rules engine - cfg, err := config.LoadConfig(repoPath) + cfg, err := cfgpkg.LoadConfig(repoPath) if err != nil { // Log warning but continue with defaults - config loading should not fail registration logging.Warn("failed to load config.toml for repo, using defaults", "repo_path", repoPath, "error", err) - cfg = config.DefaultConfig() + cfg = cfgpkg.DefaultConfig() } // Create rules engine from config rulesEngine := rules.NewEngine(&cfg.Rules) - // Create the RepoOrchestrator in IDLE state - repoOrch := &RepoOrchestrator{ + // Create the OrchestratorLifecycle in OFF state (registered but not activated) + lifecycle := &OrchestratorLifecycle{ RepoPath: repoPath, RepoID: repoID, - State: OrchestratorIdle, + State: OrchestratorOff, rulesEngine: rulesEngine, } // Store atomically - another goroutine may have registered in parallel - if existing, loaded := m.orchestrators.LoadOrStore(repoPath, repoOrch); loaded { - return existing.(*RepoOrchestrator), nil + if existing, loaded := m.orchestrators.LoadOrStore(repoPath, lifecycle); loaded { + return existing.(*OrchestratorLifecycle), nil } - logging.Info("registered repo orchestrator", "repo_path", repoPath, "state", OrchestratorIdle) + logging.Info("registered repo orchestrator", "repo_path", repoPath, "state", OrchestratorOff) - return repoOrch, nil + return lifecycle, nil } // UnregisterRepo stops and removes the orchestrator for a repository. -// If the orchestrator is active, it will be stopped first. +// If the orchestrator is active, it will be stopped first with graceful cleanup. +// This waits for active overlays to be cleaned up before returning. // Returns an error if the repo is not registered. func (m *OrchestratorManager) UnregisterRepo(repoPath string) error { - repoOrchI, ok := m.orchestrators.Load(repoPath) + lifecycleI, ok := m.orchestrators.Load(repoPath) if !ok { return fmt.Errorf("repo not registered: %s", repoPath) } - repoOrch := repoOrchI.(*RepoOrchestrator) + lifecycle := lifecycleI.(*OrchestratorLifecycle) + + lifecycle.mu.Lock() - repoOrch.mu.Lock() - defer repoOrch.mu.Unlock() + // Get orchestrator reference before cancelling + orch := lifecycle.orch // Cancel any running orchestrator - if repoOrch.cancel != nil { - repoOrch.cancel() - repoOrch.cancel = nil + if lifecycle.cancel != nil { + lifecycle.cancel() + lifecycle.cancel = nil + } + + lifecycle.mu.Unlock() + + // Wait for orchestrator cleanup outside the lock to avoid blocking other operations + if orch != nil { + const shutdownTimeout = 30 * time.Second + if cleaned, err := orch.Shutdown(shutdownTimeout); err != nil { + logging.Warn("overlay cleanup during unregister had errors", + "repo_path", repoPath, + "overlays_cleaned", cleaned, + "error", err, + ) + } else if cleaned > 0 { + logging.Info("cleaned up overlays during unregister", + "repo_path", repoPath, + "overlays_cleaned", cleaned, + ) + } } // Remove from registry m.orchestrators.Delete(repoPath) // Also clean up any active run tracking - if repoOrch.RunID != "" { - m.runs.Delete(repoOrch.RunID) + lifecycle.mu.Lock() + runID := lifecycle.RunID + lifecycle.mu.Unlock() + + if runID != "" { + m.runs.Delete(runID) m.runsByRepo.Delete(repoPath) } @@ -207,11 +257,11 @@ func (m *OrchestratorManager) UnregisterRepo(repoPath string) error { return nil } -// GetRepoOrchestrator returns the orchestrator for a repository, if registered. +// GetOrchestratorLifecycle returns the orchestrator for a repository, if registered. // Returns nil if the repo is not registered. -func (m *OrchestratorManager) GetRepoOrchestrator(repoPath string) *RepoOrchestrator { - if repoOrchI, ok := m.orchestrators.Load(repoPath); ok { - return repoOrchI.(*RepoOrchestrator) +func (m *OrchestratorManager) GetOrchestratorLifecycle(repoPath string) *OrchestratorLifecycle { + if lifecycleI, ok := m.orchestrators.Load(repoPath); ok { + return lifecycleI.(*OrchestratorLifecycle) } return nil } @@ -219,10 +269,10 @@ func (m *OrchestratorManager) GetRepoOrchestrator(repoPath string) *RepoOrchestr // GetOrchestratorState returns the current state of the orchestrator for a repo. // Returns empty string if the repo is not registered. func (m *OrchestratorManager) GetOrchestratorState(repoPath string) OrchestratorState { - if repoOrch := m.GetRepoOrchestrator(repoPath); repoOrch != nil { - repoOrch.mu.RLock() - defer repoOrch.mu.RUnlock() - return repoOrch.State + if lifecycle := m.GetOrchestratorLifecycle(repoPath); lifecycle != nil { + lifecycle.mu.RLock() + defer lifecycle.mu.RUnlock() + return lifecycle.State } return "" } @@ -242,11 +292,11 @@ func (m *OrchestratorManager) ListRegisteredRepos() []string { // Returns nil if no orchestrator exists for the repository. func (m *OrchestratorManager) GetRulesEngineForRepo(repoPath string) *rules.Engine { // Check for always-running orchestrator first (preferred path) - if repoOrchI, ok := m.orchestrators.Load(repoPath); ok { - repoOrch := repoOrchI.(*RepoOrchestrator) - repoOrch.mu.RLock() - engine := repoOrch.rulesEngine - repoOrch.mu.RUnlock() + if lifecycleI, ok := m.orchestrators.Load(repoPath); ok { + lifecycle := lifecycleI.(*OrchestratorLifecycle) + lifecycle.mu.RLock() + engine := lifecycle.rulesEngine + lifecycle.mu.RUnlock() if engine != nil { return engine } @@ -298,54 +348,54 @@ func (m *OrchestratorManager) GetOrCreateRulesEngineForRepo(repoPath string) (*r } // No orchestrator yet - check if we have one registered but without an engine - if repoOrchI, ok := m.orchestrators.Load(repoPath); ok { - repoOrch := repoOrchI.(*RepoOrchestrator) - repoOrch.mu.Lock() - defer repoOrch.mu.Unlock() + if lifecycleI, ok := m.orchestrators.Load(repoPath); ok { + lifecycle := lifecycleI.(*OrchestratorLifecycle) + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() // Double-check after acquiring lock - if repoOrch.rulesEngine != nil { - return repoOrch.rulesEngine, nil + if lifecycle.rulesEngine != nil { + return lifecycle.rulesEngine, nil } // Create engine for the registered orchestrator - cfg, err := config.LoadConfig(repoPath) + cfg, err := cfgpkg.LoadConfig(repoPath) if err != nil { return nil, fmt.Errorf("failed to load config for %s: %w", repoPath, err) } - repoOrch.rulesEngine = rules.NewEngine(&cfg.Rules) - return repoOrch.rulesEngine, nil + lifecycle.rulesEngine = rules.NewEngine(&cfg.Rules) + return lifecycle.rulesEngine, nil } // No orchestrator registered - auto-register one for this repo (lazy registration) // This provides backwards compatibility: rules can be accessed without explicit registration - repoOrch, err := m.RegisterRepo(repoPath, "") + lifecycle, err := m.RegisterRepo(repoPath, "") if err != nil { return nil, fmt.Errorf("failed to register repo %s: %w", repoPath, err) } - return repoOrch.rulesEngine, nil + return lifecycle.rulesEngine, nil } // InvalidateRulesEngine reloads the rules engine for a repo from config. // This should be called when the repo's config changes. // For backwards compatibility, this is also aliased as InvalidateStandaloneEngine. func (m *OrchestratorManager) InvalidateRulesEngine(repoPath string) { - if repoOrchI, ok := m.orchestrators.Load(repoPath); ok { - repoOrch := repoOrchI.(*RepoOrchestrator) - repoOrch.mu.Lock() - defer repoOrch.mu.Unlock() + if lifecycleI, ok := m.orchestrators.Load(repoPath); ok { + lifecycle := lifecycleI.(*OrchestratorLifecycle) + lifecycle.mu.Lock() + defer lifecycle.mu.Unlock() // Reload config and create new engine - cfg, err := config.LoadConfig(repoPath) + cfg, err := cfgpkg.LoadConfig(repoPath) if err != nil { logging.Warn("failed to reload config for rules invalidation", "repo_path", repoPath, "error", err) return } - repoOrch.rulesEngine = rules.NewEngine(&cfg.Rules) + lifecycle.rulesEngine = rules.NewEngine(&cfg.Rules) } } @@ -407,22 +457,22 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s return "", fmt.Errorf("work_dir is required") } - // Get or create the RepoOrchestrator for this repo - repoOrch, err := m.RegisterRepo(config.WorkDir, config.RepoID) + // Get or create the OrchestratorLifecycle for this repo + lifecycle, err := m.RegisterRepo(config.WorkDir, config.RepoID) if err != nil { return "", fmt.Errorf("failed to register repo: %w", err) } // Lock the orchestrator for state transition - repoOrch.mu.Lock() + lifecycle.mu.Lock() // Check if already active - if repoOrch.State == OrchestratorActive { - repoOrch.mu.Unlock() + if lifecycle.State == OrchestratorActive { + lifecycle.mu.Unlock() return "", &canopyerrors.RunActiveError{ RepoPath: config.WorkDir, - RunID: repoOrch.RunID, - StartedAt: repoOrch.StartTime, + RunID: lifecycle.RunID, + StartedAt: lifecycle.StartTime, } } @@ -431,7 +481,7 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s // Also check the legacy runsByRepo for backwards compatibility if existingRunID, loaded := m.runsByRepo.LoadOrStore(config.WorkDir, runID); loaded { - repoOrch.mu.Unlock() + lifecycle.mu.Unlock() existingID := existingRunID.(string) var startedAt time.Time if runStateI, ok := m.runs.Load(existingID); ok { @@ -449,10 +499,10 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s defer func() { if cleanupOnError { m.runsByRepo.Delete(config.WorkDir) - repoOrch.mu.Lock() - repoOrch.State = OrchestratorIdle - repoOrch.RunID = "" - repoOrch.mu.Unlock() + lifecycle.mu.Lock() + lifecycle.State = OrchestratorOff + lifecycle.RunID = "" + lifecycle.mu.Unlock() } }() @@ -470,17 +520,17 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s config.MaxPriority = -1 // No filter by default } - // Update RepoOrchestrator state before releasing lock - repoOrch.State = OrchestratorActive - repoOrch.RunID = runID - repoOrch.RunConfig = config - repoOrch.StartTime = time.Now() - repoOrch.TasksTotal = 0 - repoOrch.TasksDone = 0 - repoOrch.TasksFailed = 0 + // Update OrchestratorLifecycle state before releasing lock + lifecycle.State = OrchestratorActive + lifecycle.RunID = runID + lifecycle.RunConfig = config + lifecycle.StartTime = time.Now() + lifecycle.TasksTotal = 0 + lifecycle.TasksDone = 0 + lifecycle.TasksFailed = 0 // Release lock before creating orchestrator (may block) - repoOrch.mu.Unlock() + lifecycle.mu.Unlock() // Create orchestrator config orchConfig := &orchestrator.Config{ @@ -493,6 +543,25 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s MaxRetries: config.MaxRetries, MaxPriority: config.MaxPriority, ResolverTimeout: config.ResolverTimeout, + PollInterval: config.PollInterval, + } + + // Apply CLI rules overrides if any were provided + if hasRulesOverrides(&config) { + orchConfig.Rules = &cfgpkg.RulesSettings{ + Types: config.Types, + ExcludeTypes: config.ExcludeTypes, + Labels: config.Labels, + ExcludeLabels: config.ExcludeLabels, + Assignee: config.Assignee, + } + orchConfig.RulesOverrides = &orchestrator.RulesOverrides{ + Types: len(config.Types) > 0, + ExcludeTypes: len(config.ExcludeTypes) > 0, + Labels: len(config.Labels) > 0, + ExcludeLabels: len(config.ExcludeLabels) > 0, + Assignee: config.Assignee != "", + } } // Create orchestrator @@ -502,18 +571,28 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s return "", fmt.Errorf("failed to create orchestrator: %w", err) } + // Apply run-configured rule overrides if provided + // These rules are applied with the highest precedence (above config.toml rules) + if len(config.RuleOverrides) > 0 { + if orchEngine := orch.GetRulesEngine(); orchEngine != nil { + if err := orchEngine.ApplyOverrides(config.RuleOverrides); err != nil { + return "", fmt.Errorf("failed to apply rule overrides: %w", err) + } + } + } + // Create cancellable context for this run runCtx, cancel := context.WithCancel(ctx) - // Update RepoOrchestrator with orchestrator instance - repoOrch.mu.Lock() - repoOrch.orch = orch - repoOrch.cancel = cancel + // Update OrchestratorLifecycle with orchestrator instance + lifecycle.mu.Lock() + lifecycle.orch = orch + lifecycle.cancel = cancel // Update rules engine to use the orchestrator's engine for consistency if orchEngine := orch.GetRulesEngine(); orchEngine != nil { - repoOrch.rulesEngine = orchEngine + lifecycle.rulesEngine = orchEngine } - repoOrch.mu.Unlock() + lifecycle.mu.Unlock() // Create run state for backwards compatibility runState := &RunState{ @@ -522,15 +601,26 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s RepoID: config.RepoID, Status: RunStatusPending, Config: config, - StartTime: repoOrch.StartTime, + StartTime: lifecycle.StartTime, orch: orch, cancel: cancel, } // Register callbacks that publish directly to EventBus - callbacks := m.createEventCallbacks(runID, config.RepoID) + callbacks := m.createEventCallbacks(runID, config.RepoID, orch) orch.SetCallbacks(callbacks) + // Register merge status callback to publish merge events to EventBus + // This replaces the IPC-based merge status when running in daemon mode + orch.SetMergeStatusCallback(m.createMergeStatusCallback()) + + // Register commit callback to publish commit events to EventBus + // This replaces the IPC-based commit events when running in daemon mode + orch.SetCommitCallback(m.createCommitCallback()) + + // Register state callbacks to track Idle/Active transitions + orch.SetStateCallbacks(m.createStateCallbacks(lifecycle)) + // Set run and repo IDs on the orchestrator for agent tracking orch.SetRunID(runID) if config.RepoID != "" { @@ -549,17 +639,20 @@ func (m *OrchestratorManager) StartRun(ctx context.Context, config RunConfig) (s runState.TasksTotal = len(tasks) runState.mu.Unlock() - repoOrch.mu.Lock() - repoOrch.TasksTotal = len(tasks) - repoOrch.mu.Unlock() + lifecycle.mu.Lock() + lifecycle.TasksTotal = len(tasks) + lifecycle.mu.Unlock() } } // Publish run started event m.publishRunStarted(runID, runState.TasksTotal, config) + // Publish initial state change (orchestrator now idle/active) + m.publishStateChange(OrchestratorIdle, 0) + // Start orchestrator in background goroutine - go m.runOrchestrator(runCtx, runState, repoOrch) + go m.runOrchestrator(runCtx, runState, lifecycle) logging.Info("started orchestration run", "run_id", runID, @@ -599,13 +692,13 @@ func (m *OrchestratorManager) StopRun(runID string) error { now := time.Now() runState.EndTime = &now - // Also update the RepoOrchestrator state - if repoOrch := m.GetRepoOrchestrator(runState.RepoPath); repoOrch != nil { - repoOrch.mu.Lock() - repoOrch.State = OrchestratorIdle - repoOrch.orch = nil - repoOrch.cancel = nil - repoOrch.mu.Unlock() + // Also update the OrchestratorLifecycle state (back to off since orchestrator stopped) + if lifecycle := m.GetOrchestratorLifecycle(runState.RepoPath); lifecycle != nil { + lifecycle.mu.Lock() + lifecycle.State = OrchestratorOff + lifecycle.orch = nil + lifecycle.cancel = nil + lifecycle.mu.Unlock() } logging.Info("cancelled orchestration run", "run_id", runID) @@ -613,6 +706,49 @@ func (m *OrchestratorManager) StopRun(runID string) error { return nil } +// Activate starts the orchestrator for a repository. +// This is the preferred method for the always-active model. It wraps StartRun +// and returns the run ID for backwards compatibility. +// +// State transition: Off → Idle (or Active if work is immediately available) +func (m *OrchestratorManager) Activate(ctx context.Context, config RunConfig) (string, error) { + return m.StartRun(ctx, config) +} + +// Deactivate stops the orchestrator for a repository by run ID. +// This is the preferred method for the always-active model. It wraps StopRun. +// +// State transition: * → Off +func (m *OrchestratorManager) Deactivate(runID string) error { + return m.StopRun(runID) +} + +// DeactivateByRepo stops the orchestrator for a repository by repo path. +// Returns an error if no active orchestrator exists for the repo. +// +// State transition: * → Off +func (m *OrchestratorManager) DeactivateByRepo(repoPath string) error { + lifecycle := m.GetOrchestratorLifecycle(repoPath) + if lifecycle == nil { + return fmt.Errorf("no orchestrator registered for repo: %s", repoPath) + } + + lifecycle.mu.RLock() + runID := lifecycle.RunID + state := lifecycle.State + lifecycle.mu.RUnlock() + + if state == OrchestratorOff { + return fmt.Errorf("orchestrator is not active for repo: %s", repoPath) + } + + if runID == "" { + return fmt.Errorf("no run ID found for active orchestrator: %s", repoPath) + } + + return m.StopRun(runID) +} + // GetRunStatus returns the current status of a run. func (m *OrchestratorManager) GetRunStatus(runID string) (*RunState, error) { runStateI, ok := m.runs.Load(runID) @@ -681,9 +817,9 @@ func (m *OrchestratorManager) GetActiveRunForRepo(repoPath string) (*RunState, e } // UpdateRunConfig updates the configuration of a running orchestration. -// Supports modifying concurrency, max priority filter, and pause state. +// Supports modifying concurrency and max priority filter. // Returns an error if the run is not found or not active. -func (m *OrchestratorManager) UpdateRunConfig(runID string, concurrency *int, maxPriority *int, paused *bool) error { +func (m *OrchestratorManager) UpdateRunConfig(runID string, concurrency *int, maxPriority *int) error { runStateI, ok := m.runs.Load(runID) if !ok { return fmt.Errorf("run not found: %s", runID) @@ -715,28 +851,12 @@ func (m *OrchestratorManager) UpdateRunConfig(runID string, concurrency *int, ma logging.Info("updated run max priority", "run_id", runID, "max_priority", *maxPriority) } - // Update pause state if specified - if paused != nil { - if runState.orch != nil { - sched := runState.orch.GetScheduler() - if sched != nil { - if *paused { - sched.Pause() - logging.Info("paused run", "run_id", runID) - } else { - sched.Resume() - logging.Info("resumed run", "run_id", runID) - } - } - } - } - return nil } // runOrchestrator executes the orchestrator and handles completion. -// It updates both the RunState (for backwards compatibility) and the RepoOrchestrator. -func (m *OrchestratorManager) runOrchestrator(ctx context.Context, runState *RunState, repoOrch *RepoOrchestrator) { +// It updates both the RunState (for backwards compatibility) and the OrchestratorLifecycle. +func (m *OrchestratorManager) runOrchestrator(ctx context.Context, runState *RunState, lifecycle *OrchestratorLifecycle) { // Update status to running runState.mu.Lock() runState.Status = RunStatusRunning @@ -760,23 +880,26 @@ func (m *OrchestratorManager) runOrchestrator(ctx context.Context, runState *Run } runState.mu.Unlock() - // Transition RepoOrchestrator back to IDLE state - if repoOrch != nil { - repoOrch.mu.Lock() - repoOrch.State = OrchestratorIdle - repoOrch.orch = nil - repoOrch.cancel = nil - // Keep rulesEngine for continued rules access in IDLE state + // Transition OrchestratorLifecycle back to OFF state (orchestrator stopped) + if lifecycle != nil { + lifecycle.mu.Lock() + lifecycle.State = OrchestratorOff + lifecycle.orch = nil + lifecycle.cancel = nil + // Keep rulesEngine for continued rules access in OFF state // Reload from config to ensure fresh state - if cfg, err := config.LoadConfig(repoOrch.RepoPath); err == nil { - repoOrch.rulesEngine = rules.NewEngine(&cfg.Rules) + if cfg, err := cfgpkg.LoadConfig(lifecycle.RepoPath); err == nil { + lifecycle.rulesEngine = rules.NewEngine(&cfg.Rules) } - repoOrch.mu.Unlock() + lifecycle.mu.Unlock() } // Remove from active runs by repo m.runsByRepo.Delete(runState.RepoPath) + // Publish state change to Off + m.publishStateChange(OrchestratorOff, 0) + // Publish run completed event m.publishRunCompleted(runState) @@ -789,29 +912,55 @@ func (m *OrchestratorManager) runOrchestrator(ctx context.Context, runState *Run // createEventCallbacks creates orchestrator event callbacks that publish to the EventBus. // These callbacks replace the IPC-based callbacks when the orchestrator runs in the daemon. -func (m *OrchestratorManager) createEventCallbacks(runID, repoID string) *orchestrator.EventCallbacks { +func (m *OrchestratorManager) createEventCallbacks(runID, repoID string, orch *orchestrator.Orchestrator) *orchestrator.EventCallbacks { return &orchestrator.EventCallbacks{ OnAgentStartFn: func(ctx context.Context, taskID string, task *beads.Task) { agentID := makeAgentID(runID, taskID) + // Get retry information from orchestrator + attempt := orch.GetTaskAttempt(taskID) + maxRetries := orch.GetMaxRetries() + // Record agent ID for parent-child tracking // The orchestrator instance manages this via SetAgentID - // Publish agent started event + // Publish agent started event - this synchronously creates the agent in RuntimeState + payload := map[string]interface{}{ + "agent_id": agentID, + "run_id": runID, + "task_id": taskID, + "task_title": task.Title, + "task_description": task.Description, + "parent_agent_id": "", // Top-level agents have no parent + "repo_id": repoID, + } + // Include retry information for UI display + if attempt > 0 { + payload["attempt"] = attempt + } + if maxRetries != 0 { + payload["max_retries"] = maxRetries + } m.eventBus.Publish(events.Event{ Type: events.EventAgentStarted, Timestamp: time.Now(), - Payload: map[string]interface{}{ - "agent_id": agentID, - "run_id": runID, - "task_id": taskID, - "task_title": task.Title, - "task_description": task.Description, - "parent_agent_id": "", // Top-level agents have no parent - "repo_id": repoID, - }, + Payload: payload, }) + // Transition lifecycle to running state (agent was created by event handler above) + // This is the authoritative state change - the event handler only initializes the lifecycle + if m.state != nil { + if agent := m.state.GetAgent(agentID); agent != nil && agent.Lifecycle != nil { + if err := agent.Lifecycle.Transition(lifecycle.EventAgentSpawned, lifecycle.TransitionContext{}); err != nil { + logging.Warn("lifecycle transition failed on agent start", + "agent_id", agentID, + "event", lifecycle.EventAgentSpawned, + "error", err, + ) + } + } + } + // Update task status m.eventBus.Publish(events.Event{ Type: events.EventTaskUpdated, @@ -865,6 +1014,20 @@ func (m *OrchestratorManager) createEventCallbacks(runID, repoID string) *orches runState.mu.Unlock() } + // Transition lifecycle to queued_for_merge state (work complete, awaiting merge) + // This is the authoritative state change - the merge processor will handle subsequent transitions + if m.state != nil { + if agent := m.state.GetAgent(agentID); agent != nil && agent.Lifecycle != nil { + if err := agent.Lifecycle.Transition(lifecycle.EventWorkComplete, lifecycle.TransitionContext{}); err != nil { + logging.Warn("lifecycle transition failed on agent done", + "agent_id", agentID, + "event", lifecycle.EventWorkComplete, + "error", err, + ) + } + } + } + // Build completion payload payload := m.buildCompletionPayload(agentID, result) @@ -898,6 +1061,24 @@ func (m *OrchestratorManager) createEventCallbacks(runID, repoID string) *orches runState.mu.Unlock() } + // Transition lifecycle to failed state (work failed, no retries) + // This is the authoritative state change + if m.state != nil { + if agent := m.state.GetAgent(agentID); agent != nil && agent.Lifecycle != nil { + // AttemptsRemaining = 0 means no retries, transition directly to failed + if err := agent.Lifecycle.Transition(lifecycle.EventWorkFailed, lifecycle.TransitionContext{ + AttemptsRemaining: 0, + Error: result.Error, + }); err != nil { + logging.Warn("lifecycle transition failed on agent fail", + "agent_id", agentID, + "event", lifecycle.EventWorkFailed, + "error", err, + ) + } + } + } + // Build failure payload payload := m.buildCompletionPayload(agentID, result) payload["error"] = result.Error @@ -923,6 +1104,165 @@ func (m *OrchestratorManager) createEventCallbacks(runID, repoID string) *orches } } +// createStateCallbacks creates state transition callbacks for a OrchestratorLifecycle. +// These callbacks update the OrchestratorLifecycle's State field when the underlying +// Orchestrator transitions between Idle and Active states. +func (m *OrchestratorManager) createStateCallbacks(lifecycle *OrchestratorLifecycle) *orchestrator.StateCallbacks { + return &orchestrator.StateCallbacks{ + OnIdle: func() { + lifecycle.mu.Lock() + lifecycle.State = OrchestratorIdle + lifecycle.mu.Unlock() + logging.Debug("orchestrator entered idle state", "repo_path", lifecycle.RepoPath) + + // Publish state change event + m.publishStateChange(OrchestratorIdle, 0) + }, + OnActive: func() { + lifecycle.mu.Lock() + lifecycle.State = OrchestratorActive + lifecycle.mu.Unlock() + logging.Debug("orchestrator entered active state", "repo_path", lifecycle.RepoPath) + + // Count active agents + activeCount := m.countActiveAgents() + m.publishStateChange(OrchestratorActive, activeCount) + }, + } +} + +// createMergeStatusCallback creates a callback that publishes merge status events to the EventBus. +// This replaces the IPC-based merge status updates when the orchestrator runs in daemon mode. +func (m *OrchestratorManager) createMergeStatusCallback() mergequeue.MergeStatusCallback { + return func(event mergequeue.MergeStatusEvent) { + if m.eventBus == nil { + return + } + + payload := map[string]interface{}{ + "agent_id": event.AgentID, + "merge_status": string(event.Status), + } + + // Include optional fields only when set + if event.QueuePos > 0 { + payload["queue_pos"] = event.QueuePos + } + if event.Error != "" { + payload["error"] = event.Error + } + if event.CommitsApplied > 0 { + payload["commits_applied"] = event.CommitsApplied + } + if event.HadConflict { + payload["had_conflict"] = event.HadConflict + } + if event.ResolverSpawned { + payload["resolver_spawned"] = event.ResolverSpawned + } + + m.eventBus.Publish(events.Event{ + Type: events.EventAgentMergeStatus, + Timestamp: time.Now(), + Payload: payload, + }) + + // Also transition lifecycle state if this is a final merge status + if m.state != nil { + if agent := m.state.GetAgent(event.AgentID); agent != nil && agent.Lifecycle != nil { + switch event.Status { + case types.MergeStatusMerging: + // Transition to merging state + if err := agent.Lifecycle.Transition(lifecycle.EventMergeStarted, lifecycle.TransitionContext{}); err != nil { + logging.Debug("lifecycle transition failed on merge started", + "agent_id", event.AgentID, + "event", lifecycle.EventMergeStarted, + "error", err, + ) + } + case types.MergeStatusMerged, types.MergeStatusMergedNeedsRepair: + // Transition to merged state + if err := agent.Lifecycle.Transition(lifecycle.EventMergeSuccess, lifecycle.TransitionContext{}); err != nil { + logging.Debug("lifecycle transition failed on merge success", + "agent_id", event.AgentID, + "event", lifecycle.EventMergeSuccess, + "error", err, + ) + } + case types.MergeStatusFailed: + // Transition to merge failed state + if err := agent.Lifecycle.Transition(lifecycle.EventMergeFailed, lifecycle.TransitionContext{ + Error: event.Error, + }); err != nil { + logging.Debug("lifecycle transition failed on merge failed", + "agent_id", event.AgentID, + "event", lifecycle.EventMergeFailed, + "error", err, + ) + } + } + } + } + } +} + +// createCommitCallback creates a callback that publishes commit events to the EventBus. +// This replaces the IPC-based commit events when the orchestrator runs in daemon mode. +func (m *OrchestratorManager) createCommitCallback() mergequeue.CommitCallback { + return func(event mergequeue.CommitEvent) { + if m.eventBus == nil { + return + } + + payload := map[string]interface{}{ + "agent_id": event.AgentID, + "hash": event.Hash, + "short_hash": event.ShortHash, + "message": event.Message, + "author": event.Author, + "author_email": event.AuthorEmail, + "timestamp": event.Timestamp, + "files_changed": event.FilesChanged, + } + + m.eventBus.Publish(events.Event{ + Type: events.EventAgentCommit, + Timestamp: time.Now(), + Payload: payload, + }) + } +} + +// publishStateChange publishes an orchestrator state change event to the event bus. +func (m *OrchestratorManager) publishStateChange(state OrchestratorState, activeAgentCount int) { + if m.eventBus == nil { + return + } + + m.eventBus.Publish(events.Event{ + Type: events.EventOrchStateChanged, + Timestamp: time.Now(), + Payload: map[string]interface{}{ + "state": string(state), + "active_agent_count": activeAgentCount, + }, + }) +} + +// countActiveAgents counts the number of currently running agents. +func (m *OrchestratorManager) countActiveAgents() int { + count := 0 + if m.state != nil { + snapshot := m.state.GetSnapshot() + for _, agent := range snapshot.Agents { + if agent.Status == "running" { + count++ + } + } + } + return count +} + // buildCompletionPayload creates the payload for agent completion events. func (m *OrchestratorManager) buildCompletionPayload(agentID string, result *agent.Result) map[string]interface{} { payload := map[string]interface{}{ @@ -1015,3 +1355,101 @@ func makeAgentID(runID, taskID string) string { } return fmt.Sprintf("agent-%s-%s", prefix, taskID) } + +// KillAgent terminates a specific agent by its ID. +// It finds the run that owns the agent and delegates to the scheduler. +// Agent ID format: agent-{runID[:8]}-{taskID} +func (m *OrchestratorManager) KillAgent(agentID string) error { + // Parse agent ID to extract run ID prefix + // Format: agent-{runID[:8]}-{taskID} + parts := strings.SplitN(agentID, "-", 3) + if len(parts) < 3 || parts[0] != "agent" { + return fmt.Errorf("invalid agent ID format: %s", agentID) + } + runPrefix := parts[1] + + // Find the run with matching ID prefix + var foundRun *RunState + m.runs.Range(func(key, value interface{}) bool { + runID := key.(string) + // Check if run ID starts with the prefix + if len(runID) >= len(runPrefix) && runID[:len(runPrefix)] == runPrefix { + foundRun = value.(*RunState) + return false // stop iteration + } + return true + }) + + if foundRun == nil { + return fmt.Errorf("run not found for agent %s", agentID) + } + + // Get the scheduler from the orchestrator + foundRun.mu.RLock() + orch := foundRun.orch + foundRun.mu.RUnlock() + + if orch == nil { + return fmt.Errorf("orchestrator not available for agent %s", agentID) + } + + sched := orch.GetScheduler() + if sched == nil { + return fmt.Errorf("scheduler not available for agent %s", agentID) + } + + // Extract task ID from parts (taskID is parts[2]) + taskID := parts[2] + return sched.Kill(taskID) +} + +// hasRulesOverrides checks if any CLI rules overrides were provided in the RunConfig. +func hasRulesOverrides(cfg *RunConfig) bool { + return len(cfg.Types) > 0 || + len(cfg.ExcludeTypes) > 0 || + len(cfg.Labels) > 0 || + len(cfg.ExcludeLabels) > 0 || + cfg.Assignee != "" +} + +// CleanupAllOverlays cleans up active overlays from all running orchestrators. +// This should be called during daemon shutdown to prevent orphaned FUSE mounts. +// The timeout specifies how long to wait for cleanup to complete. +// Returns the number of overlays cleaned and any errors encountered. +func (m *OrchestratorManager) CleanupAllOverlays(timeout time.Duration) (int, error) { + var totalCleaned int + var cleanupErrors []error + + // Iterate through all active runs to find their schedulers + m.runs.Range(func(key, value interface{}) bool { + runState := value.(*RunState) + runState.mu.RLock() + orch := runState.orch + runState.mu.RUnlock() + + if orch == nil { + return true // continue iteration + } + + sched := orch.GetScheduler() + if sched == nil { + return true // continue iteration + } + + // Cleanup overlays for this scheduler + count, err := sched.CleanupAll(timeout) + totalCleaned += count + if err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("run %s: %w", key.(string), err)) + } + + return true // continue iteration + }) + + // If there were errors, return them + if len(cleanupErrors) > 0 { + return totalCleaned, fmt.Errorf("cleanup completed with %d error(s): %v", len(cleanupErrors), cleanupErrors) + } + + return totalCleaned, nil +} \ No newline at end of file diff --git a/pkg/daemon/persistence_handler.go b/pkg/daemon/persistence_handler.go index a1805a25..c5ea687d 100644 --- a/pkg/daemon/persistence_handler.go +++ b/pkg/daemon/persistence_handler.go @@ -100,7 +100,7 @@ func (h *PersistenceHandler) handleRunStarted(event Event) { // handleAgentMergeStatus updates the merge result fields when merge completes. // We only persist final merge statuses (merged or failed) to avoid noise from // intermediate statuses (pending, merging, resolving). -// Also persists validation results if present. +// Also persists validation results and lifecycle state if present. func (h *PersistenceHandler) handleAgentMergeStatus(event Event) { payload, ok := event.Payload.(map[string]interface{}) if !ok { @@ -116,6 +116,22 @@ func (h *PersistenceHandler) handleAgentMergeStatus(event Event) { mergeStatus, _ := payload["merge_status"].(string) mergeErr, _ := payload["error"].(string) + lifecycleState, _ := payload["lifecycle_state"].(string) + + // Persist lifecycle state if present (always persist lifecycle updates) + if lifecycleState != "" { + if err := h.store.UpdateAgentLifecycleState(agentID, lifecycleState); err != nil { + logging.Error("failed to update lifecycle state", + "agent_id", agentID, + "error", err, + "component", "persistence") + } else { + logging.Debug("updated lifecycle state", + "agent_id", agentID, + "lifecycle_state", lifecycleState, + "component", "persistence") + } + } // Persist validation results if present validationStatus, hasValidation := payload["validation_status"].(string) @@ -219,6 +235,11 @@ func (h *PersistenceHandler) handleAgentStarted(event Event) { taskDescription, _ := payload["task_description"].(string) repoID, _ := payload["repo_id"].(string) parentAgentID, _ := payload["parent_agent_id"].(string) + lifecycleState, _ := payload["lifecycle_state"].(string) + + // Extract retry information + attempt, _ := getIntFromPayload(payload, "attempt") + maxRetries, _ := getIntFromPayload(payload, "max_retries") // Get current run ID h.mu.RLock() @@ -232,9 +253,12 @@ func (h *PersistenceHandler) handleAgentStarted(event Event) { TaskTitle: taskTitle, TaskDescription: taskDescription, Status: persistence.AgentStatusRunning, + LifecycleState: lifecycleState, StartedAt: event.Timestamp, RepoID: repoID, ParentAgentID: parentAgentID, + Attempt: attempt, + MaxRetries: maxRetries, } if err := h.store.CreateAgent(agent); err != nil { @@ -247,6 +271,7 @@ func (h *PersistenceHandler) handleAgentStarted(event Event) { "agent_id", agentID, "task_id", taskID, "run_id", runID, + "lifecycle_state", lifecycleState, "component", "persistence") } } @@ -292,12 +317,14 @@ func (h *PersistenceHandler) handleAgentCompleted(event Event) { stdout, _ := payload["stdout"].(string) stderr, _ := payload["stderr"].(string) sessionID, _ := payload["session_id"].(string) + lifecycleState, _ := payload["lifecycle_state"].(string) finishedAt := event.Timestamp agent := &persistence.Agent{ ID: agentID, RunID: runID, Status: status, + LifecycleState: lifecycleState, FinishedAt: &finishedAt, DurationSeconds: duration, ExitCode: &exitCode, @@ -328,6 +355,7 @@ func (h *PersistenceHandler) handleAgentCompleted(event Event) { logging.Debug("updated agent completion", "agent_id", agentID, "status", status, + "lifecycle_state", lifecycleState, "component", "persistence") } } diff --git a/pkg/daemon/persistence_manager.go b/pkg/daemon/persistence_manager.go index ef1e6e8c..59671df9 100644 --- a/pkg/daemon/persistence_manager.go +++ b/pkg/daemon/persistence_manager.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "github.com/jzila/canopy/pkg/lifecycle" "github.com/jzila/canopy/pkg/logging" "github.com/jzila/canopy/pkg/persistence" ) @@ -271,9 +272,82 @@ func ConvertPersistenceAgentToState(pAgent *persistence.Agent) *AgentState { // Restore session ID for claude --resume support agent.SessionID = pAgent.SessionID + // Restore task retry tracking fields + agent.Attempt = pAgent.Attempt + agent.MaxRetries = pAgent.MaxRetries + + // Restore lifecycle state - use persisted state if available, otherwise derive from legacy fields + lifecycleState := pAgent.LifecycleState + if lifecycleState == "" { + // Migration: derive lifecycle state from legacy fields for agents without lifecycle_state + lifecycleState = string(deriveLifecycleStateFromLegacy(pAgent)) + } + agent.LifecycleState = lifecycleState + + // Create the lifecycle state machine with the restored state + // For restored agents, we initialize directly into the restored state without transitions + if lifecycleState != "" { + agent.Lifecycle = lifecycle.New( + lifecycle.WithInitialState(lifecycle.AgentLifecycleState(lifecycleState)), + ) + } + return agent } +// deriveLifecycleStateFromLegacy derives a lifecycle state from the legacy status fields. +// This is used for migrating existing agents that don't have lifecycle_state persisted. +// Note: Persistence only stores final merge statuses (merged, failed, skipped, merged_needs_repair), +// not intermediate states (pending, merging, resolving) which are runtime-only. +func deriveLifecycleStateFromLegacy(pAgent *persistence.Agent) lifecycle.AgentLifecycleState { + // Check validation status first (most specific states) + switch pAgent.ValidationStatus { + case "running": + return lifecycle.StateValidating + case "repairing": + return lifecycle.StateRepairing + case "failed": + // Validation failed - could be failed or needs_attention + if pAgent.MergeStatus == persistence.MergeStatusMergedNeedsRepair { + return lifecycle.StateNeedsAttention + } + return lifecycle.StateFailed + case "passed": + return lifecycle.StateCompleted + } + + // Check merge status - persistence only stores final statuses + switch pAgent.MergeStatus { + case persistence.MergeStatusMerged, persistence.MergeStatusResolved: + return lifecycle.StateCompleted + case persistence.MergeStatusFailed: + return lifecycle.StateMergeFailed + case persistence.MergeStatusMergedNeedsRepair: + return lifecycle.StateNeedsAttention + case persistence.MergeStatusSkipped: + return lifecycle.StateCompleted + } + + // Fall back to agent status + switch pAgent.Status { + case persistence.AgentStatusStarting: + return lifecycle.StateStarting + case persistence.AgentStatusRunning: + return lifecycle.StateRunning + case persistence.AgentStatusCompleted: + return lifecycle.StateCompleted + case persistence.AgentStatusFailed: + return lifecycle.StateFailed + case persistence.AgentStatusCancelled: + return lifecycle.StateCancelled + case persistence.AgentStatusTimedOut: + return lifecycle.StateTimedOut + } + + // Default to running for unknown states + return lifecycle.StateRunning +} + // ConvertPersistenceStatus converts persistence.AgentStatus to daemon.AgentStatus func ConvertPersistenceStatus(status persistence.AgentStatus) AgentStatus { switch status { @@ -429,6 +503,17 @@ func ApplyRestoredState(state *RuntimeState, restored *RestoredState) { // Restore agents for i := range restored.Agents { agentState := ConvertPersistenceAgentToState(&restored.Agents[i]) + + // Wire up lifecycle callbacks for non-terminal agents so any future + // transitions publish events to the EventBus for real-time UI updates. + if agentState.Lifecycle != nil && !agentState.Lifecycle.IsTerminal() { + // Create a new lifecycle with the callback and same initial state + agentState.Lifecycle = lifecycle.New( + lifecycle.WithInitialState(agentState.Lifecycle.State()), + lifecycle.WithTransitionCallback(state.makeLifecycleCallback(agentState.ID)), + ) + } + state.AddAgent(agentState) logging.Debug("restored agent", "agent_id", restored.Agents[i].ID, diff --git a/pkg/daemon/repo_handlers_test.go b/pkg/daemon/repo_handlers_test.go index 56a44760..b6029f4d 100644 --- a/pkg/daemon/repo_handlers_test.go +++ b/pkg/daemon/repo_handlers_test.go @@ -22,6 +22,15 @@ func (m *mockDaemon) GetActiveRepositoryID() string { return m.activeRepoID } +func (m *mockDaemon) GetActiveRepository() *repository.Repository { + // Return nil for tests - orchestrator state won't be populated + return nil +} + +func (m *mockDaemon) GetOrchestratorManager() *OrchestratorManager { + return nil +} + func (m *mockDaemon) SetActiveRepository(repoID string) error { m.activeRepoID = repoID return nil diff --git a/pkg/daemon/server.go b/pkg/daemon/server.go index b8ed2754..3b0dcf63 100644 --- a/pkg/daemon/server.go +++ b/pkg/daemon/server.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "net/http" + "net/url" "strings" "time" @@ -90,7 +91,7 @@ func NewServerWithDaemon(port int, state *RuntimeState, eventBus *EventBus, sche // Create orchestration handler for run control (requires daemon reference) var orchHandler *OrchestrationHandler if daemon != nil { - orchHandler = NewOrchestrationHandler(daemon.GetOrchestratorManager()) + orchHandler = NewOrchestrationHandlerWithDaemon(daemon.GetOrchestratorManager(), daemon, eventBus) } // Create rules handler for runtime rule management (requires daemon reference) @@ -220,6 +221,9 @@ func (s *Server) setupRoutes() *http.ServeMux { // REST API routes - beads operations mux.HandleFunc("/api/beads/sync", s.handleBeadsSyncRoute) // Handles POST /api/beads/sync + // REST API routes - run configuration CRUD + mux.HandleFunc("/api/config", s.handleConfigRoute) // Handles GET and PUT /api/config + // REST API routes - orchestration control (daemon-owned runs) // New: /api/runs/:run_id/... for run-scoped operations mux.HandleFunc("/api/orchestrator/", s.handleOrchestratorRoutes) // Legacy: handles all /api/orchestrator/* routes @@ -425,43 +429,88 @@ func (s *Server) handleOrchestratorRoutes(w http.ResponseWriter, r *http.Request s.orchHandler.RouteOrchestrator(w, r) } +// handleConfigRoute routes run configuration CRUD requests +func (s *Server) handleConfigRoute(w http.ResponseWriter, r *http.Request) { + if s.orchHandler == nil { + http.Error(w, "Configuration not available", http.StatusServiceUnavailable) + return + } + + switch r.Method { + case http.MethodGet: + s.orchHandler.HandleGetConfig(w, r) + case http.MethodPut: + s.orchHandler.HandlePutConfig(w, r) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + // handleReposRoutes routes repo-scoped requests for rules and config // Handles /api/repos/:repo_id/rules/* and /api/repos/:repo_id/config/* func (s *Server) handleReposRoutes(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path + // Use EscapedPath to preserve URL encoding (repo paths may contain slashes) + // EscapedPath returns the encoded form, or Path if RawPath is empty + path := r.URL.EscapedPath() // Parse: /api/repos/:repo_id/... + // Note: repo_id may be URL-encoded (e.g., %2Fhome%2Fuser%2Frepo for /home/user/repo) parts := strings.Split(strings.Trim(path, "/"), "/") if len(parts) < 4 || parts[0] != "api" || parts[1] != "repos" { http.Error(w, "Invalid path", http.StatusBadRequest) return } - repoID := parts[2] - if repoID == "" { + encodedRepoID := parts[2] + if encodedRepoID == "" { http.Error(w, "Repository ID required", http.StatusBadRequest) return } + // Decode the repo ID (which may be a URL-encoded path like %2Fhome%2Fuser%2Frepo) + repoID, err := url.PathUnescape(encodedRepoID) + if err != nil { + http.Error(w, fmt.Sprintf("Invalid repo ID encoding: %v", err), http.StatusBadRequest) + return + } + // Inject repo_id as query parameter for downstream handlers q := r.URL.Query() q.Set("repo_id", repoID) r.URL.RawQuery = q.Encode() - resource := parts[3] + // Determine the resource and remaining path parts + // parts[3] is the resource (rules, config), but it may also be encoded + resource, err := url.PathUnescape(parts[3]) + if err != nil { + http.Error(w, "Invalid path", http.StatusBadRequest) + return + } + + // Decode remaining suffix parts + suffix := make([]string, 0, len(parts)-4) + for _, p := range parts[4:] { + decoded, err := url.PathUnescape(p) + if err != nil { + http.Error(w, "Invalid path encoding", http.StatusBadRequest) + return + } + suffix = append(suffix, decoded) + } + switch resource { case "rules": if s.rulesHandler == nil { http.Error(w, "Rules management not available", http.StatusServiceUnavailable) return } - s.rulesHandler.RouteRepoRules(w, r, parts[4:]) + s.rulesHandler.RouteRepoRules(w, r, suffix) case "config": if s.configHandler == nil { http.Error(w, "Configuration queries not available", http.StatusServiceUnavailable) return } - s.configHandler.RouteRepoConfig(w, r, parts[4:]) + s.configHandler.RouteRepoConfig(w, r, suffix) default: http.Error(w, "Not found", http.StatusNotFound) } @@ -610,8 +659,15 @@ func (s *Server) handleWebSocket(w http.ResponseWriter, r *http.Request) { // sendInitialState sends the current runtime state to a newly connected client func (s *Server) sendInitialState(client *Client) { - // Get a snapshot of the current state - snapshot := s.state.GetSnapshot() + // Get a snapshot of the current state. + // Use daemon.GetState() if available, as it includes orchestrator state. + // Otherwise fall back to direct state snapshot. + var snapshot RuntimeStateSnapshot + if s.daemon != nil { + snapshot = s.daemon.GetState() + } else { + snapshot = s.state.GetSnapshot() + } // Create state sync event event := Event{ diff --git a/pkg/daemon/server_test.go b/pkg/daemon/server_test.go index 70ad6982..fef367b3 100644 --- a/pkg/daemon/server_test.go +++ b/pkg/daemon/server_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/url" + "strings" "testing" "time" @@ -279,3 +281,122 @@ func TestServerStopBeforeStart(t *testing.T) { t.Errorf("Expected no error when stopping unstarted server, got: %v", err) } } + +func TestHandleReposRoutes_URLEncodedPaths(t *testing.T) { + state := NewRuntimeState() + eventBus := NewEventBus() + scheduler := &mockScheduler{} + beadsClient := &mockBeadsClient{} + + server := NewServer(testPort, state, eventBus, scheduler, beadsClient) + + // Track received repo_id to verify URL decoding + var receivedRepoID string + mockRulesHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedRepoID = r.URL.Query().Get("repo_id") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"rules":[]}`)) + }) + + // Override the handler method for testing by wrapping it + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Set a mock rules handler that records the repo_id + server.rulesHandler = &RulesHandler{} // non-nil to pass the check + + // Call the actual route parsing, then substitute our mock + path := r.URL.EscapedPath() + parts := strings.Split(strings.Trim(path, "/"), "/") + + if len(parts) < 4 || parts[0] != "api" || parts[1] != "repos" { + http.Error(w, "Invalid path", http.StatusBadRequest) + return + } + + encodedRepoID := parts[2] + if encodedRepoID == "" { + http.Error(w, "Repository ID required", http.StatusBadRequest) + return + } + + repoID, err := url.PathUnescape(encodedRepoID) + if err != nil { + http.Error(w, "Invalid repo ID encoding", http.StatusBadRequest) + return + } + + q := r.URL.Query() + q.Set("repo_id", repoID) + r.URL.RawQuery = q.Encode() + + resource := parts[3] + if resource == "rules" { + mockRulesHandler.ServeHTTP(w, r) + } else { + http.Error(w, "Not found", http.StatusNotFound) + } + }) + + tests := []struct { + name string + path string + wantStatus int + wantRepoID string + }{ + { + name: "URL-encoded absolute path", + path: "/api/repos/%2Ftmp%2Ftest-repo/rules", + wantStatus: http.StatusOK, + wantRepoID: "/tmp/test-repo", + }, + { + name: "URL-encoded path with spaces", + path: "/api/repos/%2Fhome%2Fuser%2Fmy%20project/rules", + wantStatus: http.StatusOK, + wantRepoID: "/home/user/my project", + }, + { + name: "Non-absolute path (simple name)", + path: "/api/repos/simple-repo/rules", + wantStatus: http.StatusOK, + wantRepoID: "simple-repo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + receivedRepoID = "" // reset + req := httptest.NewRequest(http.MethodGet, tt.path, nil) + w := httptest.NewRecorder() + + testHandler.ServeHTTP(w, req) + + if w.Code != tt.wantStatus { + t.Errorf("Expected status %d, got %d. Body: %s", tt.wantStatus, w.Code, w.Body.String()) + } + + if receivedRepoID != tt.wantRepoID { + t.Errorf("Expected repo_id %q, got %q", tt.wantRepoID, receivedRepoID) + } + }) + } +} + +func TestHandleReposRoutes_EmptyRepoID(t *testing.T) { + state := NewRuntimeState() + eventBus := NewEventBus() + scheduler := &mockScheduler{} + beadsClient := &mockBeadsClient{} + + server := NewServer(testPort, state, eventBus, scheduler, beadsClient) + + // Test directly calling handleReposRoutes + req := httptest.NewRequest(http.MethodGet, "/api/repos//rules", nil) + w := httptest.NewRecorder() + + server.handleReposRoutes(w, req) + + // When repo_id is empty, we expect a 400 error + if w.Code != http.StatusBadRequest { + t.Errorf("Expected status 400, got %d. Body: %s", w.Code, w.Body.String()) + } +} diff --git a/pkg/daemon/state.go b/pkg/daemon/state.go index 6c5d5639..791458cd 100644 --- a/pkg/daemon/state.go +++ b/pkg/daemon/state.go @@ -5,22 +5,25 @@ import ( "time" "github.com/jzila/canopy/pkg/beads" + "github.com/jzila/canopy/pkg/lifecycle" "github.com/jzila/canopy/pkg/logging" "github.com/jzila/canopy/pkg/metrics" "github.com/jzila/canopy/pkg/persistence" "github.com/jzila/canopy/pkg/types" ) -// AgentStatus represents the current state of an agent -type AgentStatus string +// AgentStatus is an alias to types.AgentStatus for backwards compatibility. +// New code should import types.AgentStatus directly. +type AgentStatus = types.AgentStatus +// AgentStatus constants - aliases to types package for backwards compatibility. const ( - AgentStatusStarting AgentStatus = "starting" - AgentStatusRunning AgentStatus = "running" - AgentStatusCompleted AgentStatus = "completed" - AgentStatusFailed AgentStatus = "failed" - AgentStatusTimedOut AgentStatus = "timed_out" - AgentStatusCancelled AgentStatus = "cancelled" + AgentStatusStarting = types.AgentStatusStarting + AgentStatusRunning = types.AgentStatusRunning + AgentStatusCompleted = types.AgentStatusCompleted + AgentStatusFailed = types.AgentStatusFailed + AgentStatusTimedOut = types.AgentStatusTimedOut + AgentStatusCancelled = types.AgentStatusCancelled ) // OutputBuffer stores stdout/stderr output from an agent @@ -306,6 +309,23 @@ func getBoolFromMap(m map[string]interface{}, key string) bool { return false } +func getInt64FromMap(m map[string]interface{}, key string) (int64, bool) { + val, exists := m[key] + if !exists { + return 0, false + } + switch v := val.(type) { + case int: + return int64(v), true + case int64: + return v, true + case float64: + return int64(v), true + default: + return 0, false + } +} + // GitCommit represents a git commit made by an agent type GitCommit struct { Hash string `json:"hash"` // Full commit hash @@ -317,6 +337,14 @@ type GitCommit struct { FilesChanged []string `json:"files_changed"` // List of files modified in this commit } +// LifecycleHistoryEntry represents a single state transition in the agent lifecycle. +type LifecycleHistoryEntry struct { + From string `json:"from"` // Previous state + To string `json:"to"` // New state + Event string `json:"event"` // Event that triggered the transition + Timestamp string `json:"timestamp"` // ISO 8601 timestamp +} + // Append adds new output to the buffer (thread-safe) func (b *OutputBuffer) Append(stdout, stderr string) { b.mu.Lock() @@ -384,13 +412,16 @@ type MergeStatus = types.MergeStatus // MergeStatus constants - aliases to types package for backwards compatibility. const ( - MergeStatusNone = types.MergeStatusNone - MergeStatusPending = types.MergeStatusPending - MergeStatusAcquiring = types.MergeStatusAcquiring - MergeStatusMerging = types.MergeStatusMerging - MergeStatusResolving = types.MergeStatusResolving - MergeStatusMerged = types.MergeStatusMerged - MergeStatusFailed = types.MergeStatusFailed + MergeStatusNone = types.MergeStatusNone + MergeStatusPending = types.MergeStatusPending + MergeStatusAcquiring = types.MergeStatusAcquiring + MergeStatusMerging = types.MergeStatusMerging + MergeStatusResolving = types.MergeStatusResolving + MergeStatusMerged = types.MergeStatusMerged + MergeStatusFailed = types.MergeStatusFailed + MergeStatusResolved = types.MergeStatusResolved + MergeStatusSkipped = types.MergeStatusSkipped + MergeStatusMergedNeedsRepair = types.MergeStatusMergedNeedsRepair ) // ValidationStep is an alias to types.ValidationStep for backwards compatibility. @@ -407,8 +438,20 @@ type AgentState struct { RepoID string `json:"repo_id,omitempty"` // Repository this agent is working in ParentAgentID string `json:"parent_agent_id,omitempty"` // ID of parent agent if spawned by another agent ChildAgentIDs []string `json:"child_agent_ids,omitempty"` // IDs of child agents spawned by this agent - Status AgentStatus `json:"status"` // Current agent status - MergeStatus MergeStatus `json:"merge_status,omitempty"` // Current merge queue status + Status AgentStatus `json:"status"` // Current agent status (legacy) + MergeStatus MergeStatus `json:"merge_status,omitempty"` // Current merge queue status (legacy) + + // Lifecycle is the new state machine that runs in parallel with legacy fields. + // During the transition period (Phase 1), both systems run concurrently and + // divergence is logged for monitoring. The lifecycle field is not serialized + // to JSON as it is ephemeral runtime state. + Lifecycle *lifecycle.AgentLifecycle `json:"-"` + // LifecycleState is the current lifecycle state as a string for JSON serialization. + // Derived from the Lifecycle state machine when available. + LifecycleState string `json:"lifecycle_state,omitempty"` + // LifecycleHistory contains the state transition history for debugging. + // Populated on demand (e.g., when --show-history is requested). + LifecycleHistory []LifecycleHistoryEntry `json:"lifecycle_history,omitempty"` MergeQueuePos int `json:"merge_queue_pos,omitempty"` // Position in merge wait queue (0 = not waiting) MergeError string `json:"merge_error,omitempty"` // Error message if merge failed StartTime time.Time `json:"start_time"` // When agent started @@ -437,6 +480,8 @@ type AgentState struct { IsResume bool `json:"is_resume,omitempty"` // True if this agent was resumed after daemon restart ResumeCount int `json:"resume_count,omitempty"` // Number of times this agent has been resumed InterruptedAt *time.Time `json:"interrupted_at,omitempty"` // When the agent was interrupted (for resumed agents) + Attempt int `json:"attempt,omitempty"` // Current attempt number (1 = first try) + MaxRetries int `json:"max_retries,omitempty"` // Max retry attempts configured mu sync.RWMutex } @@ -456,6 +501,22 @@ func (a *AgentState) GetSnapshot() AgentState { // Get output buffer values safely stdout, stderr := a.Output.Get() + // Derive lifecycle state from the state machine if available + var lifecycleState string + var lifecycleHistory []LifecycleHistoryEntry + if a.Lifecycle != nil { + lifecycleState = a.Lifecycle.State().String() + // Copy history entries + for _, h := range a.Lifecycle.History() { + lifecycleHistory = append(lifecycleHistory, LifecycleHistoryEntry{ + From: h.From.String(), + To: h.To.String(), + Event: h.Event.String(), + Timestamp: h.Timestamp.Format(time.RFC3339), + }) + } + } + // Copy all fields except mutexes return AgentState{ ID: a.ID, @@ -468,6 +529,8 @@ func (a *AgentState) GetSnapshot() AgentState { ChildAgentIDs: append([]string(nil), a.ChildAgentIDs...), Status: a.Status, MergeStatus: a.MergeStatus, + LifecycleState: lifecycleState, + LifecycleHistory: lifecycleHistory, MergeQueuePos: a.MergeQueuePos, MergeError: a.MergeError, StartTime: a.StartTime, @@ -496,6 +559,8 @@ func (a *AgentState) GetSnapshot() AgentState { IsResume: a.IsResume, ResumeCount: a.ResumeCount, InterruptedAt: a.InterruptedAt, + Attempt: a.Attempt, + MaxRetries: a.MaxRetries, // mu is intentionally not copied } } @@ -549,6 +614,10 @@ type RuntimeState struct { persistentTasks map[string]*TaskState // From beads - does NOT get modified during runtime runtimeTasks map[string]*TaskState // Runtime overlay - rebuilt from agent events + // eventBus is stored when SubscribeToEventBus is called, enabling lifecycle + // transition callbacks to publish events for real-time UI updates. + eventBus *EventBus + mu sync.RWMutex } @@ -843,12 +912,21 @@ type RuntimeStateSnapshot struct { StartTime time.Time `json:"start_time"` CurrentRunID string `json:"current_run_id"` + // Orchestrator state fields + OrchestratorState string `json:"orchestrator_state,omitempty"` // off, idle, active, paused + ActiveAgentCount int `json:"active_agent_count,omitempty"` // Number of currently running agents + // Hybrid overlay architecture: dual-source task state // PersistentTasks: canonical state from beads (source of truth) // RuntimeTasks: ephemeral overlay (in_progress, agent assignments) // Frontend merges: runtime overlays persistent for display PersistentTasks map[string]*TaskState `json:"persistent_tasks,omitempty"` RuntimeTasks map[string]*TaskState `json:"runtime_tasks,omitempty"` + + // EventSequence is the sequence number of the last event published before + // this snapshot was taken. Clients should discard any events with sequence + // numbers <= this value, as they are already reflected in the snapshot. + EventSequence uint64 `json:"event_sequence,omitempty"` } // GetSnapshot returns a complete snapshot of the runtime state (thread-safe) @@ -859,6 +937,13 @@ func (r *RuntimeState) GetSnapshot() RuntimeStateSnapshot { r.mu.RLock() defer r.mu.RUnlock() + // Get current event sequence if eventBus is available. + // Clients can use this to discard events that occurred before the snapshot. + var eventSequence uint64 + if r.eventBus != nil { + eventSequence = r.eventBus.GetSequence() + } + snapshot := RuntimeStateSnapshot{ Agents: make(map[string]*AgentState), Tasks: make(map[string]*TaskState), @@ -868,6 +953,7 @@ func (r *RuntimeState) GetSnapshot() RuntimeStateSnapshot { IsPaused: r.IsPaused, StartTime: r.StartTime, CurrentRunID: r.CurrentRunID, + EventSequence: eventSequence, } // Deep copy agents @@ -940,6 +1026,12 @@ func (r *RuntimeState) GetSnapshotForRepo(repoID string) RuntimeStateSnapshot { r.mu.RLock() defer r.mu.RUnlock() + // Get current event sequence if eventBus is available. + var eventSequence uint64 + if r.eventBus != nil { + eventSequence = r.eventBus.GetSequence() + } + snapshot := RuntimeStateSnapshot{ Agents: make(map[string]*AgentState), Tasks: make(map[string]*TaskState), @@ -948,6 +1040,7 @@ func (r *RuntimeState) GetSnapshotForRepo(repoID string) RuntimeStateSnapshot { IsPaused: r.IsPaused, StartTime: r.StartTime, CurrentRunID: r.CurrentRunID, + EventSequence: eventSequence, } // Filter and deep copy agents (note: agents don't have repo_id in struct yet, @@ -1042,9 +1135,149 @@ func (s *RuntimeStateSnapshot) recalculateStats() { s.Stats = stats } +// expectedLifecycleState returns the lifecycle state that corresponds to the legacy +// Status and MergeStatus fields. Used to detect divergence during parallel rollout. +func expectedLifecycleState(status AgentStatus, mergeStatus MergeStatus, validationStatus string) lifecycle.AgentLifecycleState { + // First check validation status if merge is complete + if mergeStatus == MergeStatusMerged || mergeStatus == MergeStatusResolved { + switch validationStatus { + case "running": + return lifecycle.StateValidating + case "repairing": + return lifecycle.StateRepairing + case "failed": + return lifecycle.StateNeedsAttention + case "passed", "skipped", "": + // Fall through to check agent status + } + } + + // Check merge status + switch mergeStatus { + case MergeStatusPending: + return lifecycle.StateQueuedForMerge + case MergeStatusAcquiring, MergeStatusMerging: + return lifecycle.StateMerging + case MergeStatusResolving: + return lifecycle.StateResolving + case MergeStatusFailed: + return lifecycle.StateMergeFailed + } + + // Check agent status + switch status { + case AgentStatusStarting: + return lifecycle.StateStarting + case AgentStatusRunning: + // If we have a merge status indicating queue, use that + if mergeStatus == MergeStatusPending { + return lifecycle.StateQueuedForMerge + } + return lifecycle.StateRunning + case AgentStatusCompleted: + return lifecycle.StateCompleted + case AgentStatusFailed: + return lifecycle.StateFailed + case AgentStatusCancelled: + return lifecycle.StateCancelled + case AgentStatusTimedOut: + return lifecycle.StateTimedOut + } + + // Default to running if we can't determine + return lifecycle.StateRunning +} + +// checkLifecycleDivergence compares the lifecycle state machine state against the +// legacy status fields and logs a warning + records a metric if they diverge. +// This is used during the parallel rollout phase to detect inconsistencies. +func checkLifecycleDivergence(agent *AgentState, event string) { + if agent == nil || agent.Lifecycle == nil { + return + } + + // Read legacy fields under lock + agent.mu.RLock() + legacyStatus := agent.Status + mergeStatus := agent.MergeStatus + validationStatus := agent.ValidationStatus + agentID := agent.ID + agent.mu.RUnlock() + + lifecycleState := agent.Lifecycle.State() + expected := expectedLifecycleState(legacyStatus, mergeStatus, validationStatus) + + if lifecycleState != expected { + logging.Warn("lifecycle state divergence detected", + "agent_id", agentID, + "event", event, + "legacy_status", string(legacyStatus), + "merge_status", string(mergeStatus), + "validation_status", validationStatus, + "lifecycle_state", lifecycleState.String(), + "expected_lifecycle_state", expected.String(), + ) + metrics.RecordLifecycleDivergence(string(legacyStatus), lifecycleState.String(), event) + } +} + +// makeLifecycleCallback creates a callback that publishes lifecycle state transitions +// to the EventBus for real-time UI updates. All lifecycle state changes are published +// via EventLifecycleStateChanged, with EventAgentRunning as an additional event for +// backwards compatibility when transitioning to the running state. +func (r *RuntimeState) makeLifecycleCallback(agentID string) lifecycle.TransitionCallback { + return func(from, to lifecycle.AgentLifecycleState, event lifecycle.AgentEvent) { + r.mu.RLock() + eventBus := r.eventBus + r.mu.RUnlock() + + if eventBus == nil { + return + } + + // Skip publishing for terminal states that are already handled elsewhere + // (EventAgentCompleted/EventAgentFailed are sent with full result data) + if to == lifecycle.StateCompleted || to == lifecycle.StateFailed { + return + } + + // Publish the lifecycle state change event for all intermediate states + eventBus.Publish(Event{ + Type: EventLifecycleStateChanged, + Timestamp: time.Now(), + Payload: map[string]interface{}{ + "agent_id": agentID, + "lifecycle_state": to.String(), + "previous_state": from.String(), + "event": event.String(), + }, + }) + + // Also publish EventAgentRunning for backwards compatibility + // This ensures existing code that listens for agent:running still works + if to == lifecycle.StateRunning { + eventBus.Publish(Event{ + Type: EventAgentRunning, + Timestamp: time.Now(), + Payload: map[string]interface{}{ + "agent_id": agentID, + "lifecycle_state": to.String(), + "previous_state": from.String(), + "event": event.String(), + }, + }) + } + } +} + // SubscribeToEventBus subscribes to the EventBus and updates state from events. // Returns an unsubscribe function. This bridges IPC events to RuntimeState updates. +// Also stores the eventBus reference for publishing lifecycle transition events. func (r *RuntimeState) SubscribeToEventBus(eventBus *EventBus) func() { + r.mu.Lock() + r.eventBus = eventBus + r.mu.Unlock() + return eventBus.Subscribe(func(event Event) { r.handleEvent(event) }) @@ -1078,6 +1311,8 @@ func (r *RuntimeState) handleEvent(event Event) { case EventStatsUpdated: // Stats updates are informational, we recalculate from agents r.UpdateStats() + case EventLifecycleStateChanged: + r.handleLifecycleStateChanged(payload) } } @@ -1101,6 +1336,15 @@ func (r *RuntimeState) handleAgentStarted(payload map[string]interface{}, timest parentAgentID, _ := payload["parent_agent_id"].(string) repoID, _ := payload["repo_id"].(string) + // Extract retry information from payload + var attempt, maxRetries int + if attemptVal, ok := payload["attempt"].(float64); ok { + attempt = int(attemptVal) + } + if maxRetriesVal, ok := payload["max_retries"].(float64); ok { + maxRetries = int(maxRetriesVal) + } + if agentID == "" { return } @@ -1112,6 +1356,11 @@ func (r *RuntimeState) handleAgentStarted(payload map[string]interface{}, timest r.mu.RUnlock() } + // Initialize lifecycle state machine with transition callback for real-time UI updates + agentLifecycle := lifecycle.New( + lifecycle.WithTransitionCallback(r.makeLifecycleCallback(agentID)), + ) + agent := &AgentState{ ID: agentID, RunID: runID, @@ -1122,10 +1371,29 @@ func (r *RuntimeState) handleAgentStarted(payload map[string]interface{}, timest ParentAgentID: parentAgentID, Status: AgentStatusRunning, StartTime: timestamp, + Lifecycle: agentLifecycle, + Attempt: attempt, + MaxRetries: maxRetries, } + // IMPORTANT: Add agent to RuntimeState BEFORE lifecycle transition. + // The lifecycle transition publishes events via the callback. If we transition + // first, clients receiving the event may request a state snapshot that doesn't + // include the agent yet, causing stale state in the UI. r.AddAgent(agent) + // Transition lifecycle to running state (parallel with legacy Status field) + if err := agentLifecycle.Transition(lifecycle.EventAgentSpawned, lifecycle.TransitionContext{}); err != nil { + logging.Warn("lifecycle transition failed on agent start", + "agent_id", agentID, + "event", lifecycle.EventAgentSpawned, + "error", err, + ) + } + + // Check for divergence between legacy and lifecycle state + checkLifecycleDivergence(agent, "agent_started") + // Link child to parent agent if parent exists if parentAgentID != "" { if parent := r.GetAgent(parentAgentID); parent != nil { @@ -1284,10 +1552,282 @@ func (r *RuntimeState) handleAgentMergeStatus(payload map[string]interface{}) { queuePos, _ := getIntFromPayload(payload, "queue_pos") mergeErr, _ := payload["error"].(string) + // Extract merge result fields + commitsApplied, _ := getIntFromPayload(payload, "commits_applied") + hadConflict, _ := payload["had_conflict"].(bool) + resolverSpawned, _ := payload["resolver_spawned"].(bool) + + // Extract validation fields + validationStatus, _ := payload["validation_status"].(string) + validationError, _ := payload["validation_error"].(string) + validationDuration, _ := getInt64FromPayload(payload, "validation_duration_ms") + + // Extract validation steps (comes as []interface{} from JSON) + var validationSteps []ValidationStep + if stepsRaw, ok := payload["validation_steps"].([]interface{}); ok { + for _, stepRaw := range stepsRaw { + if stepMap, ok := stepRaw.(map[string]interface{}); ok { + step := ValidationStep{ + Name: getStringFromMap(stepMap, "name"), + Status: getStringFromMap(stepMap, "status"), + Output: getStringFromMap(stepMap, "output"), + } + if dur, ok := getInt64FromMap(stepMap, "duration_ms"); ok { + step.Duration = dur + } + validationSteps = append(validationSteps, step) + } + } + } else if stepsTyped, ok := payload["validation_steps"].([]ValidationStep); ok { + // Direct type assertion if already typed (e.g., from internal events) + validationSteps = stepsTyped + } + + // Extract repair tracking fields + repairAttempts, _ := getIntFromPayload(payload, "repair_attempts") + lastRepairOutput, _ := payload["last_repair_output"].(string) + + // Get taskID before update (immutable after creation, but read under lock for safety) + agent.mu.RLock() + taskID := agent.TaskID + agent.mu.RUnlock() + + now := time.Now() agent.Update(func(a *AgentState) { a.MergeStatus = MergeStatus(mergeStatus) a.MergeQueuePos = queuePos a.MergeError = mergeErr + + // Update agent status atomically with merge status for terminal states. + // This prevents the race condition where agent appears 'running' after + // merge completes but before EventAgentCompleted is processed. + switch MergeStatus(mergeStatus) { + case MergeStatusMerged, MergeStatusMergedNeedsRepair, MergeStatusResolved, MergeStatusSkipped: + // Merge succeeded (with or without repair/resolution) + a.Status = AgentStatusCompleted + if a.EndTime == nil { + a.EndTime = &now + } + case MergeStatusFailed: + // Merge failed + a.Status = AgentStatusFailed + if a.EndTime == nil { + a.EndTime = &now + } + } + + // Update merge result fields (only if present to avoid overwriting) + if commitsApplied > 0 { + a.Commits = commitsApplied + } + // Store conflict info in merge error if there was a conflict + if hadConflict && a.MergeError == "" { + a.MergeError = "merge had conflicts" + } + // Note: resolverSpawned is informational, no field to store it currently + _ = resolverSpawned + + // Update validation fields + if validationStatus != "" { + a.ValidationStatus = validationStatus + } + if validationError != "" { + a.ValidationError = validationError + } + if validationDuration > 0 { + a.ValidationDuration = validationDuration + } + if len(validationSteps) > 0 { + a.ValidationSteps = validationSteps + } + + // Update repair tracking fields + if repairAttempts > 0 { + a.RepairAttempts = repairAttempts + } + if lastRepairOutput != "" { + a.LastRepairOutput = lastRepairOutput + } + }) + + // Update lifecycle state machine (parallel with legacy fields) + if agent.Lifecycle != nil { + transitionLifecycleForMergeStatus(agent, mergeStatus, hadConflict, validationStatus, repairAttempts) + } + + // Check for divergence after all updates + checkLifecycleDivergence(agent, "merge_status_"+mergeStatus) + + // Update task status for terminal merge states + if taskID != "" { + switch MergeStatus(mergeStatus) { + case MergeStatusMerged, MergeStatusMergedNeedsRepair, MergeStatusResolved, MergeStatusSkipped: + r.UpdateTaskStatus(taskID, "completed", agentID) + case MergeStatusFailed: + r.UpdateTaskStatus(taskID, "failed", agentID) + } + } + + // Recalculate stats to reflect agent completion + r.UpdateStats() +} + +// transitionLifecycleForMergeStatus maps merge status changes to lifecycle events. +// Called during handleAgentMergeStatus to keep lifecycle state machine in sync. +func transitionLifecycleForMergeStatus(agent *AgentState, mergeStatus string, hadConflict bool, validationStatus string, repairAttempts int) { + if agent.Lifecycle == nil { + return + } + + agentID := agent.ID + lc := agent.Lifecycle + + // Build transition context + ctx := lifecycle.TransitionContext{ + RepairAttempt: repairAttempts, + MaxRepairAttempts: 3, // Default, would need to get from config + } + + // Determine the appropriate lifecycle event based on merge status + var event lifecycle.AgentEvent + var shouldTransition bool + + switch MergeStatus(mergeStatus) { + case MergeStatusPending: + // Agent work complete, now queued for merge + if lc.State() == lifecycle.StateRunning { + event = lifecycle.EventWorkComplete + shouldTransition = true + } + + case MergeStatusAcquiring, MergeStatusMerging: + // Merge started + if lc.State() == lifecycle.StateQueuedForMerge { + event = lifecycle.EventMergeStarted + shouldTransition = true + } + + case MergeStatusResolving: + // Merge had conflict, now resolving + if lc.State() == lifecycle.StateMerging && hadConflict { + event = lifecycle.EventMergeConflict + shouldTransition = true + } + + case MergeStatusMerged, MergeStatusResolved, MergeStatusSkipped: + // Handle based on current state and validation status + currentState := lc.State() + + // If we're in merging state and merge succeeded + if currentState == lifecycle.StateMerging { + event = lifecycle.EventMergeSuccess + ctx.ValidationEnabled = validationStatus != "" && validationStatus != "skipped" + shouldTransition = true + } else if currentState == lifecycle.StateResolving { + event = lifecycle.EventResolveSuccess + ctx.ValidationEnabled = validationStatus != "" && validationStatus != "skipped" + shouldTransition = true + } else if currentState == lifecycle.StateValidating { + // Validation completed + if validationStatus == "passed" { + event = lifecycle.EventValidationPassed + shouldTransition = true + } else if validationStatus == "skipped" { + event = lifecycle.EventValidationSkipped + shouldTransition = true + } + } else if currentState == lifecycle.StateRepairing { + // Repair complete, back to validation + event = lifecycle.EventRepairComplete + shouldTransition = true + } + + case MergeStatusMergedNeedsRepair: + // Validation failed but kept the merge + if lc.State() == lifecycle.StateValidating { + event = lifecycle.EventValidationFailed + ctx.RepairEnabled = false // No more repair attempts + ctx.StrictMode = false // Lenient mode keeps merge + shouldTransition = true + } + + case MergeStatusFailed: + // Merge failed + currentState := lc.State() + if currentState == lifecycle.StateMerging { + event = lifecycle.EventMergeFailed + shouldTransition = true + } else if currentState == lifecycle.StateResolving { + event = lifecycle.EventResolveFailed + shouldTransition = true + } else if currentState == lifecycle.StateValidating { + event = lifecycle.EventValidationFailed + ctx.StrictMode = true // Strict mode fails on validation failure + shouldTransition = true + } + } + + // Also handle validation status transitions if merge is already complete + if !shouldTransition && validationStatus != "" { + currentState := lc.State() + switch validationStatus { + case "running": + // Validation starting - should already be in validating state + case "passed": + if currentState == lifecycle.StateValidating { + event = lifecycle.EventValidationPassed + shouldTransition = true + } + case "failed": + if currentState == lifecycle.StateValidating { + event = lifecycle.EventValidationFailed + ctx.RepairEnabled = repairAttempts > 0 + shouldTransition = true + } + case "repairing": + if currentState == lifecycle.StateValidating { + event = lifecycle.EventValidationFailed + ctx.RepairEnabled = true + ctx.RepairAttempt = repairAttempts + shouldTransition = true + } + } + } + + if shouldTransition { + if err := lc.Transition(event, ctx); err != nil { + logging.Warn("lifecycle transition failed on merge status", + "agent_id", agentID, + "event", event, + "merge_status", mergeStatus, + "current_state", lc.State(), + "error", err, + ) + } + } +} + +// handleLifecycleStateChanged updates the agent's lifecycle state when notified +// of a lifecycle state transition. This keeps the RuntimeState in sync with +// the lifecycle state machine's transitions for real-time UI updates. +func (r *RuntimeState) handleLifecycleStateChanged(payload map[string]interface{}) { + agentID, _ := payload["agent_id"].(string) + if agentID == "" { + return + } + + lifecycleState, _ := payload["lifecycle_state"].(string) + if lifecycleState == "" { + return + } + + agent := r.GetAgent(agentID) + if agent == nil { + return + } + + agent.Update(func(a *AgentState) { + a.LifecycleState = lifecycleState }) } @@ -1385,6 +1925,16 @@ func (r *RuntimeState) handleAgentCompleted(payload map[string]interface{}, time } }) + // Update lifecycle state machine for completion (parallel with legacy fields) + // Note: Many completion events are already handled via merge status events, + // but agents can also complete directly (e.g., work failed, cancelled, timeout). + if agent.Lifecycle != nil { + transitionLifecycleForCompletion(agent, payload) + } + + // Check for divergence after all updates + checkLifecycleDivergence(agent, "agent_completed") + // Update task status if agent.TaskID != "" { status := "completed" @@ -1397,6 +1947,55 @@ func (r *RuntimeState) handleAgentCompleted(payload map[string]interface{}, time r.UpdateStats() } +// transitionLifecycleForCompletion handles lifecycle transitions for agent completion. +// Called during handleAgentCompleted to keep lifecycle state machine in sync. +func transitionLifecycleForCompletion(agent *AgentState, payload map[string]interface{}) { + if agent.Lifecycle == nil { + return + } + + agentID := agent.ID + lc := agent.Lifecycle + currentState := lc.State() + + // Skip if already in a terminal state + if currentState.IsTerminal() { + return + } + + // Determine the appropriate lifecycle event + var event lifecycle.AgentEvent + var shouldTransition bool + ctx := lifecycle.TransitionContext{} + + // Check if this is an error completion + errMsg, hasError := payload["error"].(string) + if hasError && errMsg != "" { + ctx.Error = errMsg + // Work failed while running + if currentState == lifecycle.StateRunning { + event = lifecycle.EventWorkFailed + ctx.AttemptsRemaining = 0 // No retries for direct failure + shouldTransition = true + } + } + + // If not an error and lifecycle isn't already terminal/advanced past running, + // we don't need to transition here - the merge status handler will do it. + // This is because agent_completed events often arrive after merge completion. + + if shouldTransition { + if err := lc.Transition(event, ctx); err != nil { + logging.Warn("lifecycle transition failed on agent completed", + "agent_id", agentID, + "event", event, + "current_state", currentState, + "error", err, + ) + } + } +} + func (r *RuntimeState) handleTaskUpdated(payload map[string]interface{}) { taskID, _ := payload["id"].(string) if taskID == "" { diff --git a/pkg/events/events.go b/pkg/events/events.go index 501c3464..b8493996 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -19,6 +19,7 @@ const ( EventRunStarted EventType = "run:started" EventRunCompleted EventType = "run:completed" EventAgentStarted EventType = "agent:started" + EventAgentRunning EventType = "agent:running" // Agent transitioned from starting to running EventAgentResumed EventType = "agent:resumed" // Agent resumed after daemon restart EventAgentOutput EventType = "agent:output" EventAgentOutputClear EventType = "agent:output_clear" @@ -31,8 +32,11 @@ const ( EventTaskUpdated EventType = "task:updated" EventOrchPaused EventType = "orch:paused" EventOrchResumed EventType = "orch:resumed" + EventOrchStateChanged EventType = "orch:state_changed" // Orchestrator state changed (off/idle/active/paused) EventStatsUpdated EventType = "stats:updated" - EventRulesChanged EventType = "rules:changed" // Rules configuration changed at runtime + EventRulesChanged EventType = "rules:changed" // Rules configuration changed at runtime + EventLifecycleStateChanged EventType = "lifecycle:state_changed" // Agent lifecycle state transition + EventConfigUpdated EventType = "config:updated" // Run configuration updated ) // IsCritical returns true if this event type must never be dropped. @@ -41,11 +45,12 @@ const ( func (et EventType) IsCritical() bool { switch et { case EventRunStarted, EventRunCompleted, - EventAgentStarted, EventAgentResumed, EventAgentCompleted, + EventAgentStarted, EventAgentRunning, EventAgentResumed, EventAgentCompleted, EventAgentDone, EventAgentFailed, EventAgentMergeStatus, - EventOrchPaused, EventOrchResumed, - EventTaskUpdated: + EventOrchPaused, EventOrchResumed, EventOrchStateChanged, + EventTaskUpdated, + EventLifecycleStateChanged: return true default: return false @@ -57,6 +62,7 @@ type Event struct { Type EventType `json:"type"` Timestamp time.Time `json:"timestamp"` Payload interface{} `json:"payload"` + Sequence uint64 `json:"sequence,omitempty"` // Monotonic sequence number for ordering } // EventHandler is a function that receives events @@ -75,6 +81,7 @@ type EventBus struct { mu sync.RWMutex subscriptions map[int]subscription nextID int + sequence uint64 // Monotonic sequence number for event ordering } // NewEventBus creates a new EventBus instance @@ -110,14 +117,18 @@ func (eb *EventBus) Subscribe(handler EventHandler) func() { // Publish sends an event to all subscribers // Handlers are called synchronously in the order they subscribed // If a handler panics, it does not affect other handlers +// Each event is assigned a monotonic sequence number for ordering. func (eb *EventBus) Publish(event Event) { - eb.mu.RLock() - // Copy subscriptions to avoid holding read lock during handler execution + eb.mu.Lock() + // Assign monotonic sequence number + eb.sequence++ + event.Sequence = eb.sequence + // Copy subscriptions to avoid holding lock during handler execution handlers := make([]EventHandler, 0, len(eb.subscriptions)) for _, sub := range eb.subscriptions { handlers = append(handlers, sub.handler) } - eb.mu.RUnlock() + eb.mu.Unlock() // Execute handlers without holding lock for _, handler := range handlers { @@ -137,6 +148,15 @@ func (eb *EventBus) Publish(event Event) { } } +// GetSequence returns the current sequence number. +// This is useful for including in snapshots so clients can discard +// events that occurred before the snapshot was taken. +func (eb *EventBus) GetSequence() uint64 { + eb.mu.RLock() + defer eb.mu.RUnlock() + return eb.sequence +} + // SubscriberCount returns the number of active subscriptions // Useful for testing and debugging func (eb *EventBus) SubscriberCount() int { diff --git a/pkg/ipc/client.go b/pkg/ipc/client.go index 23b00974..8d68b70c 100644 --- a/pkg/ipc/client.go +++ b/pkg/ipc/client.go @@ -318,12 +318,24 @@ func (c *Client) flushQueueLocked() { } } +// AgentStartOptions contains optional parameters for SendAgentStart +type AgentStartOptions struct { + Attempt int // Current attempt number (1-indexed, 0 means first attempt) + MaxRetries int // Maximum retry attempts configured (0 = no retries, -1 = infinite) +} + // SendAgentStart notifies the daemon that an agent has started executing a task // runID is the ID of the run this agent belongs to (for historical filtering) // taskDescription is optional and provides the full task description // parentAgentID is optional and specifies the ID of the parent agent if this agent was spawned by another // repoID is optional and specifies the repository ID for tracking func (c *Client) SendAgentStart(agentID, runID, taskID, taskTitle, taskDescription, parentAgentID, repoID string) error { + return c.SendAgentStartWithOptions(agentID, runID, taskID, taskTitle, taskDescription, parentAgentID, repoID, nil) +} + +// SendAgentStartWithOptions notifies the daemon that an agent has started executing a task with additional options. +// opts can be nil for default behavior (no retry information). +func (c *Client) SendAgentStartWithOptions(agentID, runID, taskID, taskTitle, taskDescription, parentAgentID, repoID string, opts *AgentStartOptions) error { payload := AgentStartPayload{ AgentID: agentID, RunID: runID, @@ -334,6 +346,11 @@ func (c *Client) SendAgentStart(agentID, runID, taskID, taskTitle, taskDescripti RepoID: repoID, } + if opts != nil { + payload.Attempt = opts.Attempt + payload.MaxRetries = opts.MaxRetries + } + return c.sendMessage(MessageTypeAgentStart, payload) } diff --git a/pkg/ipc/protocol.go b/pkg/ipc/protocol.go index 763e3d51..24dfa75a 100644 --- a/pkg/ipc/protocol.go +++ b/pkg/ipc/protocol.go @@ -140,6 +140,8 @@ type AgentStartPayload struct { IsResume bool `json:"is_resume,omitempty"` // True if this agent is being resumed after daemon restart ResumeCount int `json:"resume_count,omitempty"` // Number of times this agent has been resumed SessionID string `json:"session_id,omitempty"` // Claude CLI session ID being resumed + Attempt int `json:"attempt,omitempty"` // Current attempt number (1-indexed, 0 means not a retry) + MaxRetries int `json:"max_retries,omitempty"` // Maximum retry attempts configured (0 = no retries, -1 = infinite) } // AgentResumedPayload is sent when an agent is resumed after daemon restart diff --git a/pkg/ipc/server.go b/pkg/ipc/server.go index 58696aa7..2a44df02 100644 --- a/pkg/ipc/server.go +++ b/pkg/ipc/server.go @@ -316,6 +316,13 @@ func (s *Server) convertToEvent(msg *Message) *events.Event { if payload.RepoID != "" { eventPayload["repo_id"] = payload.RepoID } + // Include retry information for UI display + if payload.Attempt > 0 { + eventPayload["attempt"] = payload.Attempt + } + if payload.MaxRetries != 0 { + eventPayload["max_retries"] = payload.MaxRetries + } return &events.Event{ Type: events.EventAgentStarted, Timestamp: msg.Timestamp, diff --git a/pkg/lifecycle/legacy.go b/pkg/lifecycle/legacy.go new file mode 100644 index 00000000..02806693 --- /dev/null +++ b/pkg/lifecycle/legacy.go @@ -0,0 +1,126 @@ +package lifecycle + +import ( + "github.com/jzila/canopy/pkg/types" +) + +// LegacyStatus returns the legacy AgentStatus for backwards compatibility. +// This maps the unified lifecycle state to the original 6-value status enum. +func (l *AgentLifecycle) LegacyStatus() types.AgentStatus { + switch l.State() { + case StateStarting: + return types.AgentStatusStarting + case StateRunning, StateQueuedForMerge, StateMerging, StateResolving, StateValidating, StateRepairing: + return types.AgentStatusRunning + case StateCompleted: + return types.AgentStatusCompleted + case StateFailed, StateMergeFailed, StateNeedsAttention: + return types.AgentStatusFailed + case StateCancelled: + return types.AgentStatusCancelled + case StateTimedOut: + return types.AgentStatusTimedOut + default: + return types.AgentStatusRunning + } +} + +// LegacyMergeStatus returns the legacy MergeStatus for backwards compatibility. +// This maps the unified lifecycle state to the merge phase. +func (l *AgentLifecycle) LegacyMergeStatus() types.MergeStatus { + switch l.State() { + case StateStarting, StateRunning: + return types.MergeStatusNone + case StateQueuedForMerge: + return types.MergeStatusPending + case StateMerging: + return types.MergeStatusMerging + case StateResolving: + return types.MergeStatusResolving + case StateValidating, StateRepairing: + // Validation and repair happen AFTER successful merge + return types.MergeStatusMerged + case StateCompleted: + return types.MergeStatusMerged + case StateFailed, StateMergeFailed: + return types.MergeStatusFailed + case StateNeedsAttention: + return types.MergeStatusMergedNeedsRepair + case StateCancelled, StateTimedOut: + // These can occur at any phase; report as failed if we were in merge + return types.MergeStatusNone + default: + return types.MergeStatusNone + } +} + +// LegacyValidationStatus returns the legacy validation status string. +// Returns empty string for states that don't involve validation. +func (l *AgentLifecycle) LegacyValidationStatus() string { + switch l.State() { + case StateValidating: + return "running" + case StateRepairing: + return "repairing" + case StateCompleted: + // Only return "passed" if we went through validation + // Check history to see if we came from validating + history := l.History() + for i := len(history) - 1; i >= 0; i-- { + if history[i].From == StateValidating { + return "passed" + } + } + return "" + case StateNeedsAttention: + return "failed" + default: + return "" + } +} + +// LegacyStatusFromState returns the legacy AgentStatus for a given lifecycle state. +// This is a static version useful when you have a state but not the full lifecycle. +func LegacyStatusFromState(state AgentLifecycleState) types.AgentStatus { + switch state { + case StateStarting: + return types.AgentStatusStarting + case StateRunning, StateQueuedForMerge, StateMerging, StateResolving, StateValidating, StateRepairing: + return types.AgentStatusRunning + case StateCompleted: + return types.AgentStatusCompleted + case StateFailed, StateMergeFailed, StateNeedsAttention: + return types.AgentStatusFailed + case StateCancelled: + return types.AgentStatusCancelled + case StateTimedOut: + return types.AgentStatusTimedOut + default: + return types.AgentStatusRunning + } +} + +// LegacyMergeStatusFromState returns the legacy MergeStatus for a given lifecycle state. +// This is a static version useful when you have a state but not the full lifecycle. +func LegacyMergeStatusFromState(state AgentLifecycleState) types.MergeStatus { + switch state { + case StateStarting, StateRunning: + return types.MergeStatusNone + case StateQueuedForMerge: + return types.MergeStatusPending + case StateMerging: + return types.MergeStatusMerging + case StateResolving: + return types.MergeStatusResolving + case StateValidating, StateRepairing: + return types.MergeStatusMerged + case StateCompleted: + return types.MergeStatusMerged + case StateFailed, StateMergeFailed: + return types.MergeStatusFailed + case StateNeedsAttention: + return types.MergeStatusMergedNeedsRepair + default: + return types.MergeStatusNone + } +} diff --git a/pkg/lifecycle/legacy_test.go b/pkg/lifecycle/legacy_test.go new file mode 100644 index 00000000..a647b300 --- /dev/null +++ b/pkg/lifecycle/legacy_test.go @@ -0,0 +1,194 @@ +package lifecycle + +import ( + "testing" + + "github.com/jzila/canopy/pkg/types" +) + +func TestLegacyStatus(t *testing.T) { + tests := []struct { + state AgentLifecycleState + expected types.AgentStatus + }{ + {StateStarting, types.AgentStatusStarting}, + {StateRunning, types.AgentStatusRunning}, + {StateQueuedForMerge, types.AgentStatusRunning}, + {StateMerging, types.AgentStatusRunning}, + {StateResolving, types.AgentStatusRunning}, + {StateValidating, types.AgentStatusRunning}, + {StateRepairing, types.AgentStatusRunning}, + {StateCompleted, types.AgentStatusCompleted}, + {StateFailed, types.AgentStatusFailed}, + {StateMergeFailed, types.AgentStatusFailed}, + {StateNeedsAttention, types.AgentStatusFailed}, + {StateCancelled, types.AgentStatusCancelled}, + {StateTimedOut, types.AgentStatusTimedOut}, + } + + for _, tt := range tests { + t.Run(string(tt.state), func(t *testing.T) { + l := New(WithInitialState(tt.state)) + got := l.LegacyStatus() + if got != tt.expected { + t.Errorf("LegacyStatus() for %s = %s, want %s", tt.state, got, tt.expected) + } + }) + } +} + +func TestLegacyStatusFromState(t *testing.T) { + tests := []struct { + state AgentLifecycleState + expected types.AgentStatus + }{ + {StateStarting, types.AgentStatusStarting}, + {StateRunning, types.AgentStatusRunning}, + {StateQueuedForMerge, types.AgentStatusRunning}, + {StateMerging, types.AgentStatusRunning}, + {StateResolving, types.AgentStatusRunning}, + {StateValidating, types.AgentStatusRunning}, + {StateRepairing, types.AgentStatusRunning}, + {StateCompleted, types.AgentStatusCompleted}, + {StateFailed, types.AgentStatusFailed}, + {StateMergeFailed, types.AgentStatusFailed}, + {StateNeedsAttention, types.AgentStatusFailed}, + {StateCancelled, types.AgentStatusCancelled}, + {StateTimedOut, types.AgentStatusTimedOut}, + } + + for _, tt := range tests { + t.Run(string(tt.state), func(t *testing.T) { + got := LegacyStatusFromState(tt.state) + if got != tt.expected { + t.Errorf("LegacyStatusFromState(%s) = %s, want %s", tt.state, got, tt.expected) + } + }) + } +} + +func TestLegacyMergeStatus(t *testing.T) { + tests := []struct { + state AgentLifecycleState + expected types.MergeStatus + }{ + {StateStarting, types.MergeStatusNone}, + {StateRunning, types.MergeStatusNone}, + {StateQueuedForMerge, types.MergeStatusPending}, + {StateMerging, types.MergeStatusMerging}, + {StateResolving, types.MergeStatusResolving}, + {StateValidating, types.MergeStatusMerged}, + {StateRepairing, types.MergeStatusMerged}, + {StateCompleted, types.MergeStatusMerged}, + {StateFailed, types.MergeStatusFailed}, + {StateMergeFailed, types.MergeStatusFailed}, + {StateNeedsAttention, types.MergeStatusMergedNeedsRepair}, + {StateCancelled, types.MergeStatusNone}, + {StateTimedOut, types.MergeStatusNone}, + } + + for _, tt := range tests { + t.Run(string(tt.state), func(t *testing.T) { + l := New(WithInitialState(tt.state)) + got := l.LegacyMergeStatus() + if got != tt.expected { + t.Errorf("LegacyMergeStatus() for %s = %s, want %s", tt.state, got, tt.expected) + } + }) + } +} + +func TestLegacyMergeStatusFromState(t *testing.T) { + tests := []struct { + state AgentLifecycleState + expected types.MergeStatus + }{ + {StateStarting, types.MergeStatusNone}, + {StateRunning, types.MergeStatusNone}, + {StateQueuedForMerge, types.MergeStatusPending}, + {StateMerging, types.MergeStatusMerging}, + {StateResolving, types.MergeStatusResolving}, + {StateValidating, types.MergeStatusMerged}, + {StateRepairing, types.MergeStatusMerged}, + {StateCompleted, types.MergeStatusMerged}, + {StateFailed, types.MergeStatusFailed}, + {StateMergeFailed, types.MergeStatusFailed}, + {StateNeedsAttention, types.MergeStatusMergedNeedsRepair}, + } + + for _, tt := range tests { + t.Run(string(tt.state), func(t *testing.T) { + got := LegacyMergeStatusFromState(tt.state) + if got != tt.expected { + t.Errorf("LegacyMergeStatusFromState(%s) = %s, want %s", tt.state, got, tt.expected) + } + }) + } +} + +func TestLegacyValidationStatus(t *testing.T) { + tests := []struct { + name string + state AgentLifecycleState + expected string + }{ + {"Starting", StateStarting, ""}, + {"Running", StateRunning, ""}, + {"QueuedForMerge", StateQueuedForMerge, ""}, + {"Merging", StateMerging, ""}, + {"Resolving", StateResolving, ""}, + {"Validating", StateValidating, "running"}, + {"Repairing", StateRepairing, "repairing"}, + {"NeedsAttention", StateNeedsAttention, "failed"}, + {"Failed", StateFailed, ""}, + {"Cancelled", StateCancelled, ""}, + {"TimedOut", StateTimedOut, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(tt.state)) + got := l.LegacyValidationStatus() + if got != tt.expected { + t.Errorf("LegacyValidationStatus() for %s = %q, want %q", tt.state, got, tt.expected) + } + }) + } +} + +func TestLegacyValidationStatusAfterValidation(t *testing.T) { + // Test that Completed returns "passed" if we came from Validating + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + _ = l.Transition(EventValidationPassed, TransitionContext{}) + + if l.State() != StateCompleted { + t.Fatalf("expected Completed, got %s", l.State()) + } + + got := l.LegacyValidationStatus() + if got != "passed" { + t.Errorf("LegacyValidationStatus() after validation = %q, want %q", got, "passed") + } +} + +func TestLegacyValidationStatusWithoutValidation(t *testing.T) { + // Test that Completed returns "" if we didn't go through validation + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: false}) + + if l.State() != StateCompleted { + t.Fatalf("expected Completed, got %s", l.State()) + } + + got := l.LegacyValidationStatus() + if got != "" { + t.Errorf("LegacyValidationStatus() without validation = %q, want %q", got, "") + } +} diff --git a/pkg/lifecycle/state.go b/pkg/lifecycle/state.go new file mode 100644 index 00000000..06654145 --- /dev/null +++ b/pkg/lifecycle/state.go @@ -0,0 +1,399 @@ +// Package lifecycle provides a state machine for managing agent lifecycle states. +// +// The lifecycle state machine replaces the previous multi-field status model +// (Status, MergeStatus, ValidationStatus) with a single authoritative state. +// This eliminates race conditions and invalid state combinations. +// +// See docs/design/agent-lifecycle-state-machine.md for the full design. +package lifecycle + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// maxHistoryEntries limits the number of state transitions kept in memory. +const maxHistoryEntries = 100 + +// ErrInvalidTransition indicates an event is not valid for the current state. +var ErrInvalidTransition = errors.New("invalid state transition") + +// AgentLifecycleState represents the current state in the agent lifecycle. +type AgentLifecycleState string + +const ( + StateStarting AgentLifecycleState = "starting" + StateRunning AgentLifecycleState = "running" + StateQueuedForMerge AgentLifecycleState = "queued_for_merge" + StateMerging AgentLifecycleState = "merging" + StateResolving AgentLifecycleState = "resolving" + StateValidating AgentLifecycleState = "validating" + StateRepairing AgentLifecycleState = "repairing" + StateMergeFailed AgentLifecycleState = "merge_failed" + StateCompleted AgentLifecycleState = "completed" + StateFailed AgentLifecycleState = "failed" + StateNeedsAttention AgentLifecycleState = "needs_attention" + StateCancelled AgentLifecycleState = "cancelled" + StateTimedOut AgentLifecycleState = "timed_out" +) + +// String returns the string representation of the state. +func (s AgentLifecycleState) String() string { + return string(s) +} + +// IsTerminal returns true if this state is terminal (no further transitions possible). +func (s AgentLifecycleState) IsTerminal() bool { + switch s { + case StateCompleted, StateFailed, StateNeedsAttention, StateCancelled, StateTimedOut: + return true + } + return false +} + +// AgentEvent represents an event that can trigger a state transition. +type AgentEvent string + +const ( + EventAgentSpawned AgentEvent = "agent_spawned" + EventWorkComplete AgentEvent = "work_complete" + EventWorkFailed AgentEvent = "work_failed" + EventMergeStarted AgentEvent = "merge_started" + EventMergeSuccess AgentEvent = "merge_success" + EventMergeConflict AgentEvent = "merge_conflict" + EventMergeFailed AgentEvent = "merge_failed" + EventResolveSuccess AgentEvent = "resolve_success" + EventResolveFailed AgentEvent = "resolve_failed" + EventValidationPassed AgentEvent = "validation_passed" + EventValidationFailed AgentEvent = "validation_failed" + EventValidationSkipped AgentEvent = "validation_skipped" + EventRepairComplete AgentEvent = "repair_complete" + EventRetry AgentEvent = "retry" + EventRetriesExhausted AgentEvent = "retries_exhausted" + EventCancel AgentEvent = "cancel" + EventTimeout AgentEvent = "timeout" +) + +// String returns the string representation of the event. +func (e AgentEvent) String() string { + return string(e) +} + +// TransitionContext provides context for state transitions. +// This struct carries information needed to determine the correct target state. +type TransitionContext struct { + // ValidationEnabled indicates whether post-merge validation is configured. + ValidationEnabled bool + // RepairEnabled indicates whether automated repair is enabled. + RepairEnabled bool + // StrictMode indicates whether validation failures should cause failure (vs needs_attention). + StrictMode bool + // AttemptsRemaining tracks remaining retry attempts for work failures. + AttemptsRemaining int + // RepairAttempt is the current repair attempt number (1-indexed). + RepairAttempt int + // MaxRepairAttempts is the maximum number of repair attempts allowed. + MaxRepairAttempts int + // QueuePosition is the agent's position in the merge queue. + QueuePosition int + // Error is an optional error message associated with the transition. + Error string +} + +// StateTransition records a state change. +type StateTransition struct { + From AgentLifecycleState + To AgentLifecycleState + Event AgentEvent + Timestamp time.Time + Context TransitionContext +} + +// TransitionCallback is called after a successful state transition. +// The callback receives the old state, new state, and the event that triggered the transition. +type TransitionCallback func(from, to AgentLifecycleState, event AgentEvent) + +// TerminalCallback is called when a terminal state is reached. +// The callback receives the agent ID associated with this lifecycle. +// This is useful for cleanup operations that must occur regardless of +// how the terminal state was reached (normal completion, cancellation, timeout, etc.). +type TerminalCallback func(agentID string) + +// AgentLifecycle manages state transitions for a single agent. +// All methods are thread-safe. +type AgentLifecycle struct { + mu sync.RWMutex + agentID string // Agent ID for terminal callback + state AgentLifecycleState + stateHistory []StateTransition + context TransitionContext + onTransition TransitionCallback + onTerminal TerminalCallback +} + +// Option configures an AgentLifecycle instance. +type Option func(*AgentLifecycle) + +// WithTransitionCallback sets a callback that fires after each state transition. +// The callback is invoked while holding the lock, so it should be fast and non-blocking. +func WithTransitionCallback(cb TransitionCallback) Option { + return func(l *AgentLifecycle) { + l.onTransition = cb + } +} + +// WithTerminalCallback sets a callback that fires when a terminal state is reached. +// The callback is invoked while holding the lock, so it should be fast and non-blocking. +// This is useful for cleanup operations (e.g., overlay cleanup) that must occur +// regardless of how the terminal state was reached. +func WithTerminalCallback(agentID string, cb TerminalCallback) Option { + return func(l *AgentLifecycle) { + l.agentID = agentID + l.onTerminal = cb + } +} + +// WithInitialState sets the initial state (defaults to StateStarting). +func WithInitialState(state AgentLifecycleState) Option { + return func(l *AgentLifecycle) { + l.state = state + } +} + +// New creates a new AgentLifecycle starting in the StateStarting state. +func New(opts ...Option) *AgentLifecycle { + l := &AgentLifecycle{ + state: StateStarting, + stateHistory: make([]StateTransition, 0, 16), + } + for _, opt := range opts { + opt(l) + } + return l +} + +// State returns the current state (thread-safe). +func (l *AgentLifecycle) State() AgentLifecycleState { + l.mu.RLock() + defer l.mu.RUnlock() + return l.state +} + +// Context returns the current transition context (thread-safe). +func (l *AgentLifecycle) Context() TransitionContext { + l.mu.RLock() + defer l.mu.RUnlock() + return l.context +} + +// IsTerminal returns true if the current state is terminal (thread-safe). +func (l *AgentLifecycle) IsTerminal() bool { + return l.State().IsTerminal() +} + +// History returns a copy of the state transition history (thread-safe). +func (l *AgentLifecycle) History() []StateTransition { + l.mu.RLock() + defer l.mu.RUnlock() + result := make([]StateTransition, len(l.stateHistory)) + copy(result, l.stateHistory) + return result +} + +// SetState forcibly sets the lifecycle to a specific state, bypassing normal transition rules. +// This should only be used for recovery when events arrive out of order and the lifecycle +// becomes stuck in an intermediate state while the merge status indicates completion. +// The event parameter describes why the state was forced (for history tracking). +// Returns the previous state. +func (l *AgentLifecycle) SetState(newState AgentLifecycleState, event AgentEvent, ctx TransitionContext) AgentLifecycleState { + l.mu.Lock() + defer l.mu.Unlock() + + oldState := l.state + if oldState == newState { + return oldState + } + + l.state = newState + l.context = ctx + + // Record the forced transition in history + transition := StateTransition{ + From: oldState, + To: newState, + Event: event, + Timestamp: time.Now(), + Context: ctx, + } + l.stateHistory = append(l.stateHistory, transition) + + // Trim history if it exceeds the limit + if len(l.stateHistory) > maxHistoryEntries { + l.stateHistory = l.stateHistory[len(l.stateHistory)-maxHistoryEntries:] + } + + // Fire callback if registered + if l.onTransition != nil { + l.onTransition(oldState, newState, event) + } + + // Fire terminal callback if we transitioned to a terminal state + if newState.IsTerminal() && l.onTerminal != nil { + l.onTerminal(l.agentID) + } + + return oldState +} + +// Transition attempts to transition to a new state based on an event. +// Returns an error if the transition is invalid for the current state. +// The context provides additional information needed to determine the target state. +func (l *AgentLifecycle) Transition(event AgentEvent, ctx TransitionContext) error { + l.mu.Lock() + defer l.mu.Unlock() + + newState, err := l.nextState(l.state, event, ctx) + if err != nil { + return fmt.Errorf("%w: %s + %s", ErrInvalidTransition, l.state, event) + } + + oldState := l.state + l.state = newState + l.context = ctx + + // Record transition in history, maintaining bounded size + transition := StateTransition{ + From: oldState, + To: newState, + Event: event, + Timestamp: time.Now(), + Context: ctx, + } + l.stateHistory = append(l.stateHistory, transition) + + // Trim history if it exceeds the limit + if len(l.stateHistory) > maxHistoryEntries { + // Keep the last maxHistoryEntries entries + l.stateHistory = l.stateHistory[len(l.stateHistory)-maxHistoryEntries:] + } + + // Fire callback if registered (still holding lock for atomicity) + if l.onTransition != nil { + l.onTransition(oldState, newState, event) + } + + // Fire terminal callback if we transitioned to a terminal state + if newState.IsTerminal() && l.onTerminal != nil { + l.onTerminal(l.agentID) + } + + return nil +} + +// nextState determines the target state for a given current state and event. +// Returns an error if the transition is invalid. +func (l *AgentLifecycle) nextState(current AgentLifecycleState, event AgentEvent, ctx TransitionContext) (AgentLifecycleState, error) { + // Global events that can fire from any non-terminal state + if !current.IsTerminal() { + switch event { + case EventCancel: + return StateCancelled, nil + case EventTimeout: + return StateTimedOut, nil + } + } + + // State-specific transitions + switch current { + case StateStarting: + switch event { + case EventAgentSpawned: + return StateRunning, nil + } + + case StateRunning: + switch event { + case EventWorkComplete: + return StateQueuedForMerge, nil + case EventWorkFailed: + if ctx.AttemptsRemaining > 0 { + return StateRunning, nil + } + return StateFailed, nil + } + + case StateQueuedForMerge: + switch event { + case EventMergeStarted: + return StateMerging, nil + } + + case StateMerging: + switch event { + case EventMergeSuccess: + if ctx.ValidationEnabled { + return StateValidating, nil + } + return StateCompleted, nil + case EventMergeConflict: + return StateResolving, nil + case EventMergeFailed: + return StateMergeFailed, nil + } + + case StateResolving: + switch event { + case EventResolveSuccess: + if ctx.ValidationEnabled { + return StateValidating, nil + } + return StateCompleted, nil + case EventResolveFailed: + return StateMergeFailed, nil + } + + case StateValidating: + switch event { + case EventValidationPassed: + return StateCompleted, nil + case EventValidationFailed: + // Check if repair is possible + if ctx.RepairEnabled && ctx.RepairAttempt < ctx.MaxRepairAttempts { + return StateRepairing, nil + } + // Repair exhausted or disabled + if ctx.StrictMode { + return StateFailed, nil + } + return StateNeedsAttention, nil + case EventValidationSkipped: + return StateCompleted, nil + } + + case StateRepairing: + switch event { + case EventRepairComplete: + // After repair completes, re-run validation + return StateValidating, nil + } + + case StateMergeFailed: + switch event { + case EventRetry: + if ctx.AttemptsRemaining > 0 { + return StateRunning, nil + } + return StateFailed, nil + case EventRetriesExhausted: + return StateFailed, nil + } + + // Terminal states - no transitions allowed (except global cancel/timeout already handled) + case StateCompleted, StateFailed, StateNeedsAttention, StateCancelled, StateTimedOut: + // No valid transitions from terminal states + } + + return "", fmt.Errorf("no transition defined for state %s with event %s", current, event) +} diff --git a/pkg/lifecycle/state_test.go b/pkg/lifecycle/state_test.go new file mode 100644 index 00000000..c90ce854 --- /dev/null +++ b/pkg/lifecycle/state_test.go @@ -0,0 +1,757 @@ +package lifecycle + +import ( + "testing" +) + +func TestNew(t *testing.T) { + l := New() + if l.State() != StateStarting { + t.Errorf("expected initial state %s, got %s", StateStarting, l.State()) + } + if l.IsTerminal() { + t.Error("initial state should not be terminal") + } +} + +func TestNewWithInitialState(t *testing.T) { + l := New(WithInitialState(StateRunning)) + if l.State() != StateRunning { + t.Errorf("expected initial state %s, got %s", StateRunning, l.State()) + } +} + +func TestBasicWorkflow(t *testing.T) { + l := New() + + // Starting -> Running + err := l.Transition(EventAgentSpawned, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Running failed: %v", err) + } + if l.State() != StateRunning { + t.Errorf("expected %s, got %s", StateRunning, l.State()) + } + + // Running -> QueuedForMerge + err = l.Transition(EventWorkComplete, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to QueuedForMerge failed: %v", err) + } + if l.State() != StateQueuedForMerge { + t.Errorf("expected %s, got %s", StateQueuedForMerge, l.State()) + } + + // QueuedForMerge -> Merging + err = l.Transition(EventMergeStarted, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Merging failed: %v", err) + } + if l.State() != StateMerging { + t.Errorf("expected %s, got %s", StateMerging, l.State()) + } + + // Merging -> Completed (no validation) + err = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: false}) + if err != nil { + t.Fatalf("Transition to Completed failed: %v", err) + } + if l.State() != StateCompleted { + t.Errorf("expected %s, got %s", StateCompleted, l.State()) + } + if !l.IsTerminal() { + t.Error("Completed should be terminal") + } +} + +func TestValidationWorkflow(t *testing.T) { + l := New() + + // Fast forward to Merging + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + + // Merging -> Validating (with validation enabled) + err := l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + if err != nil { + t.Fatalf("Transition to Validating failed: %v", err) + } + if l.State() != StateValidating { + t.Errorf("expected %s, got %s", StateValidating, l.State()) + } + + // Validating -> Completed + err = l.Transition(EventValidationPassed, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Completed failed: %v", err) + } + if l.State() != StateCompleted { + t.Errorf("expected %s, got %s", StateCompleted, l.State()) + } +} + +func TestRepairWorkflow(t *testing.T) { + l := New() + + // Fast forward to Validating + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + + // Validating -> Repairing (repair enabled, attempts remaining) + ctx := TransitionContext{ + RepairEnabled: true, + RepairAttempt: 0, + MaxRepairAttempts: 3, + } + err := l.Transition(EventValidationFailed, ctx) + if err != nil { + t.Fatalf("Transition to Repairing failed: %v", err) + } + if l.State() != StateRepairing { + t.Errorf("expected %s, got %s", StateRepairing, l.State()) + } + + // Repairing -> Validating + err = l.Transition(EventRepairComplete, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Validating failed: %v", err) + } + if l.State() != StateValidating { + t.Errorf("expected %s, got %s", StateValidating, l.State()) + } + + // Validating -> Completed + err = l.Transition(EventValidationPassed, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Completed failed: %v", err) + } + if l.State() != StateCompleted { + t.Errorf("expected %s, got %s", StateCompleted, l.State()) + } +} + +func TestRepairExhausted(t *testing.T) { + l := New() + + // Fast forward to Validating + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + + // Validating -> NeedsAttention (repair exhausted, lenient mode) + ctx := TransitionContext{ + RepairEnabled: true, + RepairAttempt: 3, + MaxRepairAttempts: 3, + StrictMode: false, + } + err := l.Transition(EventValidationFailed, ctx) + if err != nil { + t.Fatalf("Transition to NeedsAttention failed: %v", err) + } + if l.State() != StateNeedsAttention { + t.Errorf("expected %s, got %s", StateNeedsAttention, l.State()) + } +} + +func TestStrictModeValidationFailure(t *testing.T) { + l := New() + + // Fast forward to Validating + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + + // Validating -> Failed (strict mode, repair exhausted) + ctx := TransitionContext{ + RepairEnabled: true, + RepairAttempt: 3, + MaxRepairAttempts: 3, + StrictMode: true, + } + err := l.Transition(EventValidationFailed, ctx) + if err != nil { + t.Fatalf("Transition to Failed failed: %v", err) + } + if l.State() != StateFailed { + t.Errorf("expected %s, got %s", StateFailed, l.State()) + } +} + +func TestConflictResolution(t *testing.T) { + l := New() + + // Fast forward to Merging + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + + // Merging -> Resolving + err := l.Transition(EventMergeConflict, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to Resolving failed: %v", err) + } + if l.State() != StateResolving { + t.Errorf("expected %s, got %s", StateResolving, l.State()) + } + + // Resolving -> Completed (no validation) + err = l.Transition(EventResolveSuccess, TransitionContext{ValidationEnabled: false}) + if err != nil { + t.Fatalf("Transition to Completed failed: %v", err) + } + if l.State() != StateCompleted { + t.Errorf("expected %s, got %s", StateCompleted, l.State()) + } +} + +func TestMergeFailedRetry(t *testing.T) { + l := New() + + // Fast forward to Merging + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + + // Merging -> MergeFailed + err := l.Transition(EventMergeFailed, TransitionContext{}) + if err != nil { + t.Fatalf("Transition to MergeFailed failed: %v", err) + } + if l.State() != StateMergeFailed { + t.Errorf("expected %s, got %s", StateMergeFailed, l.State()) + } + + // MergeFailed -> Running (retry with attempts remaining) + err = l.Transition(EventRetry, TransitionContext{AttemptsRemaining: 2}) + if err != nil { + t.Fatalf("Transition to Running failed: %v", err) + } + if l.State() != StateRunning { + t.Errorf("expected %s, got %s", StateRunning, l.State()) + } +} + +func TestCancelFromAnyState(t *testing.T) { + states := []AgentLifecycleState{ + StateStarting, + StateRunning, + StateQueuedForMerge, + StateMerging, + StateResolving, + StateValidating, + StateRepairing, + StateMergeFailed, + } + + for _, state := range states { + l := New(WithInitialState(state)) + err := l.Transition(EventCancel, TransitionContext{}) + if err != nil { + t.Errorf("Cancel from %s should succeed, got error: %v", state, err) + } + if l.State() != StateCancelled { + t.Errorf("Cancel from %s should result in Cancelled, got %s", state, l.State()) + } + } +} + +func TestTimeoutFromAnyState(t *testing.T) { + states := []AgentLifecycleState{ + StateStarting, + StateRunning, + StateQueuedForMerge, + StateMerging, + StateResolving, + StateValidating, + StateRepairing, + StateMergeFailed, + } + + for _, state := range states { + l := New(WithInitialState(state)) + err := l.Transition(EventTimeout, TransitionContext{}) + if err != nil { + t.Errorf("Timeout from %s should succeed, got error: %v", state, err) + } + if l.State() != StateTimedOut { + t.Errorf("Timeout from %s should result in TimedOut, got %s", state, l.State()) + } + } +} + +func TestTerminalStatesRejectTransitions(t *testing.T) { + terminalStates := []AgentLifecycleState{ + StateCompleted, + StateFailed, + StateNeedsAttention, + StateCancelled, + StateTimedOut, + } + + events := []AgentEvent{ + EventAgentSpawned, + EventWorkComplete, + EventMergeStarted, + EventCancel, + EventTimeout, + } + + for _, state := range terminalStates { + for _, event := range events { + l := New(WithInitialState(state)) + err := l.Transition(event, TransitionContext{}) + if err == nil { + t.Errorf("Transition %s from terminal state %s should fail", event, state) + } + } + } +} + +func TestInvalidTransitions(t *testing.T) { + tests := []struct { + name string + initial AgentLifecycleState + event AgentEvent + }{ + {"Starting + WorkComplete", StateStarting, EventWorkComplete}, + {"Running + MergeStarted", StateRunning, EventMergeStarted}, + {"QueuedForMerge + WorkComplete", StateQueuedForMerge, EventWorkComplete}, + {"Merging + AgentSpawned", StateMerging, EventAgentSpawned}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(tt.initial)) + err := l.Transition(tt.event, TransitionContext{}) + if err == nil { + t.Errorf("expected error for invalid transition %s + %s", tt.initial, tt.event) + } + }) + } +} + +func TestTransitionCallback(t *testing.T) { + var callbackCalled bool + var fromState, toState AgentLifecycleState + var triggeredEvent AgentEvent + + cb := func(from, to AgentLifecycleState, event AgentEvent) { + callbackCalled = true + fromState = from + toState = to + triggeredEvent = event + } + + l := New(WithTransitionCallback(cb)) + err := l.Transition(EventAgentSpawned, TransitionContext{}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + + if !callbackCalled { + t.Error("callback should have been called") + } + if fromState != StateStarting { + t.Errorf("expected from state %s, got %s", StateStarting, fromState) + } + if toState != StateRunning { + t.Errorf("expected to state %s, got %s", StateRunning, toState) + } + if triggeredEvent != EventAgentSpawned { + t.Errorf("expected event %s, got %s", EventAgentSpawned, triggeredEvent) + } +} + +func TestHistoryBounding(t *testing.T) { + var l *AgentLifecycle + + // Simulate many transitions + // Start with a valid workflow and repeat + for i := 0; i < maxHistoryEntries+50; i++ { + // Create a new lifecycle each time to avoid needing complex valid paths + l = New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + } + + // For a fresh lifecycle with just the spawned transition + for i := 0; i < maxHistoryEntries+10; i++ { + // Reset to starting state by creating new lifecycle + l = New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + } + + // Test history bounding on a single lifecycle + // We'll use cancel transitions since they're allowed from any non-terminal state + // But once cancelled, we can't transition anymore. So let's test differently. + + // Actually, let's test the history limit by directly checking the history slice + // after many transitions that are valid + + l = New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + // Now we're in Running, do work complete/retry cycles + for i := 0; i < maxHistoryEntries+20; i++ { + _ = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 1}) + } + + history := l.History() + if len(history) > maxHistoryEntries { + t.Errorf("history length %d exceeds max %d", len(history), maxHistoryEntries) + } +} + +func TestIsTerminal(t *testing.T) { + tests := []struct { + state AgentLifecycleState + terminal bool + }{ + {StateStarting, false}, + {StateRunning, false}, + {StateQueuedForMerge, false}, + {StateMerging, false}, + {StateResolving, false}, + {StateValidating, false}, + {StateRepairing, false}, + {StateMergeFailed, false}, + {StateCompleted, true}, + {StateFailed, true}, + {StateNeedsAttention, true}, + {StateCancelled, true}, + {StateTimedOut, true}, + } + + for _, tt := range tests { + if tt.state.IsTerminal() != tt.terminal { + t.Errorf("state %s: expected terminal=%v, got %v", tt.state, tt.terminal, tt.state.IsTerminal()) + } + } +} + +func TestWorkFailedWithRetry(t *testing.T) { + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + + // Work failed but attempts remaining - stay in Running + err := l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 2}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if l.State() != StateRunning { + t.Errorf("expected %s, got %s", StateRunning, l.State()) + } + + // Work failed with no attempts - go to Failed + err = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 0}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if l.State() != StateFailed { + t.Errorf("expected %s, got %s", StateFailed, l.State()) + } +} + +func TestValidationSkipped(t *testing.T) { + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: true}) + + // Validation skipped -> Completed + err := l.Transition(EventValidationSkipped, TransitionContext{}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if l.State() != StateCompleted { + t.Errorf("expected %s, got %s", StateCompleted, l.State()) + } +} + +func TestRetriesExhausted(t *testing.T) { + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeFailed, TransitionContext{}) + + // RetriesExhausted -> Failed + err := l.Transition(EventRetriesExhausted, TransitionContext{}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if l.State() != StateFailed { + t.Errorf("expected %s, got %s", StateFailed, l.State()) + } +} + +func TestResolveFailed(t *testing.T) { + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + _ = l.Transition(EventMergeConflict, TransitionContext{}) + + // ResolveFailed -> MergeFailed + err := l.Transition(EventResolveFailed, TransitionContext{}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if l.State() != StateMergeFailed { + t.Errorf("expected %s, got %s", StateMergeFailed, l.State()) + } +} + +func TestContextPreserved(t *testing.T) { + l := New() + ctx := TransitionContext{ + ValidationEnabled: true, + RepairEnabled: true, + StrictMode: false, + AttemptsRemaining: 3, + RepairAttempt: 1, + MaxRepairAttempts: 5, + QueuePosition: 7, + Error: "test error", + } + + _ = l.Transition(EventAgentSpawned, ctx) + got := l.Context() + + if got.ValidationEnabled != ctx.ValidationEnabled { + t.Errorf("ValidationEnabled mismatch") + } + if got.RepairEnabled != ctx.RepairEnabled { + t.Errorf("RepairEnabled mismatch") + } + if got.StrictMode != ctx.StrictMode { + t.Errorf("StrictMode mismatch") + } + if got.AttemptsRemaining != ctx.AttemptsRemaining { + t.Errorf("AttemptsRemaining mismatch") + } + if got.RepairAttempt != ctx.RepairAttempt { + t.Errorf("RepairAttempt mismatch") + } + if got.MaxRepairAttempts != ctx.MaxRepairAttempts { + t.Errorf("MaxRepairAttempts mismatch") + } + if got.QueuePosition != ctx.QueuePosition { + t.Errorf("QueuePosition mismatch") + } + if got.Error != ctx.Error { + t.Errorf("Error mismatch") + } +} + +func TestSetState(t *testing.T) { + t.Run("force transition bypasses state machine", func(t *testing.T) { + l := New() + + // Start at StateStarting + if l.State() != StateStarting { + t.Fatalf("expected StateStarting, got %s", l.State()) + } + + // Force to StateCompleted, skipping all intermediate states + ctx := TransitionContext{} + oldState := l.SetState(StateCompleted, EventMergeSuccess, ctx) + + if oldState != StateStarting { + t.Errorf("expected old state %s, got %s", StateStarting, oldState) + } + if l.State() != StateCompleted { + t.Errorf("expected new state %s, got %s", StateCompleted, l.State()) + } + if !l.IsTerminal() { + t.Error("StateCompleted should be terminal") + } + }) + + t.Run("no-op when setting same state", func(t *testing.T) { + l := New() + + // Force to StateRunning + l.SetState(StateRunning, EventAgentSpawned, TransitionContext{}) + historyLen := len(l.History()) + + // Set to same state - should be no-op + oldState := l.SetState(StateRunning, EventAgentSpawned, TransitionContext{}) + + if oldState != StateRunning { + t.Errorf("expected old state %s, got %s", StateRunning, oldState) + } + if len(l.History()) != historyLen { + t.Error("history should not change when setting same state") + } + }) + + t.Run("records forced transition in history", func(t *testing.T) { + l := New() + + // Progress normally to QueuedForMerge + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + + if l.State() != StateQueuedForMerge { + t.Fatalf("expected StateQueuedForMerge, got %s", l.State()) + } + initialHistoryLen := len(l.History()) + + // Force to StateCompleted (simulating recovery) + ctx := TransitionContext{Error: "recovered from stuck state"} + l.SetState(StateCompleted, EventMergeSuccess, ctx) + + history := l.History() + if len(history) != initialHistoryLen+1 { + t.Errorf("expected %d history entries, got %d", initialHistoryLen+1, len(history)) + } + + lastEntry := history[len(history)-1] + if lastEntry.From != StateQueuedForMerge { + t.Errorf("expected From=%s, got %s", StateQueuedForMerge, lastEntry.From) + } + if lastEntry.To != StateCompleted { + t.Errorf("expected To=%s, got %s", StateCompleted, lastEntry.To) + } + if lastEntry.Event != EventMergeSuccess { + t.Errorf("expected Event=%s, got %s", EventMergeSuccess, lastEntry.Event) + } + if lastEntry.Context.Error != "recovered from stuck state" { + t.Errorf("expected context.Error to be preserved") + } + }) + + t.Run("fires callback on forced transition", func(t *testing.T) { + var callbackCalled bool + var cbFrom, cbTo AgentLifecycleState + var cbEvent AgentEvent + + l := New(WithTransitionCallback(func(from, to AgentLifecycleState, event AgentEvent) { + callbackCalled = true + cbFrom = from + cbTo = to + cbEvent = event + })) + + l.SetState(StateCompleted, EventMergeSuccess, TransitionContext{}) + + if !callbackCalled { + t.Error("expected callback to be called") + } + if cbFrom != StateStarting { + t.Errorf("expected callback from=%s, got %s", StateStarting, cbFrom) + } + if cbTo != StateCompleted { + t.Errorf("expected callback to=%s, got %s", StateCompleted, cbTo) + } + if cbEvent != EventMergeSuccess { + t.Errorf("expected callback event=%s, got %s", EventMergeSuccess, cbEvent) + } + }) +} + +func TestTerminalCallback(t *testing.T) { + t.Run("fires on transition to terminal state", func(t *testing.T) { + var terminalCalled bool + var terminalAgentID string + + l := New( + WithTerminalCallback("agent-123", func(agentID string) { + terminalCalled = true + terminalAgentID = agentID + }), + ) + + // Transition to running (not terminal) + err := l.Transition(EventAgentSpawned, TransitionContext{}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if terminalCalled { + t.Error("terminal callback should not be called for non-terminal state") + } + + // Transition to failed (terminal) + err = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 0}) + if err != nil { + t.Fatalf("Transition failed: %v", err) + } + if !terminalCalled { + t.Error("terminal callback should be called for terminal state") + } + if terminalAgentID != "agent-123" { + t.Errorf("expected agentID 'agent-123', got '%s'", terminalAgentID) + } + }) + + t.Run("fires on SetState to terminal state", func(t *testing.T) { + var terminalCalled bool + var terminalAgentID string + + l := New( + WithTerminalCallback("agent-456", func(agentID string) { + terminalCalled = true + terminalAgentID = agentID + }), + ) + + // Force transition to completed (terminal) + l.SetState(StateCompleted, EventMergeSuccess, TransitionContext{}) + + if !terminalCalled { + t.Error("terminal callback should be called for terminal state via SetState") + } + if terminalAgentID != "agent-456" { + t.Errorf("expected agentID 'agent-456', got '%s'", terminalAgentID) + } + }) + + t.Run("does not fire for non-terminal SetState", func(t *testing.T) { + var terminalCalled bool + + l := New( + WithTerminalCallback("agent-789", func(agentID string) { + terminalCalled = true + }), + ) + + // Force transition to running (not terminal) + l.SetState(StateRunning, EventAgentSpawned, TransitionContext{}) + + if terminalCalled { + t.Error("terminal callback should not be called for non-terminal state") + } + }) + + t.Run("fires for all terminal states", func(t *testing.T) { + terminalStates := []AgentLifecycleState{ + StateCompleted, + StateFailed, + StateNeedsAttention, + StateCancelled, + StateTimedOut, + } + + for _, termState := range terminalStates { + var terminalCalled bool + + l := New( + WithTerminalCallback("test-agent", func(agentID string) { + terminalCalled = true + }), + ) + + l.SetState(termState, EventCancel, TransitionContext{}) + + if !terminalCalled { + t.Errorf("terminal callback should be called for state %s", termState) + } + } + }) +} diff --git a/pkg/lifecycle/transitions_test.go b/pkg/lifecycle/transitions_test.go new file mode 100644 index 00000000..41abaa06 --- /dev/null +++ b/pkg/lifecycle/transitions_test.go @@ -0,0 +1,1082 @@ +package lifecycle + +import ( + "errors" + "sync" + "sync/atomic" + "testing" +) + +// nonTerminalStates are states that allow transitions out. +var nonTerminalStates = []AgentLifecycleState{ + StateStarting, + StateRunning, + StateQueuedForMerge, + StateMerging, + StateResolving, + StateValidating, + StateRepairing, + StateMergeFailed, +} + +// terminalStates are states with no outbound transitions. +var terminalStates = []AgentLifecycleState{ + StateCompleted, + StateFailed, + StateNeedsAttention, + StateCancelled, + StateTimedOut, +} + +// allEvents lists all defined events for exhaustive testing. +var allEvents = []AgentEvent{ + EventAgentSpawned, + EventWorkComplete, + EventWorkFailed, + EventMergeStarted, + EventMergeSuccess, + EventMergeConflict, + EventMergeFailed, + EventResolveSuccess, + EventResolveFailed, + EventValidationPassed, + EventValidationFailed, + EventValidationSkipped, + EventRepairComplete, + EventRetry, + EventRetriesExhausted, + EventCancel, + EventTimeout, +} + +// transitionTest defines a test case for a state transition. +type transitionTest struct { + name string + fromState AgentLifecycleState + event AgentEvent + ctx TransitionContext + expectedState AgentLifecycleState +} + +// TestAllValidTransitions exhaustively tests all valid state transitions. +// This covers the complete transition table from the state machine design. +func TestAllValidTransitions(t *testing.T) { + tests := []transitionTest{ + // Starting state transitions + { + name: "Starting + AgentSpawned -> Running", + fromState: StateStarting, + event: EventAgentSpawned, + expectedState: StateRunning, + }, + { + name: "Starting + Cancel -> Cancelled", + fromState: StateStarting, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Starting + Timeout -> TimedOut", + fromState: StateStarting, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // Running state transitions + { + name: "Running + WorkComplete -> QueuedForMerge", + fromState: StateRunning, + event: EventWorkComplete, + expectedState: StateQueuedForMerge, + }, + { + name: "Running + WorkFailed (retries) -> Running", + fromState: StateRunning, + event: EventWorkFailed, + ctx: TransitionContext{AttemptsRemaining: 2}, + expectedState: StateRunning, + }, + { + name: "Running + WorkFailed (no retries) -> Failed", + fromState: StateRunning, + event: EventWorkFailed, + ctx: TransitionContext{AttemptsRemaining: 0}, + expectedState: StateFailed, + }, + { + name: "Running + Cancel -> Cancelled", + fromState: StateRunning, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Running + Timeout -> TimedOut", + fromState: StateRunning, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // QueuedForMerge state transitions + { + name: "QueuedForMerge + MergeStarted -> Merging", + fromState: StateQueuedForMerge, + event: EventMergeStarted, + expectedState: StateMerging, + }, + { + name: "QueuedForMerge + Cancel -> Cancelled", + fromState: StateQueuedForMerge, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "QueuedForMerge + Timeout -> TimedOut", + fromState: StateQueuedForMerge, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // Merging state transitions + { + name: "Merging + MergeSuccess (no validation) -> Completed", + fromState: StateMerging, + event: EventMergeSuccess, + ctx: TransitionContext{ValidationEnabled: false}, + expectedState: StateCompleted, + }, + { + name: "Merging + MergeSuccess (with validation) -> Validating", + fromState: StateMerging, + event: EventMergeSuccess, + ctx: TransitionContext{ValidationEnabled: true}, + expectedState: StateValidating, + }, + { + name: "Merging + MergeConflict -> Resolving", + fromState: StateMerging, + event: EventMergeConflict, + expectedState: StateResolving, + }, + { + name: "Merging + MergeFailed -> MergeFailed", + fromState: StateMerging, + event: EventMergeFailed, + expectedState: StateMergeFailed, + }, + { + name: "Merging + Cancel -> Cancelled", + fromState: StateMerging, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Merging + Timeout -> TimedOut", + fromState: StateMerging, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // Resolving state transitions + { + name: "Resolving + ResolveSuccess (no validation) -> Completed", + fromState: StateResolving, + event: EventResolveSuccess, + ctx: TransitionContext{ValidationEnabled: false}, + expectedState: StateCompleted, + }, + { + name: "Resolving + ResolveSuccess (with validation) -> Validating", + fromState: StateResolving, + event: EventResolveSuccess, + ctx: TransitionContext{ValidationEnabled: true}, + expectedState: StateValidating, + }, + { + name: "Resolving + ResolveFailed -> MergeFailed", + fromState: StateResolving, + event: EventResolveFailed, + expectedState: StateMergeFailed, + }, + { + name: "Resolving + Cancel -> Cancelled", + fromState: StateResolving, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Resolving + Timeout -> TimedOut", + fromState: StateResolving, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // Validating state transitions + { + name: "Validating + ValidationPassed -> Completed", + fromState: StateValidating, + event: EventValidationPassed, + expectedState: StateCompleted, + }, + { + name: "Validating + ValidationFailed (repair enabled, attempts left) -> Repairing", + fromState: StateValidating, + event: EventValidationFailed, + ctx: TransitionContext{RepairEnabled: true, RepairAttempt: 0, MaxRepairAttempts: 3}, + expectedState: StateRepairing, + }, + { + name: "Validating + ValidationFailed (repair enabled, at max, strict) -> Failed", + fromState: StateValidating, + event: EventValidationFailed, + ctx: TransitionContext{RepairEnabled: true, RepairAttempt: 3, MaxRepairAttempts: 3, StrictMode: true}, + expectedState: StateFailed, + }, + { + name: "Validating + ValidationFailed (repair enabled, at max, lenient) -> NeedsAttention", + fromState: StateValidating, + event: EventValidationFailed, + ctx: TransitionContext{RepairEnabled: true, RepairAttempt: 3, MaxRepairAttempts: 3, StrictMode: false}, + expectedState: StateNeedsAttention, + }, + { + name: "Validating + ValidationFailed (repair disabled, strict) -> Failed", + fromState: StateValidating, + event: EventValidationFailed, + ctx: TransitionContext{RepairEnabled: false, StrictMode: true}, + expectedState: StateFailed, + }, + { + name: "Validating + ValidationFailed (repair disabled, lenient) -> NeedsAttention", + fromState: StateValidating, + event: EventValidationFailed, + ctx: TransitionContext{RepairEnabled: false, StrictMode: false}, + expectedState: StateNeedsAttention, + }, + { + name: "Validating + ValidationSkipped -> Completed", + fromState: StateValidating, + event: EventValidationSkipped, + expectedState: StateCompleted, + }, + { + name: "Validating + Cancel -> Cancelled", + fromState: StateValidating, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Validating + Timeout -> TimedOut", + fromState: StateValidating, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // Repairing state transitions + { + name: "Repairing + RepairComplete -> Validating", + fromState: StateRepairing, + event: EventRepairComplete, + expectedState: StateValidating, + }, + { + name: "Repairing + Cancel -> Cancelled", + fromState: StateRepairing, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "Repairing + Timeout -> TimedOut", + fromState: StateRepairing, + event: EventTimeout, + expectedState: StateTimedOut, + }, + + // MergeFailed state transitions + { + name: "MergeFailed + Retry (attempts left) -> Running", + fromState: StateMergeFailed, + event: EventRetry, + ctx: TransitionContext{AttemptsRemaining: 1}, + expectedState: StateRunning, + }, + { + name: "MergeFailed + Retry (no attempts) -> Failed", + fromState: StateMergeFailed, + event: EventRetry, + ctx: TransitionContext{AttemptsRemaining: 0}, + expectedState: StateFailed, + }, + { + name: "MergeFailed + RetriesExhausted -> Failed", + fromState: StateMergeFailed, + event: EventRetriesExhausted, + expectedState: StateFailed, + }, + { + name: "MergeFailed + Cancel -> Cancelled", + fromState: StateMergeFailed, + event: EventCancel, + expectedState: StateCancelled, + }, + { + name: "MergeFailed + Timeout -> TimedOut", + fromState: StateMergeFailed, + event: EventTimeout, + expectedState: StateTimedOut, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(tt.fromState)) + err := l.Transition(tt.event, tt.ctx) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("Transition() resulted in state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// TestAllInvalidTransitionsFromNonTerminalStates tests that invalid events +// are rejected from non-terminal states. +func TestAllInvalidTransitionsFromNonTerminalStates(t *testing.T) { + // Define which events are valid for each non-terminal state + validEvents := map[AgentLifecycleState][]AgentEvent{ + StateStarting: {EventAgentSpawned, EventCancel, EventTimeout}, + StateRunning: {EventWorkComplete, EventWorkFailed, EventCancel, EventTimeout}, + StateQueuedForMerge: {EventMergeStarted, EventCancel, EventTimeout}, + StateMerging: {EventMergeSuccess, EventMergeConflict, EventMergeFailed, EventCancel, EventTimeout}, + StateResolving: {EventResolveSuccess, EventResolveFailed, EventCancel, EventTimeout}, + StateValidating: {EventValidationPassed, EventValidationFailed, EventValidationSkipped, EventCancel, EventTimeout}, + StateRepairing: {EventRepairComplete, EventCancel, EventTimeout}, + StateMergeFailed: {EventRetry, EventRetriesExhausted, EventCancel, EventTimeout}, + } + + for _, state := range nonTerminalStates { + valid := validEvents[state] + validSet := make(map[AgentEvent]bool) + for _, e := range valid { + validSet[e] = true + } + + for _, event := range allEvents { + if validSet[event] { + continue // skip valid events + } + + t.Run(string(state)+"+"+string(event), func(t *testing.T) { + l := New(WithInitialState(state)) + err := l.Transition(event, TransitionContext{}) + + if err == nil { + t.Errorf("Transition(%s, %s) should have returned error, got state %s", + state, event, l.State()) + } + + if !errors.Is(err, ErrInvalidTransition) { + t.Errorf("error should wrap ErrInvalidTransition, got: %v", err) + } + + // State should be unchanged + if l.State() != state { + t.Errorf("state changed despite error: got %s, want %s", l.State(), state) + } + }) + } + } +} + +// TestAllTransitionsFromTerminalStatesRejected verifies that no transitions +// are allowed from terminal states. +func TestAllTransitionsFromTerminalStatesRejected(t *testing.T) { + for _, state := range terminalStates { + for _, event := range allEvents { + t.Run(string(state)+"+"+string(event), func(t *testing.T) { + l := New(WithInitialState(state)) + err := l.Transition(event, TransitionContext{}) + + if err == nil { + t.Errorf("Transition(%s, %s) from terminal state should fail", state, event) + } + + if !errors.Is(err, ErrInvalidTransition) { + t.Errorf("error should wrap ErrInvalidTransition, got: %v", err) + } + + // State must remain unchanged + if l.State() != state { + t.Errorf("terminal state changed: got %s, want %s", l.State(), state) + } + }) + } + } +} + +// TestCancelFromAllNonTerminalStates verifies Cancel works from every non-terminal state. +func TestCancelFromAllNonTerminalStates(t *testing.T) { + for _, state := range nonTerminalStates { + t.Run(string(state), func(t *testing.T) { + l := New(WithInitialState(state)) + err := l.Transition(EventCancel, TransitionContext{}) + + if err != nil { + t.Errorf("Cancel from %s should succeed, got error: %v", state, err) + } + + if l.State() != StateCancelled { + t.Errorf("Cancel should result in Cancelled, got %s", l.State()) + } + }) + } +} + +// TestTimeoutFromAllNonTerminalStates verifies Timeout works from every non-terminal state. +func TestTimeoutFromAllNonTerminalStates(t *testing.T) { + for _, state := range nonTerminalStates { + t.Run(string(state), func(t *testing.T) { + l := New(WithInitialState(state)) + err := l.Transition(EventTimeout, TransitionContext{}) + + if err != nil { + t.Errorf("Timeout from %s should succeed, got error: %v", state, err) + } + + if l.State() != StateTimedOut { + t.Errorf("Timeout should result in TimedOut, got %s", l.State()) + } + }) + } +} + +// TestValidationFailedBranchCoverage tests all branches of the ValidationFailed handling. +func TestValidationFailedBranchCoverage(t *testing.T) { + tests := []struct { + name string + ctx TransitionContext + expectedState AgentLifecycleState + }{ + { + name: "repair enabled, attempt 0 of 3 -> Repairing", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 0, + MaxRepairAttempts: 3, + }, + expectedState: StateRepairing, + }, + { + name: "repair enabled, attempt 1 of 3 -> Repairing", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 1, + MaxRepairAttempts: 3, + }, + expectedState: StateRepairing, + }, + { + name: "repair enabled, attempt 2 of 3 -> Repairing", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 2, + MaxRepairAttempts: 3, + }, + expectedState: StateRepairing, + }, + { + name: "repair enabled, attempt 3 of 3, strict -> Failed", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 3, + MaxRepairAttempts: 3, + StrictMode: true, + }, + expectedState: StateFailed, + }, + { + name: "repair enabled, attempt 3 of 3, lenient -> NeedsAttention", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 3, + MaxRepairAttempts: 3, + StrictMode: false, + }, + expectedState: StateNeedsAttention, + }, + { + name: "repair enabled, attempt 5 of 3 (exceeded), strict -> Failed", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 5, + MaxRepairAttempts: 3, + StrictMode: true, + }, + expectedState: StateFailed, + }, + { + name: "repair disabled, strict -> Failed", + ctx: TransitionContext{ + RepairEnabled: false, + StrictMode: true, + }, + expectedState: StateFailed, + }, + { + name: "repair disabled, lenient -> NeedsAttention", + ctx: TransitionContext{ + RepairEnabled: false, + StrictMode: false, + }, + expectedState: StateNeedsAttention, + }, + { + name: "max repair attempts = 0, strict -> Failed", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 0, + MaxRepairAttempts: 0, + StrictMode: true, + }, + expectedState: StateFailed, + }, + { + name: "max repair attempts = 0, lenient -> NeedsAttention", + ctx: TransitionContext{ + RepairEnabled: true, + RepairAttempt: 0, + MaxRepairAttempts: 0, + StrictMode: false, + }, + expectedState: StateNeedsAttention, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(StateValidating)) + err := l.Transition(EventValidationFailed, tt.ctx) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("got state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// TestWorkFailedBranchCoverage tests all branches of WorkFailed handling. +func TestWorkFailedBranchCoverage(t *testing.T) { + tests := []struct { + name string + attemptsRemaining int + expectedState AgentLifecycleState + }{ + {"attempts = 0 -> Failed", 0, StateFailed}, + {"attempts = 1 -> Running", 1, StateRunning}, + {"attempts = 10 -> Running", 10, StateRunning}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(StateRunning)) + err := l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: tt.attemptsRemaining}) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("got state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// TestRetryBranchCoverage tests all branches of Retry handling from MergeFailed. +func TestRetryBranchCoverage(t *testing.T) { + tests := []struct { + name string + attemptsRemaining int + expectedState AgentLifecycleState + }{ + {"attempts = 0 -> Failed", 0, StateFailed}, + {"attempts = 1 -> Running", 1, StateRunning}, + {"attempts = 5 -> Running", 5, StateRunning}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(StateMergeFailed)) + err := l.Transition(EventRetry, TransitionContext{AttemptsRemaining: tt.attemptsRemaining}) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("got state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// TestMergeSuccessBranchCoverage tests ValidationEnabled branching. +func TestMergeSuccessBranchCoverage(t *testing.T) { + tests := []struct { + name string + validationEnabled bool + expectedState AgentLifecycleState + }{ + {"validation disabled -> Completed", false, StateCompleted}, + {"validation enabled -> Validating", true, StateValidating}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(StateMerging)) + err := l.Transition(EventMergeSuccess, TransitionContext{ValidationEnabled: tt.validationEnabled}) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("got state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// TestResolveSuccessBranchCoverage tests ValidationEnabled branching after resolve. +func TestResolveSuccessBranchCoverage(t *testing.T) { + tests := []struct { + name string + validationEnabled bool + expectedState AgentLifecycleState + }{ + {"validation disabled -> Completed", false, StateCompleted}, + {"validation enabled -> Validating", true, StateValidating}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := New(WithInitialState(StateResolving)) + err := l.Transition(EventResolveSuccess, TransitionContext{ValidationEnabled: tt.validationEnabled}) + + if err != nil { + t.Fatalf("Transition() returned unexpected error: %v", err) + } + + if l.State() != tt.expectedState { + t.Errorf("got state %s, want %s", l.State(), tt.expectedState) + } + }) + } +} + +// Property Tests + +// TestPropertyTerminalStatesNeverTransitionOut verifies the fundamental property +// that terminal states never allow any transitions. +func TestPropertyTerminalStatesNeverTransitionOut(t *testing.T) { + // For every terminal state + for _, state := range terminalStates { + // For every possible event + for _, event := range allEvents { + // For various contexts + contexts := []TransitionContext{ + {}, + {ValidationEnabled: true}, + {RepairEnabled: true, RepairAttempt: 0, MaxRepairAttempts: 3}, + {AttemptsRemaining: 10}, + {StrictMode: true}, + } + + for _, ctx := range contexts { + l := New(WithInitialState(state)) + initialState := l.State() + _ = l.Transition(event, ctx) // ignore error + + if l.State() != initialState { + t.Errorf("Property violation: terminal state %s changed to %s on event %s", + initialState, l.State(), event) + } + } + } + } +} + +// TestPropertyStateAfterTransitionMatchesExpected verifies that successful +// transitions always result in the documented target state. +func TestPropertyStateAfterTransitionMatchesExpected(t *testing.T) { + // Build a map of expected transitions + type transitionKey struct { + from AgentLifecycleState + event AgentEvent + } + + // This is a simplified version; real test uses context-aware expected states + simpleExpected := map[transitionKey]AgentLifecycleState{ + {StateStarting, EventAgentSpawned}: StateRunning, + {StateRunning, EventWorkComplete}: StateQueuedForMerge, + {StateQueuedForMerge, EventMergeStarted}: StateMerging, + {StateMerging, EventMergeConflict}: StateResolving, + {StateResolving, EventResolveFailed}: StateMergeFailed, + {StateValidating, EventValidationPassed}: StateCompleted, + {StateRepairing, EventRepairComplete}: StateValidating, + {StateMergeFailed, EventRetriesExhausted}: StateFailed, + } + + for key, expected := range simpleExpected { + l := New(WithInitialState(key.from)) + err := l.Transition(key.event, TransitionContext{}) + + if err != nil { + t.Errorf("Unexpected error for %s + %s: %v", key.from, key.event, err) + continue + } + + if l.State() != expected { + t.Errorf("Property violation: %s + %s = %s, want %s", + key.from, key.event, l.State(), expected) + } + } +} + +// TestPropertyHistoryAppendedCorrectly verifies that each transition +// appends exactly one entry to history. +func TestPropertyHistoryAppendedCorrectly(t *testing.T) { + l := New() + + transitions := []struct { + event AgentEvent + ctx TransitionContext + }{ + {EventAgentSpawned, TransitionContext{}}, + {EventWorkComplete, TransitionContext{}}, + {EventMergeStarted, TransitionContext{}}, + {EventMergeSuccess, TransitionContext{ValidationEnabled: true}}, + {EventValidationPassed, TransitionContext{}}, + } + + for i, tr := range transitions { + beforeLen := len(l.History()) + err := l.Transition(tr.event, tr.ctx) + afterLen := len(l.History()) + + if err != nil { + t.Fatalf("Transition %d failed: %v", i, err) + } + + if afterLen != beforeLen+1 { + t.Errorf("History length after transition %d: got %d, want %d", + i, afterLen, beforeLen+1) + } + + // Verify the last entry matches + history := l.History() + last := history[len(history)-1] + if last.Event != tr.event { + t.Errorf("Last history entry event: got %s, want %s", last.Event, tr.event) + } + } +} + +// TestPropertyHistoryFromToMatch verifies that history entries correctly +// record the from and to states. +func TestPropertyHistoryFromToMatch(t *testing.T) { + l := New() + + // Perform a series of transitions + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + _ = l.Transition(EventMergeStarted, TransitionContext{}) + + history := l.History() + + // Check consecutive entries have matching states + for i := 1; i < len(history); i++ { + prev := history[i-1] + curr := history[i] + + if prev.To != curr.From { + t.Errorf("History discontinuity at index %d: entry %d To=%s, entry %d From=%s", + i, i-1, prev.To, i, curr.From) + } + } +} + +// TestPropertyFailedTransitionPreservesState verifies that failed transitions +// do not modify any state. +func TestPropertyFailedTransitionPreservesState(t *testing.T) { + invalidTransitions := []struct { + state AgentLifecycleState + event AgentEvent + }{ + {StateStarting, EventWorkComplete}, + {StateRunning, EventMergeSuccess}, + {StateCompleted, EventAgentSpawned}, + {StateFailed, EventRetry}, + } + + for _, tr := range invalidTransitions { + l := New(WithInitialState(tr.state)) + beforeState := l.State() + beforeHistory := l.History() + beforeCtx := l.Context() + + err := l.Transition(tr.event, TransitionContext{ValidationEnabled: true}) + + if err == nil { + t.Errorf("Expected error for %s + %s", tr.state, tr.event) + continue + } + + // Verify nothing changed + if l.State() != beforeState { + t.Errorf("State changed on failed transition: got %s, want %s", l.State(), beforeState) + } + + if len(l.History()) != len(beforeHistory) { + t.Errorf("History length changed on failed transition") + } + + if l.Context() != beforeCtx { + t.Errorf("Context changed on failed transition") + } + } +} + +// Concurrency Tests + +// TestConcurrentTransitionsNoCorruption verifies that parallel Transition() +// calls don't corrupt internal state. +func TestConcurrentTransitionsNoCorruption(t *testing.T) { + const numGoroutines = 100 + const transitionsPerGoroutine = 50 + + l := New(WithInitialState(StateRunning)) + + var wg sync.WaitGroup + wg.Add(numGoroutines) + + // All goroutines will try WorkFailed with retries, which keeps us in Running + for i := 0; i < numGoroutines; i++ { + go func() { + defer wg.Done() + for j := 0; j < transitionsPerGoroutine; j++ { + _ = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 1}) + } + }() + } + + wg.Wait() + + // State should still be Running (WorkFailed with retries stays in Running) + if l.State() != StateRunning { + t.Errorf("State corrupted: got %s, want %s", l.State(), StateRunning) + } + + // History should have exactly numGoroutines * transitionsPerGoroutine entries + // (up to maxHistoryEntries) + expectedEntries := numGoroutines * transitionsPerGoroutine + if expectedEntries > maxHistoryEntries { + expectedEntries = maxHistoryEntries + } + actualEntries := len(l.History()) + if actualEntries != expectedEntries { + t.Errorf("History count: got %d, want %d", actualEntries, expectedEntries) + } +} + +// TestConcurrentReadsNoRace verifies that concurrent reads don't race. +func TestConcurrentReadsNoRace(t *testing.T) { + l := New(WithInitialState(StateRunning)) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + + var wg sync.WaitGroup + wg.Add(300) + + // Concurrent State() calls + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + _ = l.State() + }() + } + + // Concurrent IsTerminal() calls + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + _ = l.IsTerminal() + }() + } + + // Concurrent History() calls + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + _ = l.History() + }() + } + + wg.Wait() +} + +// TestConcurrentReadWriteNoRace verifies mixed read/write doesn't race. +func TestConcurrentReadWriteNoRace(t *testing.T) { + l := New(WithInitialState(StateRunning)) + + var wg sync.WaitGroup + wg.Add(200) + + // Writers + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + _ = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 1}) + }() + } + + // Readers + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + _ = l.State() + _ = l.History() + _ = l.Context() + _ = l.IsTerminal() + }() + } + + wg.Wait() +} + +// TestCallbackCalledUnderLock verifies the callback is called atomically +// with the state change. +func TestCallbackCalledUnderLock(t *testing.T) { + var callbackCalls int64 + + cb := func(from, to AgentLifecycleState, event AgentEvent) { + atomic.AddInt64(&callbackCalls, 1) + _ = to // use to prevent unused variable warning + } + + l := New(WithTransitionCallback(cb)) + + // Perform many concurrent transitions + var wg sync.WaitGroup + const numGoroutines = 50 + + _ = l.Transition(EventAgentSpawned, TransitionContext{}) // Get to Running first + + wg.Add(numGoroutines) + for i := 0; i < numGoroutines; i++ { + go func() { + defer wg.Done() + _ = l.Transition(EventWorkFailed, TransitionContext{AttemptsRemaining: 1}) + }() + } + + wg.Wait() + + // Should have 1 (for AgentSpawned) + numGoroutines callbacks + expected := int64(1 + numGoroutines) + actual := atomic.LoadInt64(&callbackCalls) + if actual != expected { + t.Errorf("Callback calls: got %d, want %d", actual, expected) + } +} + +// TestConcurrentCancelRace tests that Cancel from multiple goroutines +// results in exactly one transition to Cancelled. +func TestConcurrentCancelRace(t *testing.T) { + const iterations = 100 + + for i := 0; i < iterations; i++ { + l := New(WithInitialState(StateRunning)) + + var wg sync.WaitGroup + var successCount int64 + + // Try to cancel from multiple goroutines simultaneously + for j := 0; j < 10; j++ { + wg.Add(1) + go func() { + defer wg.Done() + err := l.Transition(EventCancel, TransitionContext{}) + if err == nil { + atomic.AddInt64(&successCount, 1) + } + }() + } + + wg.Wait() + + // Exactly one Cancel should succeed + if successCount != 1 { + t.Errorf("Iteration %d: expected 1 successful cancel, got %d", i, successCount) + } + + // State should be Cancelled + if l.State() != StateCancelled { + t.Errorf("Iteration %d: expected Cancelled, got %s", i, l.State()) + } + } +} + +// TestHistorySliceIndependentOfInternal verifies that History() returns +// a copy, not a reference to internal state. +func TestHistorySliceIndependentOfInternal(t *testing.T) { + l := New() + _ = l.Transition(EventAgentSpawned, TransitionContext{}) + _ = l.Transition(EventWorkComplete, TransitionContext{}) + + history1 := l.History() + len1 := len(history1) + + // Modify the returned slice + history1[0].Event = "modified" + + // Get a new copy + history2 := l.History() + + // Internal state should be unchanged + if history2[0].Event == "modified" { + t.Error("History() returned reference to internal slice") + } + + // Perform another transition + _ = l.Transition(EventMergeStarted, TransitionContext{}) + + // Original slice should be unchanged + if len(history1) != len1 { + t.Error("Original history slice was modified") + } +} + +// TestContextCopiedNotReferenced verifies Context() returns a copy. +func TestContextCopiedNotReferenced(t *testing.T) { + l := New() + ctx := TransitionContext{ + ValidationEnabled: true, + AttemptsRemaining: 5, + } + _ = l.Transition(EventAgentSpawned, ctx) + + got1 := l.Context() + got1.AttemptsRemaining = 999 // modify the copy + + got2 := l.Context() + if got2.AttemptsRemaining == 999 { + t.Error("Context() returned reference, not copy") + } +} diff --git a/pkg/merge/sequential.go b/pkg/merge/sequential.go index 9dbbf685..3c933117 100644 --- a/pkg/merge/sequential.go +++ b/pkg/merge/sequential.go @@ -504,7 +504,12 @@ func (m *SequentialMerger) resetToHead(headCommit string, paths []string) error // Format: " ()" or combined messages if multiple commits. // Falls back to generating a message from the task title if no agent commits exist. func (m *SequentialMerger) buildMergeCommitMessage(result *agent.Result, opts *MergeOptions) string { - beadID := result.TaskID + // Use BeadID if set (repair/resolver agents set this to the original bead ID), + // otherwise fall back to TaskID (for normal agents, TaskID is the bead ID). + beadID := result.BeadID + if beadID == "" { + beadID = result.TaskID + } // If agent made commits, use their message(s) as the primary content if result.GitState != nil && len(result.GitState.CommitMessages) > 0 { diff --git a/pkg/mergecoordinator/coordinator.go b/pkg/mergecoordinator/coordinator.go index 38d41fcb..c84ba697 100644 --- a/pkg/mergecoordinator/coordinator.go +++ b/pkg/mergecoordinator/coordinator.go @@ -165,6 +165,22 @@ func (mc *MergeCoordinator) SetCleanupCallback(callback func(ctx context.Context mc.cleanupCallback = callback } +// SetMergeStatusCallback sets a callback for merge status events. +// This is used when running in daemon mode where IPC is not available. +func (mc *MergeCoordinator) SetMergeStatusCallback(callback mergequeue.MergeStatusCallback) { + if mc.processor != nil { + mc.processor.SetMergeStatusCallback(callback) + } +} + +// SetCommitCallback sets a callback for commit events. +// This is used when running in daemon mode where IPC is not available. +func (mc *MergeCoordinator) SetCommitCallback(callback mergequeue.CommitCallback) { + if mc.processor != nil { + mc.processor.SetCommitCallback(callback) + } +} + // SetAgentID records the agentID for a taskID, enabling parent-child tracking. func (mc *MergeCoordinator) SetAgentID(taskID, agentID string) { mc.agentIDMap.Store(taskID, agentID) @@ -237,6 +253,9 @@ func (mc *MergeCoordinator) HandleFailure(ctx context.Context, taskID string, re // Clean up overlay for failed task mc.cleanup(ctx, result) + // Get agent ID for this task (needed for merge status) + agentID := mc.GetAgentID(taskID) + // Check if this is an input-blocked failure (agent paused waiting for user input) if result != nil && result.InputBlocked { // Get session ID for resume support @@ -260,10 +279,26 @@ func (mc *MergeCoordinator) HandleFailure(ctx context.Context, taskID string, re fmt.Fprintf(os.Stderr, "ERROR: failed to mark task %s as failed: %v\n", taskID, err) } + // Send merge status as failed for agents that never entered the merge queue + // This ensures the UI can display the Details button (which requires merge_status) + mc.sendMergeStatusFailed(agentID, errMsg) + // Send task status update via IPC mc.SendTaskUpdated(taskID, taskTitle, "failed") } +// sendMergeStatusFailed sends a merge status failed event via IPC if client is connected. +// This is used for agents that fail before entering the merge queue. +func (mc *MergeCoordinator) sendMergeStatusFailed(agentID, errMsg string) { + if mc.ipcClient == nil || agentID == "" { + return + } + + if err := mc.ipcClient.SendAgentMergeStatusFull(agentID, ipc.MergeStatusFailed, 0, errMsg, 0, false, false); err != nil && mc.config.Verbose { + fmt.Fprintf(os.Stderr, "warning: failed to send merge status for %s: %v\n", agentID, err) + } +} + // SendTaskUpdated sends a task status update via IPC if client is connected. func (mc *MergeCoordinator) SendTaskUpdated(taskID, title, status string) { if mc.ipcClient == nil { diff --git a/pkg/mergequeue/processor.go b/pkg/mergequeue/processor.go index 604af69d..f2a1c41e 100644 --- a/pkg/mergequeue/processor.go +++ b/pkg/mergequeue/processor.go @@ -20,6 +20,39 @@ import ( "github.com/jzila/canopy/pkg/validation" ) +// MergeStatusEvent contains data for merge status callbacks. +// This allows the daemon to receive merge status updates without requiring an IPC client. +type MergeStatusEvent struct { + AgentID string + Status ipc.MergeStatus + QueuePos int + Error string + CommitsApplied int + HadConflict bool + ResolverSpawned bool +} + +// MergeStatusCallback is called when merge status changes. +// This provides an alternative to IPC for daemon-mode operation. +type MergeStatusCallback func(event MergeStatusEvent) + +// CommitEvent contains git commit information from a merge. +// This allows the daemon to receive commit events without requiring an IPC client. +type CommitEvent struct { + AgentID string + Hash string + ShortHash string + Message string + Author string + AuthorEmail string + Timestamp string + FilesChanged []string +} + +// CommitCallback is called when a commit is created during merge. +// This provides an alternative to IPC for daemon-mode operation. +type CommitCallback func(event CommitEvent) + // Processor handles merge operations from the queue. // It processes merge requests sequentially, spawning resolver agents when conflicts occur. type Processor struct { @@ -34,6 +67,12 @@ type Processor struct { runID string // Run ID for unique agent ID generation repoID string // Repository ID for IPC tracking + // Callback for merge status events (alternative to IPC for daemon mode) + mergeStatusCallback MergeStatusCallback + + // Callback for commit events (alternative to IPC for daemon mode) + commitCallback CommitCallback + // Validation and repair fields validationConfig *validation.ValidationConfig // Validation configuration (nil = disabled) repairAgent *repairagent.RepairAgent // Repair agent for fixing validation failures @@ -85,6 +124,20 @@ func (p *Processor) SetRepoID(repoID string) { p.repoID = repoID } +// SetMergeStatusCallback sets a callback for merge status events. +// This is used when running in daemon mode where IPC is not available. +// The callback is invoked whenever merge status would be sent via IPC. +func (p *Processor) SetMergeStatusCallback(callback MergeStatusCallback) { + p.mergeStatusCallback = callback +} + +// SetCommitCallback sets a callback for commit events. +// This is used when running in daemon mode where IPC is not available. +// The callback is invoked for each commit created during merge. +func (p *Processor) SetCommitCallback(callback CommitCallback) { + p.commitCallback = callback +} + // SetValidationConfig sets the validation configuration. // If nil, validation is disabled. func (p *Processor) SetValidationConfig(config *validation.ValidationConfig) { @@ -222,18 +275,58 @@ func (p *Processor) processMerge(ctx context.Context, req *MergeRequest) *MergeR // Continue anyway - we just won't be able to revert in strict mode } - // Apply merge via merger.MergeSingle() - // Pass task title for commit message generation if agent didn't make commits + // Check if HEAD has moved ahead of the overlay's base commit + // If so, we need a 3-way merge via resolver instead of direct patch application + needsResolver := false + resolverReason := "" + if req.Result.GitState != nil && req.Result.GitState.BaseCommit != "" { + baseCommit := req.Result.GitState.BaseCommit + if preMergeCommit != "" && baseCommit != preMergeCommit { + // HEAD has moved - check if it's ahead of base + mergeBase, err := p.getMergeBase(baseCommit, preMergeCommit) + if err != nil { + if p.verbose { + fmt.Fprintf(os.Stderr, "warning: failed to get merge-base for %s: %v\n", taskID, err) + } + } else if mergeBase == baseCommit { + // HEAD is ahead of the overlay's base - must use resolver for 3-way merge + needsResolver = true + resolverReason = fmt.Sprintf("overlay stale (base %s, HEAD now %s)", baseCommit[:8], preMergeCommit[:8]) + if p.verbose { + fmt.Printf("[%s] Detected stale overlay: base=%s, current HEAD=%s, merge-base=%s\n", + taskID, baseCommit[:8], preMergeCommit[:8], mergeBase[:8]) + } + } + } + } + + // Prepare merge options for later use mergeOpts := &merge.MergeOptions{ TaskTitle: req.Task.Title, } - mergeResult, err := p.merger.MergeSingle(req.Result, mergeOpts) - if err != nil { - resp.Error = fmt.Sprintf("merge failed: %v", err) - p.markTaskFailed(ctx, taskID, resp.Error) - p.sendTaskUpdated(taskID, req.Task.Title, "failed") - p.sendMergeStatusFull(taskID, ipc.MergeStatusFailed, resp.Error, 0, false, false) - return resp + + // If overlay is stale, skip direct merge and go straight to resolver + var mergeResult *merge.Result + if needsResolver { + // Create empty merge result to trigger resolver flow + mergeResult = &merge.Result{ + Errors: []string{resolverReason}, + PatchFailed: make(map[string]bool), + } + if p.verbose { + fmt.Printf("[%s] Skipping direct merge due to stale overlay, will spawn resolver\n", taskID) + } + } else { + // Apply merge via merger.MergeSingle() + // Pass task title for commit message generation if agent didn't make commits + mergeResult, err = p.merger.MergeSingle(req.Result, mergeOpts) + if err != nil { + resp.Error = fmt.Sprintf("merge failed: %v", err) + p.markTaskFailed(ctx, taskID, resp.Error) + p.sendTaskUpdated(taskID, req.Task.Title, "failed") + p.sendMergeStatusFull(taskID, ipc.MergeStatusFailed, resp.Error, 0, false, false) + return resp + } } resp.CommitsApplied = mergeResult.CommitsApplied @@ -321,11 +414,11 @@ func (p *Processor) processMerge(ctx context.Context, req *MergeRequest) *MergeR } // Determine if we need to spawn a resolver agent - needsResolver := false - resolverReason := "" + // Note: needsResolver may already be true if we detected a stale overlay above + // In that case, resolverReason is already set - // Case 1: Merge had errors (git add/commit failed) - if len(mergeResult.Errors) > 0 { + // Case 1: Merge had errors (git add/commit failed, or stale overlay detected) + if !needsResolver && len(mergeResult.Errors) > 0 { needsResolver = true resolverReason = "merge errors: " + strings.Join(mergeResult.Errors, "; ") } @@ -737,13 +830,34 @@ func (p *Processor) markTaskFailed(ctx context.Context, taskID string, reason st // sendMergedCommits sends commit events for each commit created during merge. // This sends the actual merged commits (with repo hashes) instead of overlay commits. +// Uses callback in daemon mode, IPC client in CLI mode. func (p *Processor) sendMergedCommits(taskID string, mergeResult *merge.Result) { - if p.ipcClient == nil || mergeResult == nil { + if mergeResult == nil { return } agentID := p.makeAgentID(taskID) for _, commitInfo := range mergeResult.MergedCommits { + // Try callback first (daemon mode) + if p.commitCallback != nil { + p.commitCallback(CommitEvent{ + AgentID: agentID, + Hash: commitInfo.Hash, + ShortHash: commitInfo.ShortHash, + Message: commitInfo.Message, + Author: commitInfo.Author, + AuthorEmail: commitInfo.AuthorEmail, + Timestamp: commitInfo.Timestamp, + FilesChanged: commitInfo.FilesChanged, + }) + continue + } + + // Fall back to IPC (CLI mode) + if p.ipcClient == nil { + continue + } + commit := &ipc.AgentCommitPayload{ Hash: commitInfo.Hash, ShortHash: commitInfo.ShortHash, @@ -760,26 +874,52 @@ func (p *Processor) sendMergedCommits(taskID string, mergeResult *merge.Result) } } -// sendMergeStatus sends a merge status update via IPC if client is connected. +// sendMergeStatus sends a merge status update via IPC or callback. func (p *Processor) sendMergeStatus(taskID string, status ipc.MergeStatus, queuePos int, errMsg string) { - if p.ipcClient == nil { + agentID := p.makeAgentID(taskID) + + // Try callback first (daemon mode) + if p.mergeStatusCallback != nil { + p.mergeStatusCallback(MergeStatusEvent{ + AgentID: agentID, + Status: status, + QueuePos: queuePos, + Error: errMsg, + }) return } - agentID := p.makeAgentID(taskID) + // Fall back to IPC (CLI mode) + if p.ipcClient == nil { + return + } if err := p.ipcClient.SendAgentMergeStatus(agentID, status, queuePos, errMsg); err != nil && p.verbose { fmt.Fprintf(os.Stderr, "warning: failed to send merge status for %s: %v\n", taskID, err) } } -// sendMergeStatusFull sends a final merge status update with full details via IPC. +// sendMergeStatusFull sends a final merge status update with full details via IPC or callback. // This should be used for merged/failed statuses to include commit and conflict information. func (p *Processor) sendMergeStatusFull(taskID string, status ipc.MergeStatus, errMsg string, commitsApplied int, hadConflict, resolverSpawned bool) { - if p.ipcClient == nil { + agentID := p.makeAgentID(taskID) + + // Try callback first (daemon mode) + if p.mergeStatusCallback != nil { + p.mergeStatusCallback(MergeStatusEvent{ + AgentID: agentID, + Status: status, + Error: errMsg, + CommitsApplied: commitsApplied, + HadConflict: hadConflict, + ResolverSpawned: resolverSpawned, + }) return } - agentID := p.makeAgentID(taskID) + // Fall back to IPC (CLI mode) + if p.ipcClient == nil { + return + } if err := p.ipcClient.SendAgentMergeStatusFull(agentID, status, 0, errMsg, commitsApplied, hadConflict, resolverSpawned); err != nil && p.verbose { fmt.Fprintf(os.Stderr, "warning: failed to send merge status for %s: %v\n", taskID, err) } @@ -1156,6 +1296,17 @@ func (p *Processor) getCurrentHead() (string, error) { return strings.TrimSpace(string(output)), nil } +// getMergeBase returns the best common ancestor between two commits. +// This is used to detect if HEAD has moved ahead of an overlay's base commit. +func (p *Processor) getMergeBase(commit1, commit2 string) (string, error) { + cmd := exec.Command("git", "-C", p.outputDir, "merge-base", commit1, commit2) + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git merge-base failed: %w", err) + } + return strings.TrimSpace(string(output)), nil +} + // revertMerge reverts the repository to a previous commit state. // This is used in strict validation mode when validation fails after merge. // It performs a hard reset to discard all changes made during the merge. diff --git a/pkg/mergequeue/validation_repair_test.go b/pkg/mergequeue/validation_repair_test.go index 7e6d33a1..9fbd4baf 100644 --- a/pkg/mergequeue/validation_repair_test.go +++ b/pkg/mergequeue/validation_repair_test.go @@ -1071,3 +1071,79 @@ func TestValidateOverlayAccessible_UnreadableFile(t *testing.T) { t.Errorf("validateOverlayAccessible() error = %v, want error containing 'cannot access file'", err) } } + +// TestGetMergeBase tests the getMergeBase function which detects divergent histories. +func TestGetMergeBase(t *testing.T) { + // Create a temporary git repository for testing + tmpDir := t.TempDir() + + // Initialize git repo + cmd := exec.Command("git", "init") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + t.Fatalf("failed to init git repo: %v", err) + } + + // Configure git user for commits + _ = exec.Command("git", "-C", tmpDir, "config", "user.name", "Test User").Run() + _ = exec.Command("git", "-C", tmpDir, "config", "user.email", "test@example.com").Run() + + // Create initial commit + if err := os.WriteFile(tmpDir+"/file1.txt", []byte("initial"), 0644); err != nil { + t.Fatalf("failed to create file: %v", err) + } + _ = exec.Command("git", "-C", tmpDir, "add", ".").Run() + _ = exec.Command("git", "-C", tmpDir, "commit", "-m", "initial").Run() + + // Get the initial commit hash (base) + baseOutput, err := exec.Command("git", "-C", tmpDir, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatalf("failed to get base commit: %v", err) + } + baseCommit := strings.TrimSpace(string(baseOutput)) + + // Create a second commit (current HEAD) + if err := os.WriteFile(tmpDir+"/file2.txt", []byte("second"), 0644); err != nil { + t.Fatalf("failed to create file2: %v", err) + } + _ = exec.Command("git", "-C", tmpDir, "add", ".").Run() + _ = exec.Command("git", "-C", tmpDir, "commit", "-m", "second").Run() + + // Get current HEAD + headOutput, err := exec.Command("git", "-C", tmpDir, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatalf("failed to get HEAD: %v", err) + } + currentHead := strings.TrimSpace(string(headOutput)) + + // Create processor + p := &Processor{ + outputDir: tmpDir, + verbose: false, + } + + // Test getMergeBase - should return baseCommit since it's the ancestor + mergeBase, err := p.getMergeBase(baseCommit, currentHead) + if err != nil { + t.Fatalf("getMergeBase() unexpected error: %v", err) + } + + if mergeBase != baseCommit { + t.Errorf("getMergeBase() = %s, want %s (base should be merge-base)", mergeBase[:8], baseCommit[:8]) + } +} + +// TestGetMergeBase_NonGitDir tests getMergeBase in a non-git directory. +func TestGetMergeBase_NonGitDir(t *testing.T) { + tmpDir := t.TempDir() + + p := &Processor{ + outputDir: tmpDir, + verbose: false, + } + + _, err := p.getMergeBase("abc123", "def456") + if err == nil { + t.Error("getMergeBase() expected error in non-git directory, got nil") + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a3aee473..b41b943f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -94,6 +94,18 @@ var ( }, []string{"type"}, ) + + // LifecycleDivergenceTotal tracks the total number of times the lifecycle state + // machine diverged from the legacy status fields. Used during the parallel + // rollout phase to detect and monitor inconsistencies. + LifecycleDivergenceTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "lifecycle_divergence_total", + Help: "Total number of lifecycle state machine divergences from legacy fields", + }, + []string{"legacy_status", "lifecycle_state", "event"}, + ) ) // RecordTaskDuration records a task duration with the given status. @@ -155,3 +167,9 @@ func ObserveResolverDuration(durationSeconds float64) { func IncIPCMessages(msgType string) { IPCMessagesTotal.WithLabelValues(msgType).Inc() } + +// RecordLifecycleDivergence records a divergence between the lifecycle state machine +// and legacy status fields. Used during the parallel rollout phase. +func RecordLifecycleDivergence(legacyStatus, lifecycleState, event string) { + LifecycleDivergenceTotal.WithLabelValues(legacyStatus, lifecycleState, event).Inc() +} diff --git a/pkg/orchestrator/api.go b/pkg/orchestrator/api.go index a129689c..99c21280 100644 --- a/pkg/orchestrator/api.go +++ b/pkg/orchestrator/api.go @@ -29,12 +29,14 @@ type RunConfig struct { Concurrency int `json:"concurrency"` // DryRun shows what would execute without running agents DryRun bool `json:"dry_run"` - // Watch enables watch mode (poll for new tasks instead of exiting) - Watch bool `json:"watch"` // MaxRetries is the maximum retry count for failed tasks (-1 = infinite) MaxRetries int `json:"max_retries"` // MaxPriority filters tasks by priority (only tasks with priority <= this value) MaxPriority int `json:"max_priority"` + // RuleOverrides contains custom rules to apply for this run only. + // These rules are applied with the highest precedence (above config.toml rules). + // When the run ends, these overrides are discarded unless persisted. + RuleOverrides []config.CustomRule `json:"rule_overrides,omitempty"` } // RuleUpdate contains fields for updating an existing rule. @@ -159,6 +161,17 @@ func (r *repoAPIImpl) Start(ctx context.Context, cfg RunConfig) error { if r.state == StatePaused { return fmt.Errorf("orchestrator is paused; use Resume() instead") } + + // Apply rule overrides if provided + if len(cfg.RuleOverrides) > 0 { + engine := r.orchestrator.GetRulesEngine() + if engine != nil { + if err := engine.ApplyOverrides(cfg.RuleOverrides); err != nil { + return fmt.Errorf("failed to apply rule overrides: %w", err) + } + } + } + // In the full implementation, this would: // 1. Apply RunConfig to the orchestrator // 2. Start the orchestrator's Run loop in a goroutine @@ -193,6 +206,13 @@ func (r *repoAPIImpl) Stop(ctx context.Context) error { if r.state == StateIdle { return nil // no-op if already idle } + + // Clear rule overrides when run ends + engine := r.orchestrator.GetRulesEngine() + if engine != nil { + engine.ClearOverrides() + } + // In the full implementation, this would: // 1. Signal the orchestrator to stop // 2. Wait for in-flight tasks to complete diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 27abf261..51693c8e 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -13,6 +13,7 @@ import ( cfgpkg "github.com/jzila/canopy/pkg/config" "github.com/jzila/canopy/pkg/ipc" "github.com/jzila/canopy/pkg/mergecoordinator" + "github.com/jzila/canopy/pkg/mergequeue" "github.com/jzila/canopy/pkg/rules" "github.com/jzila/canopy/pkg/sandbox" "github.com/jzila/canopy/pkg/scheduler" @@ -72,6 +73,16 @@ func (e *EventCallbacks) OnFail(ctx context.Context, taskID string, result *agen } } +// StateCallbacks defines callbacks for orchestrator state transitions. +// These are called when the orchestrator transitions between Idle and Active states. +type StateCallbacks struct { + // OnIdle is called when the orchestrator enters idle state (no in-flight tasks) + OnIdle func() + + // OnActive is called when the orchestrator enters active state (has in-flight tasks) + OnActive func() +} + // Config holds orchestrator configuration type Config struct { WorkDir string @@ -85,8 +96,7 @@ type Config struct { ResolverTimeout time.Duration // Timeout for resolver agents (0 = use default 10m) Rules *cfgpkg.RulesSettings // CLI overrides for rules (nil = use config.toml only) RulesOverrides *RulesOverrides // Tracks which rules fields were explicitly set via CLI - Watch bool // Watch mode: keep running and poll for new tasks instead of exiting when queue is empty - PollInterval time.Duration // Interval between polling for new tasks in watch mode (default: 5s) + PollInterval time.Duration // Interval between polling for new tasks when idle (default: 5s) } // RulesOverrides tracks which rules fields were explicitly set via CLI flags. @@ -109,7 +119,8 @@ type Orchestrator struct { mergeCoordinator *mergecoordinator.MergeCoordinator tempDir string callbackManager *CallbackManager - failureCounts map[string]int // Tracks how many times each task has failed + failureCountsMu sync.RWMutex // Protects failureCounts + failureCounts map[string]int // Tracks how many times each task has failed sandboxConfig *sandbox.SandboxConfig taskFilter *cfgpkg.TaskFilter // Task filter from rules settings (legacy) rulesEngine *rules.Engine // Rules engine for per-repo task selection @@ -122,11 +133,9 @@ type Orchestrator struct { inFlight map[string]bool // Tasks currently being executed inFlightTask map[string]*beads.Task // Task objects for in-flight tasks (for concurrency limiting) - // Watch mode statistics - watchStatsMu sync.RWMutex - watchIterations int // Number of polling iterations in watch mode - watchStartTime time.Time // When watch mode started - watchTasksTotal int // Total tasks processed in watch mode + // State transition callbacks + stateCallbacks *StateCallbacks + wasActive bool // Track previous state to detect transitions } // New creates a new orchestrator @@ -276,9 +285,14 @@ func (o *Orchestrator) setupInternalCallbacks() { // Delegate merge to coordinator - it handles queueing, merge, and task completion resp := o.mergeCoordinator.EnqueueMerge(ctx, result) - // Log merge result if verbose and there was an error - if resp != nil && !resp.Success && o.config.Verbose { - fmt.Fprintf(os.Stderr, "[%s] merge failed: %s\n", taskID, resp.Error) + // If merge failed, mark result as failed so failure count is incremented + // and retry limit applies. Without this, merge failures would retry forever. + if resp != nil && !resp.Success { + result.Success = false + result.Error = resp.Error + if o.config.Verbose { + fmt.Fprintf(os.Stderr, "[%s] merge failed: %s\n", taskID, resp.Error) + } } }, OnFailFn: func(ctx context.Context, taskID string, result *agent.Result) { @@ -305,6 +319,16 @@ func (o *Orchestrator) cleanupOverlay(ctx context.Context, result *agent.Result) } } +// CleanupOverlayByTaskID explicitly cleans up the overlay for a task by its ID. +// This can be called as a safety net when a terminal state is reached, ensuring +// overlay cleanup even if the normal callback chain was bypassed. +// Safe to call even if no overlay exists for the task ID. +func (o *Orchestrator) CleanupOverlayByTaskID(taskID string) { + if o.scheduler != nil { + o.scheduler.CleanupOverlay(taskID) + } +} + // WithCallbacks is a builder-style method to register callbacks func (o *Orchestrator) WithCallbacks(callbacks *EventCallbacks) *Orchestrator { o.SetCallbacks(callbacks) @@ -317,6 +341,20 @@ func (o *Orchestrator) SetIPCClient(client *ipc.Client) { o.mergeCoordinator.SetIPCClient(client) } +// SetMergeStatusCallback sets a callback for merge status events. +// This is used when running in daemon mode where IPC is not available. +// The callback is invoked whenever merge status would be sent via IPC. +func (o *Orchestrator) SetMergeStatusCallback(callback mergequeue.MergeStatusCallback) { + o.mergeCoordinator.SetMergeStatusCallback(callback) +} + +// SetCommitCallback sets a callback for commit events. +// This is used when running in daemon mode where IPC is not available. +// The callback is invoked for each commit created during merge. +func (o *Orchestrator) SetCommitCallback(callback mergequeue.CommitCallback) { + o.mergeCoordinator.SetCommitCallback(callback) +} + // SetRepoID sets the repository ID for IPC tracking. // This ID is passed to resolver agents for parent-child tracking. func (o *Orchestrator) SetRepoID(repoID string) { @@ -366,11 +404,57 @@ func (o *Orchestrator) GetAgentID(taskID string) string { return o.mergeCoordinator.GetAgentID(taskID) } -// Run executes the orchestration loop until no ready tasks remain. +// SetStateCallbacks sets callbacks for orchestrator state transitions (Idle/Active). +// These callbacks are invoked when the orchestrator transitions between having +// in-flight tasks (Active) and having no work to do (Idle). +func (o *Orchestrator) SetStateCallbacks(callbacks *StateCallbacks) { + o.stateCallbacks = callbacks +} + +// GetTaskAttempt returns the current attempt number for a task (1-indexed). +// Returns 1 for first attempt, 2 for first retry, etc. +// The attempt is calculated as: number of previous failures + 1. +func (o *Orchestrator) GetTaskAttempt(taskID string) int { + o.failureCountsMu.RLock() + defer o.failureCountsMu.RUnlock() + // Attempt = previous failures + 1 + // If no failures, this is attempt 1 (first try) + return o.failureCounts[taskID] + 1 +} + +// GetMaxRetries returns the configured maximum retry attempts. +// Returns 0 = no retries, -1 = infinite, positive = retry limit. +func (o *Orchestrator) GetMaxRetries() int { + return o.config.MaxRetries +} + +// notifyStateChange checks if state has changed and invokes appropriate callback. +func (o *Orchestrator) notifyStateChange(inFlightCount int) { + if o.stateCallbacks == nil { + return + } + + isActive := inFlightCount > 0 + if isActive && !o.wasActive { + // Transition to Active + o.wasActive = true + if o.stateCallbacks.OnActive != nil { + o.stateCallbacks.OnActive() + } + } else if !isActive && o.wasActive { + // Transition to Idle + o.wasActive = false + if o.stateCallbacks.OnIdle != nil { + o.stateCallbacks.OnIdle() + } + } +} + +// Run executes the orchestration loop, polling continuously for work until context is cancelled. // This uses dynamic task assignment: each worker calls bd ready to get // fresh tasks, ensuring newly-unblocked tasks are picked up immediately. -// In watch mode, the orchestrator polls for new tasks instead of exiting -// when the queue is empty. +// The orchestrator polls continuously: when work is available it processes tasks (Active state), +// when no work is available it sleeps for PollInterval (Idle state). func (o *Orchestrator) Run(ctx context.Context) error { // Start the merge coordinator's processor goroutine // It will process merge requests from the queue until context is cancelled @@ -381,33 +465,20 @@ func (o *Orchestrator) Run(ctx context.Context) error { return o.dryRun(ctx) } - // Set default poll interval for watch mode + // Set default poll interval pollInterval := o.config.PollInterval if pollInterval == 0 { pollInterval = 5 * time.Second } - // Initialize watch mode statistics - if o.config.Watch { - o.watchStatsMu.Lock() - o.watchStartTime = time.Now() - o.watchIterations = 0 - o.watchTasksTotal = 0 - o.watchStatsMu.Unlock() - - if o.config.Verbose { - fmt.Printf("Watch mode enabled, polling every %v\n", pollInterval) - } + if o.config.Verbose { + fmt.Printf("Orchestrator started, polling every %v when idle\n", pollInterval) } - // Use the slotManager for bounded concurrency (supports dynamic resizing) - // Track active workers and results var wg sync.WaitGroup var resultsMu sync.Mutex var results []*agent.Result - var runError error - var runErrorMu sync.Mutex // Create a channel to signal when workers should check for new tasks // Workers send on this channel when they complete a task @@ -419,33 +490,36 @@ func (o *Orchestrator) Run(ctx context.Context) error { resultsMu.Lock() results = append(results, result) + resultsMu.Unlock() - // Track success/failure counts + // Track success/failure counts (using dedicated mutex) + o.failureCountsMu.Lock() if result.Success { delete(o.failureCounts, taskID) } else { o.failureCounts[taskID]++ + failureCount := o.failureCounts[taskID] // Check if retries exhausted - if o.config.MaxRetries != -1 && o.failureCounts[taskID] > o.config.MaxRetries { - // Mark as permanently failed in beads - if err := o.beadsClient.Fail(ctx, taskID, fmt.Sprintf("Task failed after %d attempts", o.failureCounts[taskID])); err != nil { - fmt.Fprintf(os.Stderr, "warning: could not mark task %s as failed in beads: %v\n", taskID, err) + if o.config.MaxRetries != -1 && failureCount > o.config.MaxRetries { + // Mark as permanently failed in beads - closes task and adds needs-investigation label + if err := o.beadsClient.FailPermanently(ctx, taskID, fmt.Sprintf("Task failed after %d attempts", failureCount)); err != nil { + fmt.Fprintf(os.Stderr, "warning: could not mark task %s as permanently failed in beads: %v\n", taskID, err) } - fmt.Fprintf(os.Stderr, "ERROR: Task %s has failed %d times and will not be retried\n", taskID, o.failureCounts[taskID]) - - // Signal error but don't stop immediately - let other workers finish - // Note: In watch mode, we continue watching even after task failures - if !o.config.Watch { - runErrorMu.Lock() - if runError == nil { - runError = fmt.Errorf("task %s failed after %d retry attempts", taskID, o.config.MaxRetries) - } - runErrorMu.Unlock() + fmt.Fprintf(os.Stderr, "ERROR: Task %s has failed %d times and will not be retried\n", taskID, failureCount) + } else if o.config.MaxRetries == -1 || failureCount <= o.config.MaxRetries { + // Retry will occur - log retry info + maxAttempts := o.config.MaxRetries + 1 + if o.config.MaxRetries == -1 { + fmt.Fprintf(os.Stderr, "[%s] Merge failed (attempt %d/∞): %s. Retrying...\n", + taskID, failureCount, result.Error) + } else { + fmt.Fprintf(os.Stderr, "[%s] Merge failed (attempt %d/%d): %s. Retrying...\n", + taskID, failureCount, maxAttempts, result.Error) } } } - resultsMu.Unlock() + o.failureCountsMu.Unlock() // Signal that a task completed (non-blocking) select { @@ -454,119 +528,31 @@ func (o *Orchestrator) Run(ctx context.Context) error { } } - // Initial check for tasks - task, shouldStop, err := o.getNextTask(ctx) - if err != nil { - return err - } - - // Handle initial state when no tasks are available - if shouldStop || task == nil { - if o.config.Watch { - // In watch mode, wait for tasks to appear - if o.config.Verbose { - fmt.Println("No ready tasks, watching for new tasks...") - } - } else { - // Normal mode: exit immediately - if o.config.Verbose { - if shouldStop { - fmt.Println("No tasks to execute, orchestration complete") - } else { - fmt.Println("No ready tasks, orchestration complete") - } - } - return nil - } - } - - // Start the first worker with the initial task (if we have one) - if task != nil { - if o.config.Verbose { - fmt.Printf("Starting task: %s: %s\n", task.ID, task.Title) - } - - // Track task in watch mode stats - if o.config.Watch { - o.watchStatsMu.Lock() - o.watchTasksTotal++ - o.watchStatsMu.Unlock() - } - - // Acquire slot before spawning worker - if err := o.slotManager.Acquire(ctx); err != nil { - return err - } - wg.Add(1) - go func(t *beads.Task) { - defer wg.Done() - defer o.slotManager.Release() - - o.scheduler.ExecuteTask(ctx, t, completionCallback) - }(task) - } - - // Main loop: keep spawning workers as slots become available + // Main loop: poll continuously until context is cancelled for { select { case <-ctx.Done(): // Wait for in-flight tasks to complete wg.Wait() - if o.config.Watch && o.config.Verbose { - o.printWatchStats() - } return ctx.Err() - - case <-taskComplete: - // A task completed, try to get more work - default: - // Try to acquire a slot (non-blocking check first) - } - - // Check if we have an error that should stop us (only in non-watch mode) - if !o.config.Watch { - runErrorMu.Lock() - if runError != nil { - runErrorMu.Unlock() - // Wait for in-flight tasks - wg.Wait() - return runError - } - runErrorMu.Unlock() } // Try to acquire a slot if err := o.slotManager.Acquire(ctx); err != nil { // Context cancelled wg.Wait() - if o.config.Watch && o.config.Verbose { - o.printWatchStats() - } return err } // Got a slot, get the next task - task, shouldStop, err := o.getNextTask(ctx) + task, _, err := o.getNextTask(ctx) if err != nil { o.slotManager.Release() wg.Wait() return err } - if shouldStop { - // Stop condition met - release slot and wait for in-flight tasks - o.slotManager.Release() - wg.Wait() - if o.config.Verbose { - fmt.Println("Stop condition met, orchestration complete") - if o.config.Watch { - o.printWatchStats() - } - } - return nil - } - if task == nil { // No task available right now o.slotManager.Release() @@ -577,35 +563,19 @@ func (o *Orchestrator) Run(ctx context.Context) error { o.inFlightMu.RUnlock() if inFlightCount == 0 { - // No in-flight tasks and no ready tasks - if o.config.Watch { - // Watch mode: poll for new tasks - o.watchStatsMu.Lock() - o.watchIterations++ - o.watchStatsMu.Unlock() - - if o.config.Verbose { - fmt.Printf("Watching for new tasks (iteration %d)...\n", o.watchIterations) - } - - select { - case <-ctx.Done(): - wg.Wait() - if o.config.Verbose { - o.printWatchStats() - } - return ctx.Err() - case <-time.After(pollInterval): - // Poll again - continue - } - } else { - // Normal mode: we're done + // No in-flight tasks and no ready tasks - idle state + // Poll for new tasks after interval + if o.config.Verbose { + fmt.Println("Idle, polling for new tasks...") + } + + select { + case <-ctx.Done(): wg.Wait() - if o.config.Verbose { - fmt.Println("No more tasks, orchestration complete") - } - return nil + return ctx.Err() + case <-time.After(pollInterval): + // Poll again + continue } } @@ -613,9 +583,6 @@ func (o *Orchestrator) Run(ctx context.Context) error { select { case <-ctx.Done(): wg.Wait() - if o.config.Watch && o.config.Verbose { - o.printWatchStats() - } return ctx.Err() case <-taskComplete: // A task completed, loop back to try again @@ -628,13 +595,6 @@ func (o *Orchestrator) Run(ctx context.Context) error { fmt.Printf("Starting task: %s: %s\n", task.ID, task.Title) } - // Track task in watch mode stats - if o.config.Watch { - o.watchStatsMu.Lock() - o.watchTasksTotal++ - o.watchStatsMu.Unlock() - } - wg.Add(1) go func(t *beads.Task) { defer wg.Done() @@ -645,49 +605,6 @@ func (o *Orchestrator) Run(ctx context.Context) error { } } -// printWatchStats prints watch mode statistics to stderr. -func (o *Orchestrator) printWatchStats() { - o.watchStatsMu.RLock() - defer o.watchStatsMu.RUnlock() - - duration := time.Since(o.watchStartTime) - fmt.Fprintf(os.Stderr, "Watch mode: ran for %v, processed %d tasks across %d iterations\n", - duration.Round(time.Second), o.watchTasksTotal, o.watchIterations) -} - -// WatchStats contains statistics for watch mode operation. -type WatchStats struct { - Enabled bool `json:"enabled"` - StartTime time.Time `json:"start_time,omitempty"` - Duration time.Duration `json:"duration,omitempty"` - Iterations int `json:"iterations"` - TasksTotal int `json:"tasks_total"` -} - -// GetWatchStats returns current watch mode statistics. -// Returns nil if watch mode is not enabled. -func (o *Orchestrator) GetWatchStats() *WatchStats { - if !o.config.Watch { - return &WatchStats{Enabled: false} - } - - o.watchStatsMu.RLock() - defer o.watchStatsMu.RUnlock() - - return &WatchStats{ - Enabled: true, - StartTime: o.watchStartTime, - Duration: time.Since(o.watchStartTime), - Iterations: o.watchIterations, - TasksTotal: o.watchTasksTotal, - } -} - -// IsWatchMode returns true if the orchestrator is running in watch mode. -func (o *Orchestrator) IsWatchMode() bool { - return o.config.Watch -} - // dryRun shows what tasks would execute without actually running them func (o *Orchestrator) dryRun(ctx context.Context) error { // Get ready tasks from beads @@ -727,6 +644,18 @@ func (o *Orchestrator) GetScheduler() *scheduler.Scheduler { return o.scheduler } +// Shutdown gracefully stops the orchestrator and waits for cleanup to complete. +// This ensures all active overlays are unmounted before returning. +// The timeout specifies how long to wait for overlay cleanup (not task completion). +// Returns the number of overlays cleaned and any errors encountered. +func (o *Orchestrator) Shutdown(timeout time.Duration) (int, error) { + // Clean up overlays via the scheduler + if o.scheduler != nil { + return o.scheduler.CleanupAll(timeout) + } + return 0, nil +} + // filterTasksByMaxPriority filters tasks to only include those with priority <= maxPriority. // This is a hard filter applied after fetching tasks from beads. // Deprecated: Use TaskFilter.FilterTasks() instead for full rules support. @@ -790,16 +719,27 @@ func (o *Orchestrator) filterTasksWithEngine(tasks []beads.Task) []beads.Task { // markInFlight marks a task as currently in-flight func (o *Orchestrator) markInFlight(taskID string) { o.inFlightMu.Lock() - defer o.inFlightMu.Unlock() + wasEmpty := len(o.inFlight) == 0 o.inFlight[taskID] = true + count := len(o.inFlight) + o.inFlightMu.Unlock() + + // Notify state change if we transitioned from empty to non-empty + if wasEmpty { + o.notifyStateChange(count) + } } // unmarkInFlight removes a task from the in-flight set func (o *Orchestrator) unmarkInFlight(taskID string) { o.inFlightMu.Lock() - defer o.inFlightMu.Unlock() delete(o.inFlight, taskID) delete(o.inFlightTask, taskID) + count := len(o.inFlight) + o.inFlightMu.Unlock() + + // Notify state change (might transition to idle if count is 0) + o.notifyStateChange(count) } // isInFlight returns whether a task is currently in-flight @@ -894,4 +834,4 @@ func mergeRulesSettings(base, cliRules *cfgpkg.RulesSettings, overrides *RulesOv } return result -} +} \ No newline at end of file diff --git a/pkg/persistence/migrate.go b/pkg/persistence/migrate.go index d1963ef4..6e51d45c 100644 --- a/pkg/persistence/migrate.go +++ b/pkg/persistence/migrate.go @@ -5,7 +5,7 @@ import ( "fmt" ) -const currentSchemaVersion = 14 +const currentSchemaVersion = 17 // migrate runs all pending database migrations func (s *Store) migrate() error { @@ -101,6 +101,18 @@ func (s *Store) runMigration(version int) error { if err := s.migrateV14(tx); err != nil { return err } + case 15: + if err := s.migrateV15(tx); err != nil { + return err + } + case 16: + if err := s.migrateV16(tx); err != nil { + return err + } + case 17: + if err := s.migrateV17(tx); err != nil { + return err + } default: return fmt.Errorf("unknown migration version: %d", version) } @@ -435,3 +447,64 @@ func (s *Store) migrateV14(tx *sql.Tx) error { _, err := tx.Exec(schema) return err } + +// migrateV15 adds lifecycle_state column to agents table for unified state machine tracking. +// Existing agents will have NULL lifecycle_state, which is handled by deriving it from +// legacy fields (status, merge_status, validation_status) on first read. +func (s *Store) migrateV15(tx *sql.Tx) error { + migrations := []string{ + // Add lifecycle_state column (nullable for migration compatibility) + // Valid values match lifecycle.AgentLifecycleState: starting, running, queued_for_merge, + // merging, resolving, validating, repairing, merge_failed, completed, failed, + // needs_attention, cancelled, timed_out + `ALTER TABLE agents ADD COLUMN lifecycle_state TEXT`, + } + + for _, m := range migrations { + if _, err := tx.Exec(m); err != nil { + return fmt.Errorf("failed to execute migration: %s: %w", m, err) + } + } + + return nil +} + +// migrateV16 creates the run_configs table for persisting per-repository orchestrator settings +func (s *Store) migrateV16(tx *sql.Tx) error { + schema := ` + CREATE TABLE IF NOT EXISTS run_configs ( + repo_id TEXT PRIMARY KEY, + concurrency INTEGER NOT NULL DEFAULT 4, + max_priority INTEGER NOT NULL DEFAULT 4, + use_bwrap INTEGER NOT NULL DEFAULT 1, + max_retries INTEGER NOT NULL DEFAULT 3, + updated_at INTEGER DEFAULT (strftime('%s', 'now')) + ); + ` + + _, err := tx.Exec(schema) + if err != nil { + return fmt.Errorf("failed to create run_configs table: %w", err) + } + + return nil +} + +// migrateV17 adds task retry tracking fields (attempt and max_retries) to agents table. +// These are separate from RepairAttempts which tracks validation repair attempts. +func (s *Store) migrateV17(tx *sql.Tx) error { + migrations := []string{ + // Add attempt column (1 = first try, 2 = first retry, etc.) + `ALTER TABLE agents ADD COLUMN attempt INTEGER DEFAULT 1`, + // Add max_retries column (how many retries are allowed) + `ALTER TABLE agents ADD COLUMN max_retries INTEGER DEFAULT 3`, + } + + for _, m := range migrations { + if _, err := tx.Exec(m); err != nil { + return fmt.Errorf("failed to execute migration: %s: %w", m, err) + } + } + + return nil +} diff --git a/pkg/persistence/store.go b/pkg/persistence/store.go index 61015098..e37782b9 100644 --- a/pkg/persistence/store.go +++ b/pkg/persistence/store.go @@ -110,6 +110,16 @@ type ActiveOverlay struct { Status string `json:"status"` // active, completed, orphaned } +// RunConfig represents persisted orchestrator settings for a repository +type RunConfig struct { + RepoID string `json:"repo_id"` + Concurrency int `json:"concurrency"` + MaxPriority int `json:"max_priority"` + UseBwrap bool `json:"use_bwrap"` + MaxRetries int `json:"max_retries"` + UpdatedAt int64 `json:"updated_at,omitempty"` +} + // Agent represents a single agent execution within a run type Agent struct { ID string `json:"id"` @@ -118,6 +128,7 @@ type Agent struct { TaskTitle string `json:"task_title"` TaskDescription string `json:"task_description,omitempty"` // Full task description from beads Status AgentStatus `json:"status"` + LifecycleState string `json:"lifecycle_state,omitempty"` // Unified lifecycle state (source of truth) StartedAt time.Time `json:"started_at"` FinishedAt *time.Time `json:"finished_at,omitempty"` DurationSeconds float64 `json:"duration_seconds,omitempty"` @@ -154,6 +165,9 @@ type Agent struct { LastRepairOutput string `json:"last_repair_output,omitempty"` // Output/error from the last repair attempt // Session tracking for claude --resume support SessionID string `json:"session_id,omitempty"` // Claude CLI session ID for resumability + // Task retry tracking fields + Attempt int `json:"attempt"` // Current attempt number (1 = first try) + MaxRetries int `json:"max_retries"` // Max retry attempts configured } // RunFilter specifies criteria for querying runs @@ -354,12 +368,13 @@ func (t *Tx) CreateRun(run *Run) error { // the existing record is updated with the new values. func (t *Tx) CreateAgent(agent *Agent) error { query := ` - INSERT INTO agents (id, run_id, task_id, task_title, task_description, status, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, session_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO agents (id, run_id, task_id, task_title, task_description, status, lifecycle_state, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, session_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET run_id = excluded.run_id, task_description = excluded.task_description, status = excluded.status, + lifecycle_state = excluded.lifecycle_state, started_at = excluded.started_at, finished_at = excluded.finished_at, duration_seconds = excluded.duration_seconds, @@ -401,6 +416,7 @@ func (t *Tx) CreateAgent(agent *Agent) error { agent.TaskTitle, nullString(agent.TaskDescription), string(agent.Status), + nullString(agent.LifecycleState), agent.StartedAt.Unix(), finishedAt, agent.DurationSeconds, @@ -667,12 +683,13 @@ func (s *Store) ListRuns(filter RunFilter) (*RunListResult, error) { // the existing record is updated with the new values. func (s *Store) CreateAgent(agent *Agent) error { query := ` - INSERT INTO agents (id, run_id, task_id, task_title, task_description, status, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, session_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO agents (id, run_id, task_id, task_title, task_description, status, lifecycle_state, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, session_id, attempt, max_retries) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET run_id = excluded.run_id, task_description = excluded.task_description, status = excluded.status, + lifecycle_state = excluded.lifecycle_state, started_at = excluded.started_at, finished_at = excluded.finished_at, duration_seconds = excluded.duration_seconds, @@ -700,7 +717,9 @@ func (s *Store) CreateAgent(agent *Agent) error { validation_duration_ms = excluded.validation_duration_ms, validation_error = excluded.validation_error, validation_steps = excluded.validation_steps, - session_id = excluded.session_id + session_id = excluded.session_id, + attempt = excluded.attempt, + max_retries = excluded.max_retries ` var finishedAt *int64 if agent.FinishedAt != nil { @@ -714,6 +733,7 @@ func (s *Store) CreateAgent(agent *Agent) error { agent.TaskTitle, nullString(agent.TaskDescription), string(agent.Status), + nullString(agent.LifecycleState), agent.StartedAt.Unix(), finishedAt, agent.DurationSeconds, @@ -744,6 +764,8 @@ func (s *Store) CreateAgent(agent *Agent) error { nullString(agent.ValidationError), nullString(agent.ValidationSteps), nullString(agent.SessionID), + agent.Attempt, + agent.MaxRetries, ) return err } @@ -761,6 +783,7 @@ func (s *Store) UpdateAgent(agent *Agent) error { query := ` UPDATE agents SET status = ?, + lifecycle_state = ?, finished_at = ?, duration_seconds = ?, exit_code = ?, @@ -799,6 +822,7 @@ func (s *Store) UpdateAgent(agent *Agent) error { } _, err := s.db.Exec(query, string(agent.Status), + nullString(agent.LifecycleState), finishedAt, agent.DurationSeconds, agent.ExitCode, @@ -847,6 +871,7 @@ func (s *Store) UpdateAgentCompletion(agent *Agent) error { query := ` UPDATE agents SET status = ?, + lifecycle_state = ?, finished_at = ?, duration_seconds = ?, exit_code = ?, @@ -873,6 +898,7 @@ func (s *Store) UpdateAgentCompletion(agent *Agent) error { } _, err := s.db.Exec(query, string(agent.Status), + nullString(agent.LifecycleState), finishedAt, agent.DurationSeconds, agent.ExitCode, @@ -955,22 +981,30 @@ func (s *Store) UpdateAgentRepairState(agentID string, repairAttempts int, lastR return err } +// UpdateAgentLifecycleState updates only the lifecycle_state field of an agent. +// This is the primary method for persisting lifecycle state transitions. +func (s *Store) UpdateAgentLifecycleState(agentID string, lifecycleState string) error { + query := `UPDATE agents SET lifecycle_state = ? WHERE id = ?` + _, err := s.db.Exec(query, nullString(lifecycleState), agentID) + return err +} + // agentColumns lists all columns for agent queries -const agentColumns = `id, run_id, task_id, task_title, task_description, status, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, repair_attempts, last_repair_output, session_id` +const agentColumns = `id, run_id, task_id, task_title, task_description, status, lifecycle_state, started_at, finished_at, duration_seconds, exit_code, error_message, stdout, stderr, input_tokens, output_tokens, total_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, git_commits_created, num_turns, result_message, repo_id, archived, parent_agent_id, merge_status, merge_commits_applied, merge_had_conflict, merge_resolver_spawned, merge_error, validation_status, validation_duration_ms, validation_error, validation_steps, repair_attempts, last_repair_output, session_id, attempt, max_retries` // agentColumnsWithPrefix returns the agent columns with a table alias prefix. // This is used for queries with JOINs to disambiguate column names. func agentColumnsWithPrefix(prefix string) string { cols := []string{ - "id", "run_id", "task_id", "task_title", "task_description", "status", "started_at", "finished_at", - "duration_seconds", "exit_code", "error_message", "stdout", "stderr", + "id", "run_id", "task_id", "task_title", "task_description", "status", "lifecycle_state", + "started_at", "finished_at", "duration_seconds", "exit_code", "error_message", "stdout", "stderr", "input_tokens", "output_tokens", "total_tokens", "cache_creation_tokens", "cache_read_tokens", "cost_usd", "files_changed", "git_commits_created", "num_turns", "result_message", "repo_id", "archived", "parent_agent_id", "merge_status", "merge_commits_applied", "merge_had_conflict", "merge_resolver_spawned", "merge_error", "validation_status", "validation_duration_ms", "validation_error", "validation_steps", - "repair_attempts", "last_repair_output", "session_id", + "repair_attempts", "last_repair_output", "session_id", "attempt", "max_retries", } result := make([]string, len(cols)) for i, col := range cols { @@ -1561,6 +1595,7 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { var startedAt, finishedAt sql.NullInt64 var exitCode sql.NullInt64 var status string + var lifecycleState sql.NullString var taskDescription sql.NullString var errorMessage, stdout, stderr, resultMessage, repoID, parentAgentID sql.NullString var archived sql.NullInt64 @@ -1571,6 +1606,7 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { var repairAttempts sql.NullInt64 var lastRepairOutput sql.NullString var sessionID sql.NullString + var attempt, maxRetries sql.NullInt64 err := row.Scan( &agent.ID, @@ -1579,6 +1615,7 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { &agent.TaskTitle, &taskDescription, &status, + &lifecycleState, &startedAt, &finishedAt, &agent.DurationSeconds, @@ -1611,6 +1648,8 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { &repairAttempts, &lastRepairOutput, &sessionID, + &attempt, + &maxRetries, ) if err == sql.ErrNoRows { return nil, nil @@ -1620,6 +1659,7 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { } agent.Status = AgentStatus(status) + agent.LifecycleState = lifecycleState.String agent.TaskDescription = taskDescription.String agent.StartedAt = time.Unix(startedAt.Int64, 0) if finishedAt.Valid { @@ -1649,6 +1689,8 @@ func (s *Store) scanAgent(row *sql.Row) (*Agent, error) { agent.RepairAttempts = int(repairAttempts.Int64) agent.LastRepairOutput = lastRepairOutput.String agent.SessionID = sessionID.String + agent.Attempt = int(attempt.Int64) + agent.MaxRetries = int(maxRetries.Int64) return &agent, nil } @@ -1658,6 +1700,7 @@ func (s *Store) scanAgentFromRows(rows *sql.Rows) (*Agent, error) { var startedAt, finishedAt sql.NullInt64 var exitCode sql.NullInt64 var status string + var lifecycleState sql.NullString var taskDescription sql.NullString var errorMessage, stdout, stderr, resultMessage, repoID, parentAgentID sql.NullString var archived sql.NullInt64 @@ -1668,6 +1711,7 @@ func (s *Store) scanAgentFromRows(rows *sql.Rows) (*Agent, error) { var repairAttempts sql.NullInt64 var lastRepairOutput sql.NullString var sessionID sql.NullString + var attempt, maxRetries sql.NullInt64 err := rows.Scan( &agent.ID, @@ -1676,6 +1720,7 @@ func (s *Store) scanAgentFromRows(rows *sql.Rows) (*Agent, error) { &agent.TaskTitle, &taskDescription, &status, + &lifecycleState, &startedAt, &finishedAt, &agent.DurationSeconds, @@ -1708,12 +1753,15 @@ func (s *Store) scanAgentFromRows(rows *sql.Rows) (*Agent, error) { &repairAttempts, &lastRepairOutput, &sessionID, + &attempt, + &maxRetries, ) if err != nil { return nil, fmt.Errorf("failed to scan agent: %w", err) } agent.Status = AgentStatus(status) + agent.LifecycleState = lifecycleState.String agent.TaskDescription = taskDescription.String agent.StartedAt = time.Unix(startedAt.Int64, 0) if finishedAt.Valid { @@ -1743,6 +1791,8 @@ func (s *Store) scanAgentFromRows(rows *sql.Rows) (*Agent, error) { agent.RepairAttempts = int(repairAttempts.Int64) agent.LastRepairOutput = lastRepairOutput.String agent.SessionID = sessionID.String + agent.Attempt = int(attempt.Int64) + agent.MaxRetries = int(maxRetries.Int64) return &agent, nil } @@ -2078,3 +2128,72 @@ func (s *Store) scanOverlayFromRows(rows *sql.Rows) (*ActiveOverlay, error) { overlay.SessionID = sessionID.String return &overlay, nil } + +// DefaultRunConfig returns a RunConfig with default values +func DefaultRunConfig(repoID string) *RunConfig { + return &RunConfig{ + RepoID: repoID, + Concurrency: 4, + MaxPriority: 4, + UseBwrap: true, + MaxRetries: 3, + } +} + +// GetRunConfig retrieves the run configuration for a repository. +// Returns a default configuration if no record exists for the repo. +func (s *Store) GetRunConfig(repoID string) (*RunConfig, error) { + query := `SELECT repo_id, concurrency, max_priority, use_bwrap, max_retries, updated_at FROM run_configs WHERE repo_id = ?` + row := s.db.QueryRow(query, repoID) + + var config RunConfig + var useBwrap int + var updatedAt sql.NullInt64 + + err := row.Scan( + &config.RepoID, + &config.Concurrency, + &config.MaxPriority, + &useBwrap, + &config.MaxRetries, + &updatedAt, + ) + if err == sql.ErrNoRows { + // Return default config if not found + return DefaultRunConfig(repoID), nil + } + if err != nil { + return nil, fmt.Errorf("failed to get run config: %w", err) + } + + config.UseBwrap = useBwrap == 1 + config.UpdatedAt = updatedAt.Int64 + + return &config, nil +} + +// SaveRunConfig creates or updates the run configuration for a repository. +// Uses upsert pattern to handle both new and existing configurations. +func (s *Store) SaveRunConfig(repoID string, config *RunConfig) error { + query := ` + INSERT INTO run_configs (repo_id, concurrency, max_priority, use_bwrap, max_retries, updated_at) + VALUES (?, ?, ?, ?, ?, strftime('%s', 'now')) + ON CONFLICT(repo_id) DO UPDATE SET + concurrency = excluded.concurrency, + max_priority = excluded.max_priority, + use_bwrap = excluded.use_bwrap, + max_retries = excluded.max_retries, + updated_at = excluded.updated_at + ` + _, err := s.db.Exec(query, + repoID, + config.Concurrency, + config.MaxPriority, + boolToInt(config.UseBwrap), + config.MaxRetries, + ) + if err != nil { + return fmt.Errorf("failed to save run config: %w", err) + } + return nil +} diff --git a/pkg/repairagent/spawner.go b/pkg/repairagent/spawner.go index 643b2866..feba1384 100644 --- a/pkg/repairagent/spawner.go +++ b/pkg/repairagent/spawner.go @@ -148,6 +148,9 @@ func (r *RepairAgent) Repair(ctx context.Context, repairCtx *RepairContext, pare // Execute the repair agent agentResult := r.executor.Execute(ctx, repairTask, overlay, nil, nil) + // Set the original bead ID for commit messages (TaskID is the synthetic repair agent ID) + agentResult.BeadID = repairCtx.TaskID + result.AgentResult = agentResult result.Success = agentResult.Success result.Duration = time.Since(start) @@ -269,6 +272,9 @@ func (r *RepairAgent) RepairPreCommit(ctx context.Context, repairCtx *PreCommitR // Execute the repair agent agentResult := r.executor.Execute(ctx, repairTask, overlay, nil, nil) + // Set the original bead ID for commit messages (TaskID is the synthetic repair agent ID) + agentResult.BeadID = repairCtx.TaskID + result.AgentResult = agentResult result.Success = agentResult.Success result.Duration = time.Since(start) diff --git a/pkg/resolver/resolver.go b/pkg/resolver/resolver.go index 4e4e0fa3..d03aee34 100644 --- a/pkg/resolver/resolver.go +++ b/pkg/resolver/resolver.go @@ -189,6 +189,9 @@ func (r *Resolver) Resolve(ctx context.Context, conflict *ConflictContext) (*Res // Execute the resolver agent agentResult := r.executor.Execute(ctx, resolverTask, overlay, nil, nil) + // Set the original bead ID for commit messages (TaskID is the synthetic resolver agent ID) + agentResult.BeadID = conflict.TaskID + // Cleanup overlay _ = overlay.Unmount() // Don't cleanup directories yet - they're needed for merge diff --git a/pkg/rules/engine.go b/pkg/rules/engine.go index 8a37d488..6d721c7b 100644 --- a/pkg/rules/engine.go +++ b/pkg/rules/engine.go @@ -9,13 +9,25 @@ import ( "github.com/jzila/canopy/pkg/config" ) +// internalRule wraps a CustomRule with its source for internal tracking. +type internalRule struct { + config.CustomRule + Source config.RuleSource +} + // Engine evaluates task selection rules against candidate tasks. // Rules are stored in a single unified slice, evaluated in order. +// The three-tier precedence hierarchy is: +// 1. Override rules (run-configured, highest priority) +// 2. Config rules (from .canopy/config.toml) +// 3. Default rules (built-in, lowest priority) +// // The configSnapshot tracks the original config state for persistence tracking. type Engine struct { settings *config.RulesSettings // Filter settings (priority, types, labels, etc.) - rules []config.CustomRule // Unified rules slice (config + runtime) + rules []internalRule // Unified rules slice with source tracking configSnapshot []config.CustomRule // Snapshot of rules loaded from config (for persistence tracking) + defaultRules []config.CustomRule // Default rules (formalized baseline) mu sync.RWMutex } @@ -23,14 +35,20 @@ type Engine struct { // Rules from cfg.Custom are loaded into the unified rules slice. func NewEngine(cfg *config.RulesSettings) *Engine { e := &Engine{ - settings: cfg, - rules: nil, + settings: cfg, + rules: nil, + defaultRules: nil, } - // Load config rules into unified slice and snapshot + // Load config rules into unified slice and snapshot with source tracking if cfg != nil && len(cfg.Custom) > 0 { - e.rules = make([]config.CustomRule, len(cfg.Custom)) - copy(e.rules, cfg.Custom) + e.rules = make([]internalRule, len(cfg.Custom)) + for i, r := range cfg.Custom { + e.rules[i] = internalRule{ + CustomRule: r, + Source: config.RuleSourceConfig, + } + } e.configSnapshot = make([]config.CustomRule, len(cfg.Custom)) copy(e.configSnapshot, cfg.Custom) } @@ -38,6 +56,32 @@ func NewEngine(cfg *config.RulesSettings) *Engine { return e } +// NewEngineWithDefaults creates a new rule evaluation engine with explicit default rules. +// This allows establishing a baseline ruleset that is always present. +// Precedence: override rules > config rules > default rules. +func NewEngineWithDefaults(cfg *config.RulesSettings, defaults []config.CustomRule) *Engine { + e := NewEngine(cfg) + + // Store default rules + if len(defaults) > 0 { + e.defaultRules = make([]config.CustomRule, len(defaults)) + copy(e.defaultRules, defaults) + + // Prepend defaults to rules slice (lowest priority, evaluated first) + // Note: rules are evaluated in order, so defaults come first + defaultInternals := make([]internalRule, len(defaults)) + for i, r := range defaults { + defaultInternals[i] = internalRule{ + CustomRule: r, + Source: config.RuleSourceDefault, + } + } + e.rules = append(defaultInternals, e.rules...) + } + + return e +} + // EvalResult contains the result of evaluating a task against rules. type EvalResult struct { // Allow indicates the task passed all rules and should be considered @@ -56,10 +100,24 @@ type EvalResult struct { // AddRule adds a rule to the end of the unified rules slice. // Rules are evaluated in order: DENY stops and rejects, ALLOW continues. +// New rules added at runtime are marked as override source. func (e *Engine) AddRule(rule config.CustomRule) { e.mu.Lock() defer e.mu.Unlock() - e.rules = append(e.rules, rule) + e.rules = append(e.rules, internalRule{ + CustomRule: rule, + Source: config.RuleSourceOverride, + }) +} + +// AddRuleWithSource adds a rule with an explicit source. +func (e *Engine) AddRuleWithSource(rule config.CustomRule, source config.RuleSource) { + e.mu.Lock() + defer e.mu.Unlock() + e.rules = append(e.rules, internalRule{ + CustomRule: rule, + Source: source, + }) } // RemoveRule removes a rule by name from the unified rules slice. @@ -68,7 +126,7 @@ func (e *Engine) RemoveRule(name string) bool { e.mu.Lock() defer e.mu.Unlock() for i, r := range e.rules { - if r.Name == name { + if r.CustomRule.Name == name { e.rules = append(e.rules[:i], e.rules[i+1:]...) return true } @@ -76,16 +134,86 @@ func (e *Engine) RemoveRule(name string) bool { return false } -// ClearRuntimeRules removes all non-persisted rules (resets to config snapshot). +// ClearRuntimeRules removes all override rules, keeping only default and config rules. +// This is called when a run ends to discard run-specific rule overrides. func (e *Engine) ClearRuntimeRules() { e.mu.Lock() defer e.mu.Unlock() - if e.configSnapshot != nil { - e.rules = make([]config.CustomRule, len(e.configSnapshot)) - copy(e.rules, e.configSnapshot) - } else { - e.rules = nil + + // Rebuild rules slice with only default and config rules + var newRules []internalRule + + // Add back default rules first + for _, r := range e.defaultRules { + newRules = append(newRules, internalRule{ + CustomRule: r, + Source: config.RuleSourceDefault, + }) } + + // Add back config rules from snapshot + for _, r := range e.configSnapshot { + newRules = append(newRules, internalRule{ + CustomRule: r, + Source: config.RuleSourceConfig, + }) + } + + e.rules = newRules +} + +// ClearOverrides removes only override rules, keeping default and config rules. +// This is an alias for ClearRuntimeRules for clarity. +func (e *Engine) ClearOverrides() { + e.ClearRuntimeRules() +} + +// ApplyOverrides adds a set of override rules to the engine. +// These rules are added with RuleSourceOverride and will be removed +// when ClearOverrides() is called (typically at run end). +// Rules with the same name as existing rules will be skipped. +func (e *Engine) ApplyOverrides(rules []config.CustomRule) error { + e.mu.Lock() + defer e.mu.Unlock() + + // Build a set of existing rule names for dedup + existingNames := make(map[string]bool) + for _, r := range e.rules { + existingNames[r.CustomRule.Name] = true + } + + // Add override rules + for _, rule := range rules { + if existingNames[rule.Name] { + // Skip duplicate - could also return error but skipping is more lenient + continue + } + // Set default enabled if not specified + if rule.Enabled == nil { + enabled := true + rule.Enabled = &enabled + } + e.rules = append(e.rules, internalRule{ + CustomRule: rule, + Source: config.RuleSourceOverride, + }) + existingNames[rule.Name] = true + } + + return nil +} + +// HasOverrides returns true if there are any override rules currently active. +func (e *Engine) HasOverrides() bool { + e.mu.RLock() + defer e.mu.RUnlock() + + for _, r := range e.rules { + if r.Source == config.RuleSourceOverride { + return true + } + } + return false } // Action represents what a rule does when its conditions match. @@ -98,10 +226,11 @@ const ( ActionAllow Action = "allow" ) -// RuntimeRule extends CustomRule with persistence metadata. +// RuntimeRule extends CustomRule with source and persistence metadata. type RuntimeRule struct { config.CustomRule - Persisted bool `json:"persisted"` // true if rule exists in config snapshot + Source config.RuleSource `json:"source"` // where the rule comes from (default/config/override) + Persisted bool `json:"persisted"` // true if rule exists in config snapshot } // RulesSnapshot contains all rules and settings for API responses. @@ -130,11 +259,12 @@ func (e *Engine) GetSnapshot() RulesSnapshot { RuntimeRules: make([]RuntimeRule, 0), } - // Build unified rules list with persistence status + // Build unified rules list with source and persistence status for _, rule := range e.rules { - persisted := e.isRulePersisted(rule.Name) + persisted := e.isRulePersisted(rule.CustomRule.Name) rr := RuntimeRule{ - CustomRule: rule, + CustomRule: rule.CustomRule, + Source: rule.Source, Persisted: persisted, } snapshot.Rules = append(snapshot.Rules, rr) @@ -161,21 +291,31 @@ func (e *Engine) isRulePersisted(name string) bool { return false } -// isListPersisted checks if the entire rules list matches the config snapshot. -// Returns true only if: -// - The lists have the same length (no additions or deletions) -// - Rules appear in the same order -// - Each rule in the current list matches the corresponding rule in the snapshot +// isListPersisted checks if the rules list matches the config snapshot. +// Returns true only if there are no override rules and the config rules +// match the snapshot (no additions, deletions, or reorders). // Must be called with mu held (at least read lock). func (e *Engine) isListPersisted() bool { + // Count non-default rules (config + override) + var nonDefaultRules []internalRule + for _, r := range e.rules { + if r.Source != config.RuleSourceDefault { + nonDefaultRules = append(nonDefaultRules, r) + } + } + // Different lengths means additions or deletions occurred - if len(e.rules) != len(e.configSnapshot) { + if len(nonDefaultRules) != len(e.configSnapshot) { return false } - // Compare each rule in order - for i, rule := range e.rules { - if rule.Name != e.configSnapshot[i].Name { + // Compare each non-default rule in order + for i, rule := range nonDefaultRules { + if rule.CustomRule.Name != e.configSnapshot[i].Name { + return false + } + // If there are any override rules, not fully persisted + if rule.Source == config.RuleSourceOverride { return false } } @@ -183,37 +323,43 @@ func (e *Engine) isListPersisted() bool { return true } -// GetRuntimeRules returns a copy of all non-persisted rules. +// GetRuntimeRules returns a copy of all override rules (non-persisted, run-specific). func (e *Engine) GetRuntimeRules() []config.CustomRule { e.mu.RLock() defer e.mu.RUnlock() var result []config.CustomRule for _, r := range e.rules { - if !e.isRulePersisted(r.Name) { - result = append(result, r) + if r.Source == config.RuleSourceOverride { + result = append(result, r.CustomRule) } } return result } +// GetOverrideRules returns a copy of all override rules (alias for GetRuntimeRules). +func (e *Engine) GetOverrideRules() []config.CustomRule { + return e.GetRuntimeRules() +} + // PersistRule marks a rule as persisted by adding it to the config snapshot. // The rule must exist in the unified rules slice and not already be persisted. +// When an override rule is persisted, its source changes to config. // Returns the persisted rule on success. func (e *Engine) PersistRule(name string) (*config.CustomRule, error) { e.mu.Lock() defer e.mu.Unlock() // Find the rule in the unified slice - var foundRule *config.CustomRule + var foundIndex int = -1 for i := range e.rules { - if e.rules[i].Name == name { - foundRule = &e.rules[i] + if e.rules[i].CustomRule.Name == name { + foundIndex = i break } } - if foundRule == nil { + if foundIndex == -1 { return nil, fmt.Errorf("rule %q not found", name) } @@ -222,9 +368,14 @@ func (e *Engine) PersistRule(name string) (*config.CustomRule, error) { return nil, fmt.Errorf("rule %q is already persisted", name) } + foundRule := &e.rules[foundIndex].CustomRule + // Add to config snapshot (marks as persisted) e.configSnapshot = append(e.configSnapshot, *foundRule) + // Change source from override to config + e.rules[foundIndex].Source = config.RuleSourceConfig + // Also update settings.Custom for persistence to disk if e.settings == nil { e.settings = &config.RulesSettings{} @@ -234,17 +385,19 @@ func (e *Engine) PersistRule(name string) (*config.CustomRule, error) { return foundRule, nil } -// PersistAllRules marks all non-persisted rules as persisted. +// PersistAllRules marks all override rules as persisted (changes source to config). // Returns the list of rule names that were persisted. func (e *Engine) PersistAllRules() ([]string, error) { e.mu.Lock() defer e.mu.Unlock() var persisted []string - for _, rule := range e.rules { - if !e.isRulePersisted(rule.Name) { - e.configSnapshot = append(e.configSnapshot, rule) - persisted = append(persisted, rule.Name) + for i := range e.rules { + rule := &e.rules[i] + if rule.Source == config.RuleSourceOverride { + e.configSnapshot = append(e.configSnapshot, rule.CustomRule) + rule.Source = config.RuleSourceConfig + persisted = append(persisted, rule.CustomRule.Name) } } @@ -308,9 +461,10 @@ func (e *Engine) GetRule(name string) *RuntimeRule { defer e.mu.RUnlock() for _, rule := range e.rules { - if rule.Name == name { + if rule.CustomRule.Name == name { return &RuntimeRule{ - CustomRule: rule, + CustomRule: rule.CustomRule, + Source: rule.Source, Persisted: e.isRulePersisted(name), } } @@ -326,8 +480,8 @@ func (e *Engine) UpdateRule(name string, enabled bool) error { defer e.mu.Unlock() for i := range e.rules { - if e.rules[i].Name == name { - e.rules[i].Enabled = &enabled + if e.rules[i].CustomRule.Name == name { + e.rules[i].CustomRule.Enabled = &enabled return nil } } @@ -345,7 +499,7 @@ func (e *Engine) ReorderRule(name string, newPosition int) error { // Find the rule first currentIndex := -1 for i, r := range e.rules { - if r.Name == name { + if r.CustomRule.Name == name { currentIndex = i break } @@ -370,14 +524,21 @@ func (e *Engine) ReorderRule(name string, newPosition int) error { e.rules = append(e.rules[:currentIndex], e.rules[currentIndex+1:]...) // Insert at new position - e.rules = append(e.rules[:newPosition], append([]config.CustomRule{rule}, e.rules[newPosition:]...)...) + e.rules = append(e.rules[:newPosition], append([]internalRule{rule}, e.rules[newPosition:]...)...) return nil } // AddRuleWithValidation adds a rule after validating it. // Returns an error if validation fails or if a rule with the same name exists. +// New rules are added as override source by default. func (e *Engine) AddRuleWithValidation(rule config.CustomRule) error { + return e.AddRuleWithValidationAndSource(rule, config.RuleSourceOverride) +} + +// AddRuleWithValidationAndSource adds a rule with explicit source after validating it. +// Returns an error if validation fails or if a rule with the same name exists. +func (e *Engine) AddRuleWithValidationAndSource(rule config.CustomRule, source config.RuleSource) error { // Validate the rule if rule.Name == "" { return fmt.Errorf("rule name is required") @@ -403,7 +564,7 @@ func (e *Engine) AddRuleWithValidation(rule config.CustomRule) error { // Check for duplicate name in unified rules slice for _, r := range e.rules { - if r.Name == rule.Name { + if r.CustomRule.Name == rule.Name { return fmt.Errorf("rule %q already exists", rule.Name) } } @@ -414,7 +575,10 @@ func (e *Engine) AddRuleWithValidation(rule config.CustomRule) error { rule.Enabled = &enabled } - e.rules = append(e.rules, rule) + e.rules = append(e.rules, internalRule{ + CustomRule: rule, + Source: source, + }) return nil } @@ -592,18 +756,19 @@ func (e *Engine) Evaluate(task *beads.Task, inFlight map[string]bool, inFlightTa // Rules are evaluated in order. DENY stops evaluation and rejects. // ALLOW continues to the next rule (or accepts if last). for _, rule := range e.rules { + r := rule.CustomRule // Skip disabled rules - if rule.Enabled != nil && !*rule.Enabled { + if r.Enabled != nil && !*r.Enabled { continue } - if matches := EvaluateCondition(rule.Condition, task); matches { - action := parseAction(rule.Action) + if matches := EvaluateCondition(r.Condition, task); matches { + action := parseAction(r.Action) switch action { case ActionDeny: - reason := rule.Reason + reason := r.Reason if reason == "" { - reason = "Denied by rule: " + rule.Name + reason = "Denied by rule: " + r.Name } return EvalResult{Skip: true, SkipReason: reason} diff --git a/pkg/sandbox/overlay_linux.go b/pkg/sandbox/overlay_linux.go index c3d3cee7..e7987b25 100644 --- a/pkg/sandbox/overlay_linux.go +++ b/pkg/sandbox/overlay_linux.go @@ -461,6 +461,23 @@ func (o *Overlay) GetChanges() ([]FileChange, error) { return nil } + // Check for character device whiteout (fuse-overlayfs convention) + // fuse-overlayfs marks deletions as char devices with major:minor 0:0 + info, err := d.Info() + if err == nil { + if info.Mode()&os.ModeCharDevice != 0 { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + if stat.Rdev == 0 { // major:minor 0:0 = whiteout + changes = append(changes, FileChange{ + Path: relPath, + Type: ChangeDeleted, + }) + return nil + } + } + } + } + // Check if file exists in lower (original) lowerPath := filepath.Join(o.LowerDir, relPath) _, lowerErr := os.Stat(lowerPath) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 3e862e21..22e96007 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -472,6 +472,27 @@ func (s *Scheduler) Kill(agentID string) error { return fmt.Errorf("agent %s not found", agentID) } +// CleanupOverlay cleans up the overlay for a specific task. +// This is safe to call even if the overlay doesn't exist or has already been cleaned up. +// Returns true if an overlay was found and cleaned up, false otherwise. +func (s *Scheduler) CleanupOverlay(taskID string) bool { + s.overlaysMu.Lock() + overlay, exists := s.activeOverlays[taskID] + if exists { + delete(s.activeOverlays, taskID) + } + s.overlaysMu.Unlock() + + if !exists { + return false + } + + // Unmount and cleanup - ignore errors as this is best-effort + _ = overlay.Unmount() + _ = overlay.Cleanup() + return true +} + // CleanupAll synchronously unmounts all active overlays. // This should be called on shutdown to ensure no orphaned FUSE mounts remain. // Returns the number of overlays cleaned and any errors encountered. diff --git a/pkg/types/types.go b/pkg/types/types.go index e5c996eb..04ea54fc 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -5,6 +5,20 @@ // we ensure type safety across package boundaries and eliminate conversion overhead. package types +// AgentStatus represents the current state of an agent. +// This type is used consistently across IPC communication, daemon state tracking, +// lifecycle management, and persistence storage. +type AgentStatus string + +const ( + AgentStatusStarting AgentStatus = "starting" + AgentStatusRunning AgentStatus = "running" + AgentStatusCompleted AgentStatus = "completed" + AgentStatusFailed AgentStatus = "failed" + AgentStatusTimedOut AgentStatus = "timed_out" + AgentStatusCancelled AgentStatus = "cancelled" +) + // MergeStatus represents the current phase of merge processing for an agent's changes. // This type is used consistently across IPC communication, daemon state tracking, // and persistence storage. diff --git a/web/dashboard/src/api/client.ts b/web/dashboard/src/api/client.ts index dfdb3e7e..ca783e3c 100644 --- a/web/dashboard/src/api/client.ts +++ b/web/dashboard/src/api/client.ts @@ -278,6 +278,15 @@ export async function getRuns(filter?: RunListFilter): Promise return fetchJson(endpoint); } +// RuleOverride for run-configured rules +export interface RuleOverride { + name: string; + condition: string; + action: 'deny' | 'allow'; + enabled?: boolean; + reason?: string; +} + // Orchestration types - matches Go backend (pkg/daemon/orchestration_handler.go) export interface StartRunRequest { work_dir: string; @@ -286,6 +295,7 @@ export interface StartRunRequest { max_priority?: number; use_bwrap?: boolean; max_retries?: number; + rule_overrides?: RuleOverride[]; } export interface StartRunResponse { @@ -333,6 +343,56 @@ export async function stopRun(runId: string): Promise { }); } +// Activate/Deactivate API - preferred endpoints for always-active orchestrator model + +export interface ActivateRequest { + work_dir: string; + repo_id?: string; + concurrency?: number; + max_priority?: number; + use_bwrap?: boolean; + max_retries?: number; + rule_overrides?: RuleOverride[]; +} + +export interface ActivateResponse { + success: boolean; + run_id?: string; + error?: string; +} + +export interface DeactivateRequest { + run_id?: string; + repo_path?: string; +} + +export interface DeactivateResponse { + success: boolean; + error?: string; +} + +/** + * Activate the orchestrator for a repository. + * This is the preferred API for the always-active orchestrator model. + */ +export async function activate(request: ActivateRequest): Promise { + return fetchJson('/api/orchestrator/activate', { + method: 'POST', + body: JSON.stringify(request), + }); +} + +/** + * Deactivate the orchestrator. + * Can specify either run_id or repo_path. + */ +export async function deactivate(request: DeactivateRequest): Promise { + return fetchJson('/api/orchestrator/deactivate', { + method: 'POST', + body: JSON.stringify(request), + }); +} + export async function getRunStatus(runId?: string): Promise { const endpoint = runId ? `/api/orchestrator/run?id=${runId}` : '/api/orchestrator/run'; return fetchJson(endpoint); @@ -361,6 +421,9 @@ export interface UpdateConfigRequest { assignee?: string; } +// RuleSource indicates where a rule comes from in the precedence hierarchy +export type RuleSource = 'default' | 'config' | 'override'; + // Unified Rule interface - single format for all rules export interface Rule { name: string; @@ -368,6 +431,7 @@ export interface Rule { action: 'deny' | 'allow'; enabled: boolean; persisted: boolean; + source: RuleSource; } // RulesState contains the unified rules list with list-level persisted flag @@ -578,4 +642,40 @@ export async function getRulesSettings(repoPath: string): Promise(`/api/repos/${encodeURIComponent(repoPath)}/config/rules`); } +// Run configuration API types - matches Go backend (pkg/daemon/orchestration_handler.go) + +export interface RunConfigApiResponse { + concurrency: number; + max_priority: number; + use_bwrap: boolean; + max_retries: number; + error?: string; +} + +export interface RunConfigApiRequest { + concurrency: number; + max_priority: number; + use_bwrap: boolean; + max_retries: number; +} + +/** + * Get the saved run configuration for the current repository. + * Returns defaults if no configuration has been saved. + */ +export async function getRunConfig(): Promise { + return fetchJson('/api/config'); +} + +/** + * Save run configuration for the current repository. + * Broadcasts config_updated WebSocket event to sync across tabs. + */ +export async function saveRunConfig(config: RunConfigApiRequest): Promise { + return fetchJson('/api/config', { + method: 'PUT', + body: JSON.stringify(config), + }); +} + export { ApiError }; diff --git a/web/dashboard/src/components/agents/AgentCard.tsx b/web/dashboard/src/components/agents/AgentCard.tsx index ab0e0a5d..39803ebf 100644 --- a/web/dashboard/src/components/agents/AgentCard.tsx +++ b/web/dashboard/src/components/agents/AgentCard.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; -import { Clock, Zap, DollarSign, XCircle, GitCommit, Archive, ExternalLink, GitMerge, AlertTriangle, ChevronDown, ChevronRight } from 'lucide-react'; -import type { AgentState } from '../../stores/stateStore'; +import { Clock, Zap, DollarSign, XCircle, GitCommit, Archive, ExternalLink, GitMerge, AlertTriangle, ChevronDown, ChevronRight, RefreshCw } from 'lucide-react'; +import type { AgentState, LifecycleState } from '../../stores/stateStore'; import { useStateStore } from '../../stores/stateStore'; import { killAgent, archiveAgent } from '../../api/client'; import { WorkerChainTimeline } from './WorkerChainTimeline'; @@ -13,13 +13,106 @@ interface AgentCardProps { onArchiveToggle?: (agentId: string, archived: boolean) => void; } -const STATUS_COLORS: Record = { +// Format retry badge text: "Attempt N of M" or "Retry N" for infinite retries +const formatRetryBadge = (attempt: number, maxRetries: number): string | null => { + // Only show badge if this is a retry (attempt > 1) + if (attempt <= 1) return null; + + if (maxRetries === -1) { + // Infinite retries: show "Retry N" (attempt 2 = Retry 1, etc.) + return `Retry ${attempt - 1}`; + } + + // Finite retries: show "Attempt N of M+1" (maxRetries + 1 = total attempts) + return `Attempt ${attempt} of ${maxRetries + 1}`; +}; + +// Lifecycle state colors - more detailed states from the state machine +const LIFECYCLE_COLORS: Record = { starting: 'bg-yellow-500', running: 'bg-blue-500', + queued_for_merge: 'bg-purple-500', + merging: 'bg-indigo-500', + resolving: 'bg-amber-500', + validating: 'bg-cyan-500', + repairing: 'bg-orange-500', + merge_failed: 'bg-red-500', completed: 'bg-green-500', failed: 'bg-red-500', - timed_out: 'bg-orange-500', + needs_attention: 'bg-orange-600', cancelled: 'bg-gray-500', + timed_out: 'bg-orange-500', +}; + +// User-friendly display labels for lifecycle states +const LIFECYCLE_LABELS: Record = { + starting: 'Starting', + running: 'Running', + queued_for_merge: 'Queued', + merging: 'Merging', + resolving: 'Resolving', + validating: 'Validating', + repairing: 'Repairing', + merge_failed: 'Merge Failed', + completed: 'Completed', + failed: 'Failed', + needs_attention: 'Needs Attention', + cancelled: 'Cancelled', + timed_out: 'Timed Out', +}; + +// Get display info for lifecycle state with optional context (queue pos, repair attempt) +const getLifecycleDisplay = ( + state: LifecycleState, + queuePos?: number, + repairAttempts?: number, + maxRepairAttempts?: number +): { label: string; color: string } => { + const baseLabel = LIFECYCLE_LABELS[state]; + const color = LIFECYCLE_COLORS[state]; + + // Add context for specific states + if (state === 'queued_for_merge' && queuePos !== undefined && queuePos > 0) { + return { label: `Queued (#${queuePos})`, color }; + } + + if (state === 'repairing' && repairAttempts !== undefined) { + const maxAttempts = maxRepairAttempts ?? 3; + return { label: `Repairing (${repairAttempts}/${maxAttempts})`, color }; + } + + return { label: baseLabel, color }; +}; + +// Derive lifecycle state from legacy fields for backwards compatibility +// Used when lifecycle_state is not present (older backend versions) +const deriveLifecycleState = (agent: AgentState): LifecycleState => { + // First check validation/repair status if merge is complete + if (agent.merge_status === 'merged' || agent.merge_status === 'merged_needs_repair') { + if (agent.validation_status === 'running') return 'validating'; + if (agent.validation_status === 'repairing') return 'repairing'; + if (agent.validation_status === 'failed') return 'needs_attention'; + // If validation passed or not present, fall through + } + + // Check merge status + if (agent.merge_status === 'pending') return 'queued_for_merge'; + if (agent.merge_status === 'acquiring' || agent.merge_status === 'merging') return 'merging'; + if (agent.merge_status === 'resolving') return 'resolving'; + if (agent.merge_status === 'failed') return 'merge_failed'; + + // Check agent status + switch (agent.status) { + case 'starting': return 'starting'; + case 'running': return 'running'; + case 'completed': return 'completed'; + case 'failed': return 'failed'; + case 'cancelled': return 'cancelled'; + case 'timed_out': return 'timed_out'; + } + + // Default fallback + return 'running'; }; const formatElapsedTime = (startTime: string, endTime: string | null): string => { @@ -74,6 +167,13 @@ export const AgentCard: React.FC = ({ a => a.parent_agent_id === agent.id ); + // Check if there's an active resolver for this agent's task + const activeResolver = childAgents.find( + child => (child.status === 'running' || child.status === 'starting') && + !child.task_id.includes('repair') + ); + const hasActiveResolver = Boolean(activeResolver); + // Update elapsed time every second for running agents useEffect(() => { if (agent.status === 'running' || agent.status === 'starting') { @@ -149,9 +249,17 @@ export const AgentCard: React.FC = ({ step => step.status === 'failed' )?.name; + // Use lifecycle_state if available, otherwise derive from legacy fields + const lifecycleState = agent.lifecycle_state ?? deriveLifecycleState(agent); + const lifecycleDisplay = getLifecycleDisplay( + lifecycleState, + agent.merge_queue_pos, + agent.repair_attempts, + 3 // Default max repair attempts + ); + const isRunning = agent.status === 'running' || agent.status === 'starting'; const isFinished = agent.status === 'completed' || agent.status === 'failed' || agent.status === 'timed_out' || agent.status === 'cancelled'; - const statusColor = STATUS_COLORS[agent.status] || 'bg-gray-500'; return (
= ({ {agent.task_id} )} - - {agent.status} + + {hasActiveResolver ? 'Resolving' : lifecycleDisplay.label} + {/* Retry indicator badge - only shown for retried tasks */} + {agent.attempt !== undefined && agent.max_retries !== undefined && formatRetryBadge(agent.attempt, agent.max_retries) && ( + + + {formatRetryBadge(agent.attempt, agent.max_retries)} + + )}

{agent.task_title} @@ -247,12 +365,18 @@ export const AgentCard: React.FC = ({ {formatCost(agent.token_usage.cost_usd)} - {agent.commits > 0 && ( -
- - {agent.commits} -
- )} + {/* Show merge_commits_applied for merged tasks (most accurate), fall back to agent.commits */} + {(() => { + const commitCount = (agent.merge_status === 'merged' || agent.merge_status === 'merged_needs_repair') + ? (agent.merge_commits_applied ?? agent.commits) + : agent.commits; + return commitCount > 0 ? ( +
+ + {commitCount} +
+ ) : null; + })()} {/* Merge and validation status indicators */} diff --git a/web/dashboard/src/components/agents/AgentDetail.tsx b/web/dashboard/src/components/agents/AgentDetail.tsx index 05791f0d..c44bc578 100644 --- a/web/dashboard/src/components/agents/AgentDetail.tsx +++ b/web/dashboard/src/components/agents/AgentDetail.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState } from 'react'; import { Clock, Hash, @@ -11,11 +11,19 @@ import { XCircle, Timer, Activity, + ChevronDown, + ChevronRight, + RefreshCw, + Users, + ExternalLink, } from 'lucide-react'; +import { CommitList } from './CommitList'; import type { AgentState, MergeStatus } from '../../stores/stateStore'; +import { useStateStore } from '../../stores/stateStore'; interface AgentDetailProps { agent: AgentState; + onSelectAgent?: (agentId: string) => void; } // Format duration in a readable way @@ -169,10 +177,114 @@ const Section: React.FC<{ title: string; children: React.ReactNode }> = ({ title ); -export const AgentDetail: React.FC = ({ agent }) => { +// Collapsible section component for expandable content +const CollapsibleSection: React.FC<{ + title: string; + badge?: number; + defaultExpanded?: boolean; + children: React.ReactNode; +}> = ({ title, badge, defaultExpanded = false, children }) => { + const [isExpanded, setIsExpanded] = useState(defaultExpanded); + + return ( +
+ + {isExpanded && ( +
{children}
+ )} +
+ ); +}; + +// Child agent row component - clickable to switch view +const ChildAgentRow: React.FC<{ + childAgent: AgentState; + onSelect?: (agentId: string) => void; +}> = ({ childAgent, onSelect }) => { + const isResolver = !childAgent.task_id.includes('repair'); + const isRepair = childAgent.task_id.includes('repair'); + const isRunning = childAgent.status === 'running' || childAgent.status === 'starting'; + + const statusColors: Record = { + starting: 'bg-yellow-100 dark:bg-yellow-900/30 text-yellow-700 dark:text-yellow-300', + running: 'bg-blue-100 dark:bg-blue-900/30 text-blue-700 dark:text-blue-300', + completed: 'bg-green-100 dark:bg-green-900/30 text-green-700 dark:text-green-300', + failed: 'bg-red-100 dark:bg-red-900/30 text-red-700 dark:text-red-300', + timed_out: 'bg-orange-100 dark:bg-orange-900/30 text-orange-700 dark:text-orange-300', + cancelled: 'bg-gray-100 dark:bg-gray-700 text-gray-600 dark:text-gray-300', + }; + + return ( +
onSelect?.(childAgent.id)} + className={` + flex items-center justify-between p-2 rounded-lg border transition-all + ${onSelect ? 'cursor-pointer hover:border-blue-400 dark:hover:border-blue-500' : ''} + ${isRunning + ? 'border-amber-300 dark:border-amber-600 bg-amber-50 dark:bg-amber-900/20' + : 'border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800' + } + `} + > +
+
+ {isResolver ? ( + + ) : isRepair ? ( + + ) : ( + + )} +
+
+
+ {isResolver ? 'Conflict Resolver' : isRepair ? 'Repair Agent' : 'Child Agent'} +
+
{childAgent.id}
+
+
+
+ + {isRunning ? (isResolver ? 'Resolving' : 'Running') : childAgent.status} + + {onSelect && ( + + )} +
+
+ ); +}; + +export const AgentDetail: React.FC = ({ agent, onSelectAgent }) => { + const agents = useStateStore((state) => state.agents); const hasTokenUsage = agent.token_usage && agent.token_usage.total_tokens > 0; const hasMergeStatus = agent.merge_status && agent.merge_status !== 'pending'; + // Get child agents (resolvers, repair agents) + const childAgents = Object.values(agents).filter( + a => a.parent_agent_id === agent.id + ).sort((a, b) => new Date(a.start_time).getTime() - new Date(b.start_time).getTime()); + + // Get parent agent if this is a child + const parentAgent = agent.parent_agent_id ? agents[agent.parent_agent_id] : null; + return (
{/* Task Information */} @@ -225,6 +337,42 @@ export const AgentDetail: React.FC = ({ agent }) => {
+ {/* Retry Information - shown when this is a retry attempt */} + {agent.attempt !== undefined && agent.attempt > 1 && ( +
+
+
+ + + {agent.max_retries === -1 + ? `Retry ${agent.attempt - 1} (infinite retries enabled)` + : `Attempt ${agent.attempt} of ${(agent.max_retries ?? 0) + 1}`} + +
+ + {/* Show previous failure reason if available */} + {(agent.merge_error || agent.validation_error) && ( +
+
Previous Failure Reason
+
+ {agent.merge_error || agent.validation_error} +
+
+ )} + + {/* Show last repair output if available */} + {agent.last_repair_output && ( +
+
Last Repair Output
+
+ {agent.last_repair_output} +
+
+ )} +
+
+ )} + {/* Timing */}
@@ -284,20 +432,22 @@ export const AgentDetail: React.FC = ({ agent }) => { {/* Work Summary */}
-
- } - label="Files Changed" - value={agent.changes.toString()} - /> - } - label="Commits" - value={(agent.git_commits?.length || agent.commits || 0).toString()} - /> -
+ } + label="Files Changed" + value={agent.changes.toString()} + />
+ {/* Commits - Collapsible list */} + + + + {/* Merge Status */} {hasMergeStatus && (
@@ -309,6 +459,50 @@ export const AgentDetail: React.FC = ({ agent }) => {
)} + {/* Related Agents - show parent and children with click-to-view */} + {(childAgents.length > 0 || parentAgent) && ( +
+
+ {/* Parent agent link */} + {parentAgent && ( +
onSelectAgent?.(parentAgent.id)} + className={` + flex items-center justify-between p-2 rounded-lg border transition-all + ${onSelectAgent ? 'cursor-pointer hover:border-blue-400 dark:hover:border-blue-500' : ''} + border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 + `} + > +
+
+ +
+
+
Parent Worker
+
{parentAgent.id}
+
+
+
+ {parentAgent.task_title} + {onSelectAgent && ( + + )} +
+
+ )} + + {/* Child agents */} + {childAgents.map(child => ( + + ))} +
+
+ )} + {/* Agent IDs */}
@@ -316,17 +510,6 @@ export const AgentDetail: React.FC = ({ agent }) => { {agent.run_id && ( } label="Run ID" value={agent.run_id} monospace /> )} - {agent.parent_agent_id && ( - } label="Parent Agent" value={agent.parent_agent_id} monospace /> - )} - {agent.child_agent_ids && agent.child_agent_ids.length > 0 && ( - } - label="Child Agents" - value={agent.child_agent_ids.join(', ')} - monospace - /> - )}
diff --git a/web/dashboard/src/components/agents/AgentGrid.tsx b/web/dashboard/src/components/agents/AgentGrid.tsx index c1cf1a22..b36cb68f 100644 --- a/web/dashboard/src/components/agents/AgentGrid.tsx +++ b/web/dashboard/src/components/agents/AgentGrid.tsx @@ -1,10 +1,9 @@ import React from 'react'; import { Activity, X, Filter, ListTodo, CheckCircle, XCircle, Archive } from 'lucide-react'; import { AgentCardGroup } from './AgentCardGroup'; -import { RunControlBar } from '../runs/RunControlBar'; import type { AgentGroup } from '../../hooks/useAgentFiltering'; import type { StatusFilter } from '../../hooks/useAgentFiltering'; -import type { Stats, PauseState } from '../../stores/stateStore'; +import type { Stats } from '../../stores/stateStore'; export interface AgentGridProps { /** Grouped agents to display */ @@ -35,32 +34,6 @@ export interface AgentGridProps { selectedBeadTitle?: string | undefined; /** Called when bead filter should be cleared */ onClearBeadFilter?: () => void; - /** Whether the orchestrator is paused */ - isPaused: boolean; - /** Whether the orchestrator is paused by agent */ - isPausedByAgent: boolean; - /** Current pause state */ - pauseState: PauseState; - /** Whether a pause request is in progress */ - isPauseLoading: boolean; - /** Whether a resume request is in progress */ - isResumeLoading: boolean; - /** Current run ID (empty if no active run) */ - currentRunId: string; - /** Whether connected to the backend */ - connected: boolean; - /** Called when pause is clicked */ - onPause: () => void; - /** Called when resume is clicked */ - onResume: () => void; - /** Whether a start run request is in progress */ - isStartingRun: boolean; - /** Whether a stop run request is in progress */ - isStoppingRun: boolean; - /** Called when start run is clicked (opens config dialog) */ - onStartRun: () => void; - /** Called when stop run is clicked */ - onStopRun: () => void; } /** @@ -70,7 +43,6 @@ export interface AgentGridProps { * - Responsive grid layout (1-4 columns based on viewport) * - Empty state display * - Status filter toggles - * - Pause/Resume controls * - Delegation to AgentCardGroup for rendering */ export const AgentGrid: React.FC = ({ @@ -88,19 +60,6 @@ export const AgentGrid: React.FC = ({ selectedBeadId, selectedBeadTitle, onClearBeadFilter, - isPaused, - isPausedByAgent, - pauseState, - isPauseLoading, - isResumeLoading, - currentRunId, - connected, - onPause, - onResume, - isStartingRun, - isStoppingRun, - onStartRun, - onStopRun, }) => { const toggleFilter = (filter: StatusFilter) => { onStatusFilterChange(statusFilter === filter ? 'all' : filter); @@ -229,24 +188,6 @@ export const AgentGrid: React.FC = ({ {archivedAgentCount} - - {/* Run Control Bar (Start/Stop/Pause/Resume) */} - ); diff --git a/web/dashboard/src/components/agents/ValidationStatusBadge.tsx b/web/dashboard/src/components/agents/ValidationStatusBadge.tsx index d087bdf0..928064c9 100644 --- a/web/dashboard/src/components/agents/ValidationStatusBadge.tsx +++ b/web/dashboard/src/components/agents/ValidationStatusBadge.tsx @@ -52,6 +52,20 @@ const STATUS_CONFIG: Record, + label: 'Preparing repair', + bgColor: 'bg-orange-50 dark:bg-orange-900/20', + textColor: 'text-orange-600 dark:text-orange-400', + borderColor: 'border-orange-200 dark:border-orange-800', + }, + spawning_repair: { + icon: , + label: 'Spawning repair agent', + bgColor: 'bg-orange-50 dark:bg-orange-900/20', + textColor: 'text-orange-600 dark:text-orange-400', + borderColor: 'border-orange-200 dark:border-orange-800', + }, repairing: { icon: , label: 'Repair in progress', @@ -82,6 +96,12 @@ export const ValidationStatusBadge: React.FC = ({ if (status === 'failed' && failedStep) { label = `Validation failed (${failedStep})`; } + if (status === 'pending_repair' && repairAttempts !== undefined) { + label = `Preparing repair (${repairAttempts}/${maxRepairAttempts})`; + } + if (status === 'spawning_repair' && repairAttempts !== undefined) { + label = `Spawning repair agent (${repairAttempts}/${maxRepairAttempts})`; + } if (status === 'repairing' && repairAttempts !== undefined) { label = `Repair in progress (${repairAttempts}/${maxRepairAttempts})`; } diff --git a/web/dashboard/src/components/layout/Dashboard.tsx b/web/dashboard/src/components/layout/Dashboard.tsx index ca3dfba7..1069f928 100644 --- a/web/dashboard/src/components/layout/Dashboard.tsx +++ b/web/dashboard/src/components/layout/Dashboard.tsx @@ -3,7 +3,8 @@ import { useStateStore } from '../../stores/stateStore'; import type { RunConfig } from '../../stores/stateStore'; import { useWebSocket, useAgentFiltering, useResizablePane, useResizableWidth } from '../../hooks'; import type { StatusFilter } from '../../hooks'; -import { pauseOrch, resumeOrch, getState, getRepositories, activateRepository, getRuns, startRun, stopRun } from '../../api/client'; +import { pauseOrch, resumeOrch, getState, getRepositories, activateRepository, getRuns, startRun, stopRun, saveRunConfig, getRules } from '../../api/client'; +import type { Rule } from '../../api/client'; import { BeadsPane } from '../beads/BeadsPane'; import { DashboardHeader } from './DashboardHeader'; import { TerminalPanel } from './TerminalPanel'; @@ -22,6 +23,11 @@ export const Dashboard: React.FC = () => { // Orchestrator control state const [isPauseLoading, setIsPauseLoading] = useState(false); const [isResumeLoading, setIsResumeLoading] = useState(false); + const [isSavingConfig, setIsSavingConfig] = useState(false); + const [configDialogMode, setConfigDialogMode] = useState<'start' | 'configure'>('start'); + // Rules state for run config dialog + const [configRulesForDialog, setConfigRulesForDialog] = useState([]); + const [isLoadingConfigRules, setIsLoadingConfigRules] = useState(false); // Theme state const [isDark, setIsDark] = useState(() => { @@ -117,8 +123,6 @@ export const Dashboard: React.FC = () => { // State from store const agents = useStateStore((state) => state.agents); const stats = useStateStore((state) => state.stats); - const isPaused = useStateStore((state) => state.isPaused); - const isPausedByAgent = useStateStore((state) => state.isPausedByAgent); const pauseState = useStateStore((state) => state.pauseState); const selectedAgentId = useStateStore((state) => state.selectedAgentId); const setSelectedAgent = useStateStore((state) => state.setSelectedAgent); @@ -134,20 +138,34 @@ export const Dashboard: React.FC = () => { const activeRunId = useStateStore((state) => state.activeRunId); const isRunsLoading = useStateStore((state) => state.isRunsLoading); const setRuns = useStateStore((state) => state.setRuns); + const mergeRuns = useStateStore((state) => state.mergeRuns); const setActiveRunId = useStateStore((state) => state.setActiveRunId); const setRunsLoading = useStateStore((state) => state.setRunsLoading); const selectedBeadId = useStateStore((state) => state.selectedBeadId); const setSelectedBead = useStateStore((state) => state.setSelectedBead); const tasks = useStateStore((state) => state.tasks); + // Orchestrator state + const orchestratorState = useStateStore((state) => state.orchestratorState); + const activeAgentCount = useStateStore((state) => state.activeAgentCount); // Run control state const isStartingRun = useStateStore((state) => state.isStartingRun); const isStoppingRun = useStateStore((state) => state.isStoppingRun); const runConfig = useStateStore((state) => state.runConfig); const showRunConfigDialog = useStateStore((state) => state.showRunConfigDialog); - const setStartingRun = useStateStore((state) => state.setStartingRun); - const setStoppingRun = useStateStore((state) => state.setStoppingRun); const setShowRunConfigDialog = useStateStore((state) => state.setShowRunConfigDialog); - const setCurrentRunId = useStateStore((state) => state.setCurrentRunId); + const setRunConfig = useStateStore((state) => state.setRunConfig); + const loadSavedConfig = useStateStore((state) => state.loadSavedConfig); + // Optimistic update actions + const startOptimisticRun = useStateStore((state) => state.startOptimisticRun); + const confirmRunStarted = useStateStore((state) => state.confirmRunStarted); + const rollbackRunStart = useStateStore((state) => state.rollbackRunStart); + const startOptimisticStop = useStateStore((state) => state.startOptimisticStop); + const confirmRunStopped = useStateStore((state) => state.confirmRunStopped); + const rollbackRunStop = useStateStore((state) => state.rollbackRunStop); + const startOptimisticPause = useStateStore((state) => state.startOptimisticPause); + const rollbackPause = useStateStore((state) => state.rollbackPause); + const startOptimisticResume = useStateStore((state) => state.startOptimisticResume); + const rollbackResume = useStateStore((state) => state.rollbackResume); // Use the agent filtering hook const { groupedAgents, archivedCount } = useAgentFiltering({ @@ -182,6 +200,9 @@ export const Dashboard: React.FC = () => { } setRepositories(repoResponse.repositories, activeId); + + // Load saved run configuration + await loadSavedConfig(); } catch (error) { console.error('Failed to load initial state:', error); // Attempt to load repositories separately if combined fetch failed @@ -199,7 +220,7 @@ export const Dashboard: React.FC = () => { }; loadInitialState(); - }, [syncState, setRepositories]); + }, [syncState, setRepositories, loadSavedConfig]); // Load runs when activeRepoId changes useEffect(() => { @@ -209,7 +230,10 @@ export const Dashboard: React.FC = () => { try { setRunsLoading(true); const response = await getRuns({ repo_id: activeRepoId, limit: 50 }); - setRuns(response.runs); + // Use mergeRuns instead of setRuns to avoid race condition where + // a WebSocket run:started event adds a run before this API call completes. + // mergeRuns preserves any runs added via WebSocket that aren't yet in the API response. + mergeRuns(response.runs); setActiveRunId(''); } catch (error) { console.error('Failed to load runs:', error); @@ -220,18 +244,23 @@ export const Dashboard: React.FC = () => { }; loadRuns(); - }, [activeRepoId, setRuns, setActiveRunId, setRunsLoading]); + }, [activeRepoId, mergeRuns, setRuns, setActiveRunId, setRunsLoading]); - // Event handlers + // Event handlers - using optimistic updates const handlePause = async () => { if (isPauseLoading) return; + // Optimistic update - immediately show paused state + startOptimisticPause(); + setIsPauseLoading(true); + try { - setIsPauseLoading(true); await pauseOrch(); - // State update comes from WebSocket event + // WebSocket event will confirm the pause with final state } catch (error) { console.error('Failed to pause orchestration:', error); + // Rollback to previous state on error + rollbackPause(error instanceof Error ? error.message : 'Failed to pause'); } finally { setIsPauseLoading(false); } @@ -240,12 +269,17 @@ export const Dashboard: React.FC = () => { const handleResume = async () => { if (isResumeLoading) return; + // Optimistic update - immediately show resumed state + startOptimisticResume(); + setIsResumeLoading(true); + try { - setIsResumeLoading(true); await resumeOrch(); - // State update comes from WebSocket event + // WebSocket event will confirm the resume with final state } catch (error) { console.error('Failed to resume orchestration:', error); + // Rollback to previous state on error + rollbackResume(error instanceof Error ? error.message : 'Failed to resume'); } finally { setIsResumeLoading(false); } @@ -284,15 +318,60 @@ export const Dashboard: React.FC = () => { } }, [agents, handleSelectAgent]); - // Run control handlers - const handleOpenRunConfig = useCallback(() => { - setShowRunConfigDialog(true); - }, [setShowRunConfigDialog]); + // Load rules for the run config dialog + const loadRulesForDialog = useCallback(async () => { + const activeRepo = repositories.find((r) => r.id === activeRepoId); + if (!activeRepo?.path) return; + setIsLoadingConfigRules(true); + try { + const response = await getRules(activeRepo.path); + setConfigRulesForDialog(response.rules); + } catch (error) { + console.error('Failed to load rules for dialog:', error); + setConfigRulesForDialog([]); + } finally { + setIsLoadingConfigRules(false); + } + }, [repositories, activeRepoId]); + + // Run control handlers const handleCloseRunConfig = useCallback(() => { setShowRunConfigDialog(false); + setConfigDialogMode('start'); }, [setShowRunConfigDialog]); + const handleConfigure = useCallback(() => { + setConfigDialogMode('configure'); + loadRulesForDialog(); + setShowRunConfigDialog(true); + }, [setShowRunConfigDialog, loadRulesForDialog]); + + const handleSaveConfig = useCallback(async (config: RunConfig) => { + setIsSavingConfig(true); + try { + const response = await saveRunConfig({ + concurrency: config.concurrency, + max_priority: config.max_priority, + use_bwrap: config.use_bwrap, + max_retries: config.max_retries, + }); + + if (!response.error) { + // Update local state (WebSocket event will also update it for cross-tab sync) + setRunConfig(config); + setShowRunConfigDialog(false); + setConfigDialogMode('start'); + } else { + console.error('Failed to save config:', response.error); + } + } catch (error) { + console.error('Failed to save config:', error); + } finally { + setIsSavingConfig(false); + } + }, [setRunConfig, setShowRunConfigDialog]); + const handleStartRun = useCallback(async (config: RunConfig) => { const activeRepo = repositories.find((repo) => repo.id === activeRepoId); if (!activeRepo) { @@ -300,8 +379,13 @@ export const Dashboard: React.FC = () => { return; } + // Generate a temporary run ID for optimistic update + const tempRunId = `temp-${Date.now()}`; + + // Optimistic update - immediately show starting state + startOptimisticRun(tempRunId); + try { - setStartingRun(true); const response = await startRun({ work_dir: activeRepo.path, repo_id: activeRepo.id, @@ -312,20 +396,22 @@ export const Dashboard: React.FC = () => { }); if (response.success && response.run_id) { - setCurrentRunId(response.run_id); + // Confirm with the actual run ID + confirmRunStarted(response.run_id); setShowRunConfigDialog(false); // Refresh state after starting run const state = await getState(); syncState(state); } else { - console.error('Failed to start run:', response.error); + // Rollback on failure response + rollbackRunStart(response.error || 'Failed to start run'); } } catch (error) { console.error('Failed to start run:', error); - } finally { - setStartingRun(false); + // Rollback on exception + rollbackRunStart(error instanceof Error ? error.message : 'Failed to start run'); } - }, [activeRepoId, repositories, setStartingRun, setCurrentRunId, setShowRunConfigDialog, syncState]); + }, [activeRepoId, repositories, startOptimisticRun, confirmRunStarted, rollbackRunStart, setShowRunConfigDialog, syncState]); const handleStopRun = useCallback(async () => { const runId = useStateStore.getState().currentRunId; @@ -334,28 +420,76 @@ export const Dashboard: React.FC = () => { return; } + // Optimistic update - immediately show stopping state + startOptimisticStop(); + try { - setStoppingRun(true); const response = await stopRun(runId); if (response.success) { - // The currentRunId will be cleared by run:completed WebSocket event + // Confirm the stop + confirmRunStopped(); // Refresh state after stopping run const state = await getState(); syncState(state); } else { - console.error('Failed to stop run:', response.error); + // Rollback on failure response + rollbackRunStop(response.error || 'Failed to stop run'); } } catch (error) { console.error('Failed to stop run:', error); - } finally { - setStoppingRun(false); + // Rollback on exception + rollbackRunStop(error instanceof Error ? error.message : 'Failed to stop run'); + } + }, [startOptimisticStop, confirmRunStopped, rollbackRunStop, syncState]); + + // Activate orchestrator - starts with current run config + const handleActivate = useCallback(async () => { + const activeRepo = repositories.find((repo) => repo.id === activeRepoId); + if (!activeRepo) { + console.error('No active repository selected'); + return; + } + + // Generate a temporary run ID for optimistic update + const tempRunId = `temp-${Date.now()}`; + + // Optimistic update - immediately show activating state + startOptimisticRun(tempRunId); + + try { + const config = useStateStore.getState().runConfig; + const response = await startRun({ + work_dir: activeRepo.path, + repo_id: activeRepo.id, + concurrency: config.concurrency, + max_priority: config.max_priority, + use_bwrap: config.use_bwrap, + max_retries: config.max_retries, + }); + + if (response.success && response.run_id) { + // Confirm with the actual run ID + confirmRunStarted(response.run_id); + // Refresh state after activating + const state = await getState(); + syncState(state); + } else { + // Rollback on failure response + rollbackRunStart(response.error || 'Failed to activate orchestrator'); + } + } catch (error) { + console.error('Failed to activate orchestrator:', error); + // Rollback on exception + rollbackRunStart(error instanceof Error ? error.message : 'Failed to activate orchestrator'); } - }, [setStoppingRun, syncState]); + }, [activeRepoId, repositories, startOptimisticRun, confirmRunStarted, rollbackRunStart, syncState]); + + // Deactivate orchestrator - same as stop run + const handleDeactivate = handleStopRun; const selectedAgent = selectedAgentId ? agents[selectedAgentId] : null; const totalAgentCount = Object.keys(agents).length; - const currentRunId = useStateStore((state) => state.currentRunId); return (
@@ -371,6 +505,18 @@ export const Dashboard: React.FC = () => { activeRunId={activeRunId} isRunsLoading={isRunsLoading} onRunSelect={setActiveRunId} + orchestratorState={orchestratorState} + activeAgentCount={activeAgentCount} + pauseState={pauseState} + isActivating={isStartingRun} + isDeactivating={isStoppingRun} + isPauseLoading={isPauseLoading} + isResumeLoading={isResumeLoading} + onActivate={handleActivate} + onDeactivate={handleDeactivate} + onPause={handlePause} + onResume={handleResume} + onConfigure={handleConfigure} stats={stats} /> @@ -410,19 +556,6 @@ export const Dashboard: React.FC = () => { selectedBeadId={selectedBeadId} selectedBeadTitle={selectedBeadId ? tasks[selectedBeadId]?.title : undefined} onClearBeadFilter={() => setSelectedBead(null)} - isPaused={isPaused} - isPausedByAgent={isPausedByAgent} - pauseState={pauseState} - isPauseLoading={isPauseLoading} - isResumeLoading={isResumeLoading} - currentRunId={currentRunId} - connected={connected} - onPause={handlePause} - onResume={handleResume} - isStartingRun={isStartingRun} - isStoppingRun={isStoppingRun} - onStartRun={handleOpenRunConfig} - onStopRun={handleStopRun} />
@@ -444,6 +577,7 @@ export const Dashboard: React.FC = () => { isResizing={isResizing} onResizeStart={handleResizeStart} onClose={() => setSelectedAgent(null)} + onSelectAgent={handleSelectAgent} /> )} @@ -454,9 +588,14 @@ export const Dashboard: React.FC = () => { isOpen={showRunConfigDialog} onClose={handleCloseRunConfig} onStart={handleStartRun} + onSave={handleSaveConfig} isStarting={isStartingRun} + isSaving={isSavingConfig} initialConfig={runConfig} repoName={repositories.find((r) => r.id === activeRepoId)?.name} + mode={configDialogMode} + configRules={configRulesForDialog} + isLoadingRules={isLoadingConfigRules} /> ); diff --git a/web/dashboard/src/components/layout/DashboardHeader.tsx b/web/dashboard/src/components/layout/DashboardHeader.tsx index 59d5f0cc..80c37d22 100644 --- a/web/dashboard/src/components/layout/DashboardHeader.tsx +++ b/web/dashboard/src/components/layout/DashboardHeader.tsx @@ -8,9 +8,10 @@ import { Moon, } from 'lucide-react'; import type { Repository, Run } from '../../api/client'; -import type { Stats } from '../../stores/stateStore'; +import type { Stats, OrchestratorState, PauseState } from '../../stores/stateStore'; import { RepoSelector } from './RepoSelector'; import { RunSelector } from './RunSelector'; +import { OrchestratorStateIndicator } from './OrchestratorStateIndicator'; export interface DashboardHeaderProps { // Connection & state @@ -32,6 +33,20 @@ export interface DashboardHeaderProps { isRunsLoading: boolean; onRunSelect: (runId: string) => void; + // Orchestrator state + orchestratorState: OrchestratorState; + activeAgentCount: number; + pauseState: PauseState; + isActivating: boolean; + isDeactivating: boolean; + isPauseLoading: boolean; + isResumeLoading: boolean; + onActivate: () => void; + onDeactivate: () => void; + onPause: () => void; + onResume: () => void; + onConfigure: () => void; + // Stats stats: Stats; } @@ -69,6 +84,18 @@ export const DashboardHeader: React.FC = ({ activeRunId, isRunsLoading, onRunSelect, + orchestratorState, + activeAgentCount, + pauseState, + isActivating, + isDeactivating, + isPauseLoading, + isResumeLoading, + onActivate, + onDeactivate, + onPause, + onResume, + onConfigure, stats, }) => { return ( @@ -88,6 +115,30 @@ export const DashboardHeader: React.FC = ({ isLoading={isRepoSwitching} disabled={!connected} /> + + {/* Separator */} +
+ + {/* Orchestrator State Indicator */} + + + {/* Separator */} +
+ = ({ isLoading={isRunsLoading} disabled={!connected || isRepoSwitching} /> -
-
- - {connected ? 'Connected' : 'Disconnected'} - -
+ )} + + {showConfigureButton && ( + + )} + + {showPauseButton && ( + + )} + + {showResumeButton && ( + + )} + + {showDeactivateButton && ( + + )} +
+
+ ); +}; diff --git a/web/dashboard/src/components/layout/TerminalPanel.tsx b/web/dashboard/src/components/layout/TerminalPanel.tsx index 8df7b0af..b129e489 100644 --- a/web/dashboard/src/components/layout/TerminalPanel.tsx +++ b/web/dashboard/src/components/layout/TerminalPanel.tsx @@ -19,6 +19,8 @@ export interface TerminalPanelProps { onResizeStart: (e: React.MouseEvent) => void; /** Called when close button is clicked */ onClose: () => void; + /** Called when a different agent is selected (e.g., clicking on child agent) */ + onSelectAgent?: (agentId: string) => void; } /** @@ -36,6 +38,7 @@ export const TerminalPanel: React.FC = ({ isResizing, onResizeStart, onClose, + onSelectAgent, }) => { const [activeTab, setActiveTab] = useState('feed'); @@ -157,7 +160,7 @@ export const TerminalPanel: React.FC = ({
)} - {activeTab === 'detail' && } + {activeTab === 'detail' && } diff --git a/web/dashboard/src/components/rules/EditRuleModal.test.tsx b/web/dashboard/src/components/rules/EditRuleModal.test.tsx index 716629f3..8b4476a6 100644 --- a/web/dashboard/src/components/rules/EditRuleModal.test.tsx +++ b/web/dashboard/src/components/rules/EditRuleModal.test.tsx @@ -16,6 +16,7 @@ const createMockRule = (overrides: Partial = {}): Rule => ({ action: 'deny', enabled: true, persisted: true, + source: 'config', ...overrides, }); @@ -415,6 +416,7 @@ describe('EditRuleModal', () => { action: 'deny', enabled: true, persisted: false, // Always false after edit + source: 'config', }); }); diff --git a/web/dashboard/src/components/rules/EditRuleModal.tsx b/web/dashboard/src/components/rules/EditRuleModal.tsx index 7d406636..ae626dce 100644 --- a/web/dashboard/src/components/rules/EditRuleModal.tsx +++ b/web/dashboard/src/components/rules/EditRuleModal.tsx @@ -177,12 +177,14 @@ export const EditRuleModal: React.FC = ({ } // Create updated rule with persisted=false since this is an in-memory edit + // Keep the original source if editing, otherwise default to override const updatedRule: Rule = { name: rule.name, conditions: validConditions, action, enabled, persisted: false, + source: rule.source || 'override', }; onSave(updatedRule); diff --git a/web/dashboard/src/components/rules/RulesPanel.test.tsx b/web/dashboard/src/components/rules/RulesPanel.test.tsx index 4e9ea88b..3a8862d1 100644 --- a/web/dashboard/src/components/rules/RulesPanel.test.tsx +++ b/web/dashboard/src/components/rules/RulesPanel.test.tsx @@ -32,6 +32,7 @@ const createMockRule = (overrides: Partial = {}): Rule => ({ action: 'deny', enabled: true, persisted: true, + source: 'config', ...overrides, }); diff --git a/web/dashboard/src/components/runs/RunConfigDialog.tsx b/web/dashboard/src/components/runs/RunConfigDialog.tsx index 8d3f39b5..bb6a59dd 100644 --- a/web/dashboard/src/components/runs/RunConfigDialog.tsx +++ b/web/dashboard/src/components/runs/RunConfigDialog.tsx @@ -1,45 +1,227 @@ import React, { useState, useEffect, useRef } from 'react'; -import { X, Settings, Loader2 } from 'lucide-react'; -import type { RunConfig } from '../../stores/stateStore'; +import { X, Settings, Loader2, ChevronDown, ChevronUp, Plus, Trash2, FileText, Zap } from 'lucide-react'; +import type { RunConfig, RuleOverride } from '../../stores/stateStore'; import { DEFAULT_RUN_CONFIG } from '../../stores/stateStore'; +import type { Rule } from '../../api/client'; + +type DialogMode = 'start' | 'configure'; + +// Extended rule for the dialog that tracks config rules and their override state +interface DialogRule { + name: string; + conditions: string[]; + action: 'deny' | 'allow'; + enabled: boolean; + source: 'config' | 'override'; // Where the rule came from + modified: boolean; // Whether it's been modified from config +} interface RunConfigDialogProps { isOpen: boolean; onClose: () => void; onStart: (config: RunConfig) => void; + onSave?: (config: RunConfig) => void; isStarting: boolean; + isSaving?: boolean; initialConfig?: RunConfig; repoName?: string | undefined; + mode?: DialogMode; + configRules?: Rule[]; // Rules loaded from config file + isLoadingRules?: boolean; +} + +// Default empty rule for new rule creation +const DEFAULT_RULE: RuleOverride = { + name: '', + condition: '', + action: 'deny', + enabled: true, +}; + +// Helper to convert config rules to dialog rules +function configRulesToDialogRules(configRules: Rule[]): DialogRule[] { + return configRules.map((rule) => ({ + name: rule.name, + conditions: rule.conditions, + action: rule.action, + enabled: rule.enabled, + source: 'config' as const, + modified: false, + })); +} + +// Helper to convert rule overrides to dialog rules +function overridesToDialogRules(overrides: RuleOverride[]): DialogRule[] { + return overrides.map((override) => ({ + name: override.name, + conditions: [override.condition], + action: override.action, + enabled: override.enabled !== false, + source: 'override' as const, + modified: false, + })); +} + +// Helper to convert dialog rules back to rule overrides for the config +function dialogRulesToOverrides(dialogRules: DialogRule[], originalConfigRules: Rule[]): RuleOverride[] { + const overrides: RuleOverride[] = []; + + for (const dialogRule of dialogRules) { + if (dialogRule.source === 'override') { + // Runtime-only rules always get included + overrides.push({ + name: dialogRule.name, + condition: dialogRule.conditions.join(' AND '), + action: dialogRule.action, + enabled: dialogRule.enabled, + }); + } else if (dialogRule.modified) { + // Config rules that were modified become overrides + const originalRule = originalConfigRules.find((r) => r.name === dialogRule.name); + if (originalRule) { + // Only include if actually different from config + if (originalRule.enabled !== dialogRule.enabled || + originalRule.action !== dialogRule.action) { + overrides.push({ + name: dialogRule.name, + condition: dialogRule.conditions.join(' AND '), + action: dialogRule.action, + enabled: dialogRule.enabled, + }); + } + } + } + } + + return overrides; } export const RunConfigDialog: React.FC = ({ isOpen, onClose, onStart, + onSave, isStarting, + isSaving = false, initialConfig = DEFAULT_RUN_CONFIG, repoName, + mode = 'start', + configRules = [], + isLoadingRules = false, }) => { const [config, setConfig] = useState(initialConfig); + const [dialogRules, setDialogRules] = useState([]); + const [rulesExpanded, setRulesExpanded] = useState(false); + const [newRule, setNewRule] = useState(DEFAULT_RULE); + const [showAddRule, setShowAddRule] = useState(false); const dialogRef = useRef(null); - // Reset config when dialog opens + // Reset config and merge rules when dialog opens useEffect(() => { if (isOpen) { setConfig(initialConfig); + + // Merge config rules with any existing overrides + const configDialogRules = configRulesToDialogRules(configRules); + const overrideDialogRules = overridesToDialogRules(initialConfig.rule_overrides || []); + + // Apply overrides to config rules + const mergedRules = configDialogRules.map((configRule) => { + const override = overrideDialogRules.find((o) => o.name === configRule.name); + if (override) { + return { + ...configRule, + enabled: override.enabled, + action: override.action, + modified: true, + }; + } + return configRule; + }); + + // Add any override-only rules (not in config) + const configRuleNames = new Set(configDialogRules.map((r) => r.name)); + const additionalOverrides = overrideDialogRules.filter((o) => !configRuleNames.has(o.name)); + + setDialogRules([...mergedRules, ...additionalOverrides]); + setRulesExpanded(configRules.length > 0 || (initialConfig.rule_overrides?.length ?? 0) > 0); + setShowAddRule(false); + setNewRule(DEFAULT_RULE); + } + }, [isOpen, initialConfig, configRules]); + + // Helper to add a new rule + const handleAddRule = () => { + if (!newRule.name || !newRule.condition) return; + const newDialogRule: DialogRule = { + name: newRule.name, + conditions: [newRule.condition], + action: newRule.action, + enabled: newRule.enabled !== false, + source: 'override', + modified: false, + }; + setDialogRules([...dialogRules, newDialogRule]); + setNewRule(DEFAULT_RULE); + setShowAddRule(false); + }; + + // Helper to remove a rule (only for override/runtime rules) + const handleRemoveRule = (index: number) => { + const rule = dialogRules[index]; + if (rule && rule.source === 'override') { + const newRules = [...dialogRules]; + newRules.splice(index, 1); + setDialogRules(newRules); } - }, [isOpen, initialConfig]); + }; + + // Helper to toggle a rule's enabled state + const handleToggleRule = (index: number) => { + const newRules = [...dialogRules]; + const currentRule = newRules[index]; + if (currentRule) { + newRules[index] = { + ...currentRule, + enabled: !currentRule.enabled, + modified: currentRule.source === 'config' ? true : currentRule.modified, + }; + setDialogRules(newRules); + } + }; + + // Helper to reset a config rule to its original state + const handleResetRule = (index: number) => { + const rule = dialogRules[index]; + if (rule && rule.source === 'config') { + const originalRule = configRules.find((r) => r.name === rule.name); + if (originalRule) { + const newRules = [...dialogRules]; + newRules[index] = { + name: originalRule.name, + conditions: originalRule.conditions, + action: originalRule.action, + enabled: originalRule.enabled, + source: 'config', + modified: false, + }; + setDialogRules(newRules); + } + } + }; + + const isProcessing = isStarting || isSaving; // Handle escape key useEffect(() => { const handleEscape = (event: KeyboardEvent) => { - if (event.key === 'Escape' && isOpen && !isStarting) { + if (event.key === 'Escape' && isOpen && !isProcessing) { onClose(); } }; document.addEventListener('keydown', handleEscape); return () => document.removeEventListener('keydown', handleEscape); - }, [isOpen, isStarting, onClose]); + }, [isOpen, isProcessing, onClose]); // Handle click outside useEffect(() => { @@ -47,7 +229,7 @@ export const RunConfigDialog: React.FC = ({ if ( dialogRef.current && !dialogRef.current.contains(event.target as Node) && - !isStarting + !isProcessing ) { onClose(); } @@ -57,14 +239,35 @@ export const RunConfigDialog: React.FC = ({ document.addEventListener('mousedown', handleClickOutside); return () => document.removeEventListener('mousedown', handleClickOutside); } - }, [isOpen, isStarting, onClose]); + }, [isOpen, isProcessing, onClose]); if (!isOpen) return null; + // Build the final config with rule overrides + const buildFinalConfig = (): RunConfig => { + const ruleOverrides = dialogRulesToOverrides(dialogRules, configRules); + return { + ...config, + rule_overrides: ruleOverrides, + }; + }; + const handleStart = () => { - onStart(config); + onStart(buildFinalConfig()); }; + const handleSave = () => { + if (onSave) { + onSave(buildFinalConfig()); + } + }; + + // Count active rules and modified rules + const activeRuleCount = dialogRules.filter((r) => r.enabled).length; + const configRuleCount = dialogRules.filter((r) => r.source === 'config').length; + const overrideRuleCount = dialogRules.filter((r) => r.source === 'override').length; + const modifiedRuleCount = dialogRules.filter((r) => r.source === 'config' && r.modified).length; + return (
= ({

- Start Run + {mode === 'configure' ? 'Configure Run Settings' : 'Start Run'}

+ + {/* Rules Section */} +
+ + + {rulesExpanded && ( +
+ {/* Loading state */} + {isLoadingRules && dialogRules.length === 0 && ( +
+ + Loading rules... +
+ )} + + {/* Config rules section */} + {configRuleCount > 0 && ( +
+
+ + From Config ({configRuleCount}) +
+ {dialogRules.filter((r) => r.source === 'config').map((rule) => { + const originalIndex = dialogRules.findIndex((r) => r.name === rule.name); + return ( +
+
+
+ + {rule.name} + + {rule.modified && ( + + modified + + )} +
+
+ {rule.modified && ( + + )} + +
+
+
+ + {rule.action.toUpperCase()} + + + {rule.conditions.join(' AND ')} + +
+
+ ); + })} +
+ )} + + {/* Runtime override rules section */} + {overrideRuleCount > 0 && ( +
+
+ + Runtime Only ({overrideRuleCount}) +
+ {dialogRules.filter((r) => r.source === 'override').map((rule) => { + const originalIndex = dialogRules.findIndex((r) => r.name === rule.name); + return ( +
+
+ + {rule.name} + +
+ + +
+
+
+ + {rule.action.toUpperCase()} + + + {rule.conditions.join(' AND ')} + +
+
+ ); + })} +
+ )} + + {/* Empty state */} + {!isLoadingRules && dialogRules.length === 0 && ( +
+

No rules configured

+

Add rules below to filter tasks for this run

+
+ )} + + {/* Add new rule form */} + {showAddRule ? ( +
+
+ setNewRule({ ...newRule, name: e.target.value })} + className="w-full px-2 py-1 text-sm border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100" + /> + setNewRule({ ...newRule, condition: e.target.value })} + className="w-full px-2 py-1 text-sm border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 font-mono" + /> +
+ +
+ + +
+
+
+ ) : ( + + )} +
+ )} +
{/* Footer */}
- + {mode === 'configure' ? ( + + ) : ( + + )}
diff --git a/web/dashboard/src/components/runs/RunControlBar.tsx b/web/dashboard/src/components/runs/RunControlBar.tsx deleted file mode 100644 index 1ae6461d..00000000 --- a/web/dashboard/src/components/runs/RunControlBar.tsx +++ /dev/null @@ -1,234 +0,0 @@ -import React from 'react'; -import { Play, Square, Pause, Loader2 } from 'lucide-react'; -import type { PauseState } from '../../stores/stateStore'; - -export interface RunControlBarProps { - /** Whether there's an active run */ - hasActiveRun: boolean; - /** Current run ID (for display) */ - currentRunId: string; - /** Whether connected to the backend */ - connected: boolean; - /** Whether the orchestrator is paused */ - isPaused: boolean; - /** Whether paused by agent (auto-pause) */ - isPausedByAgent: boolean; - /** Current pause state */ - pauseState: PauseState; - /** Loading states */ - isPauseLoading: boolean; - isResumeLoading: boolean; - isStartingRun: boolean; - isStoppingRun: boolean; - /** Event handlers */ - onPause: () => void; - onResume: () => void; - onStartRun: () => void; - onStopRun: () => void; -} - -/** - * Get the display text for the pause button based on pause state - */ -function getPauseButtonText( - pauseState: PauseState, - isPauseLoading: boolean, - isResumeLoading: boolean -): string { - if (isPauseLoading) return 'Pausing...'; - if (isResumeLoading) return 'Resuming...'; - - switch (pauseState) { - case 'paused_user': - return 'Paused'; - case 'paused_agent': - return 'Agent Active...'; - case 'paused_both': - return 'Paused (agent)'; - default: - return 'Pause'; - } -} - -/** - * Get the tooltip text for the pause/resume button - */ -function getPauseButtonTooltip( - pauseState: PauseState, - isPaused: boolean, - hasActiveRun: boolean -): string { - if (!hasActiveRun) { - return 'No active run'; - } - if (isPaused) { - switch (pauseState) { - case 'paused_user': - return 'Orchestrator paused by user. Click to resume spawning new agents.'; - case 'paused_agent': - return 'Orchestrator paused while agent is active (resolving conflicts or repairing). Will auto-resume when complete.'; - case 'paused_both': - return 'Orchestrator paused by user while agent is active. Click to allow new agents after agent completes.'; - default: - return 'Click to resume spawning new agents'; - } - } - return 'Pause spawning of new agents. Running agents will continue until completion.'; -} - -/** - * Run control bar component that provides Start/Stop and Pause/Resume controls. - * This component is designed to be placed in the status filter bar area. - */ -export const RunControlBar: React.FC = ({ - hasActiveRun, - currentRunId: _currentRunId, - connected, - isPaused, - isPausedByAgent, - pauseState, - isPauseLoading, - isResumeLoading, - isStartingRun, - isStoppingRun, - onPause, - onResume, - onStartRun, - onStopRun, -}) => { - const buttonText = getPauseButtonText(pauseState, isPauseLoading, isResumeLoading); - const buttonTooltip = getPauseButtonTooltip(pauseState, isPaused, hasActiveRun); - const isPauseDisabled = !hasActiveRun || !connected; - - // Start Run Button - const StartButton = () => ( - - ); - - // Stop Run Button - const StopButton = () => ( - - ); - - // Pause/Resume Button - const PauseResumeButton = () => { - if (isPaused) { - return ( - - ); - } - - return ( - - ); - }; - - return ( -
- {/* Show Start or Stop based on active run status */} - {hasActiveRun ? ( - <> - - - - ) : ( - - )} -
- ); -}; diff --git a/web/dashboard/src/components/runs/index.ts b/web/dashboard/src/components/runs/index.ts index b228732a..27f66c46 100644 --- a/web/dashboard/src/components/runs/index.ts +++ b/web/dashboard/src/components/runs/index.ts @@ -1,3 +1 @@ export { RunConfigDialog } from './RunConfigDialog'; -export { RunControlBar } from './RunControlBar'; -export type { RunControlBarProps } from './RunControlBar'; diff --git a/web/dashboard/src/hooks/useWebSocket.ts b/web/dashboard/src/hooks/useWebSocket.ts index e8fc785a..4134d639 100644 --- a/web/dashboard/src/hooks/useWebSocket.ts +++ b/web/dashboard/src/hooks/useWebSocket.ts @@ -1,11 +1,12 @@ import { useEffect, useRef, useCallback } from 'react'; import { useStateStore } from '../stores/stateStore'; // Event types that match the Go backend (wire_events.go) -// Backend sends: { type, timestamp, payload } +// Backend sends: { type, timestamp, payload, sequence } interface WebSocketEvent { type: string; timestamp: string; payload: unknown; + sequence?: number; // Monotonic sequence number for event ordering } interface AgentStartedEvent { type: 'agent:started'; @@ -17,6 +18,15 @@ interface AgentStartedEvent { task_title: string; task_description?: string; parent_agent_id?: string; + attempt?: number; // Current attempt number (1 = first try, 2 = first retry, etc.) + max_retries?: number; // Maximum retry attempts configured (0 = no retries, -1 = infinite) + }; +} +interface AgentRunningEvent { + type: 'agent:running'; + timestamp: string; + payload: { + agent_id: string; }; } interface AgentOutputEvent { @@ -68,8 +78,8 @@ interface AgentCompletedEvent { duration: number; input_tokens: number; output_tokens: number; - cache_creation_tokens?: number; - cache_read_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; cost_usd: number; files_changed: number; commits_created: number; @@ -91,7 +101,10 @@ interface TaskUpdatedEvent { // Merge status types matching Go backend (ipc/protocol.go) type MergeStatus = 'pending' | 'acquiring' | 'merging' | 'resolving' | 'merged' | 'failed' | 'skipped' | 'merged_needs_repair'; // Validation status types matching Go backend (validation/executor.go) -type ValidationStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped' | 'repairing'; +// Includes repair-related intermediate states: pending_repair (deciding to spawn), spawning_repair (creating agent), repairing (agent executing) +type ValidationStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped' | 'pending_repair' | 'spawning_repair' | 'repairing'; +// Lifecycle state types matching Go backend (lifecycle/state.go) +type LifecycleState = 'starting' | 'running' | 'queued_for_merge' | 'merging' | 'resolving' | 'validating' | 'repairing' | 'merge_failed' | 'completed' | 'failed' | 'needs_attention' | 'cancelled' | 'timed_out'; // Validation step result matching Go backend (ipc/protocol.go) interface ValidationStep { name: string; @@ -107,6 +120,7 @@ interface AgentMergeStatusEvent { merge_status: MergeStatus; queue_pos?: number; error?: string; + commits_applied?: number; // Validation results validation_status?: ValidationStatus; validation_steps?: ValidationStep[]; @@ -169,6 +183,26 @@ interface RulesChangedEvent { rule?: Rule; }; } +interface ConfigUpdatedEvent { + type: 'config:updated'; + timestamp: string; + payload: { + concurrency: number; + max_priority: number; + use_bwrap: boolean; + max_retries: number; + }; +} +interface LifecycleStateChangedEvent { + type: 'lifecycle:state_changed'; + timestamp: string; + payload: { + agent_id: string; + lifecycle_state: LifecycleState; + previous_state: LifecycleState; + event: string; + }; +} // Backend git commit format interface BackendGitCommit { hash: string; @@ -187,6 +221,7 @@ interface BackendAgentState { task_title: string; task_description?: string; status: string; + lifecycle_state?: string; // New unified lifecycle state start_time: string; end_time: string | null; duration: number; @@ -226,6 +261,9 @@ interface BackendAgentState { validation_error?: string; repair_attempts?: number; last_repair_output?: string; + // Retry information + attempt?: number; // Current attempt number (1 = first try, 2 = first retry, etc.) + max_retries?: number; // Maximum retry attempts configured (0 = no retries, -1 = infinite) } // Backend task state format interface BackendTaskState { @@ -251,6 +289,10 @@ interface BackendRuntimeState { completed_tasks: number; failed_tasks: number; running_tasks: number; + total_input_tokens: number; + total_output_tokens: number; + total_cache_creation_tokens: number; + total_cache_read_tokens: number; total_tokens: number; total_cost_usd: number; total_duration: number; @@ -265,9 +307,16 @@ interface BackendRuntimeState { pause_state: PauseState; start_time: string; current_run_id: string; + orchestrator_state?: OrchestratorState; + active_agent_count?: number; + // Event sequence number - events with sequence <= this are already reflected + // in the snapshot state and should be discarded by the client + event_sequence?: number; } // Pause state enum matching Go backend (ipc/protocol.go) type PauseState = 'running' | 'paused_user' | 'paused_agent' | 'paused_both'; +// Orchestrator state enum matching Go backend (daemon/orchestrator_manager.go) +type OrchestratorState = 'off' | 'idle' | 'active' | 'paused'; interface StateSyncEvent { type: 'state:sync'; timestamp: string; @@ -295,9 +344,19 @@ interface OrchResumedEvent { timestamp: string; payload: OrchPauseStatusPayload; } +interface OrchStateChangedPayload { + state: OrchestratorState; + active_agent_count: number; +} +interface OrchStateChangedEvent { + type: 'orch:state_changed'; + timestamp: string; + payload: OrchStateChangedPayload; +} type EventType = | StateSyncEvent | AgentStartedEvent + | AgentRunningEvent | AgentOutputEvent | AgentOutputClearEvent | AgentLiveFeedEvent @@ -308,9 +367,12 @@ type EventType = | StatsUpdatedEvent | OrchPausedEvent | OrchResumedEvent + | OrchStateChangedEvent | RunStartedEvent | RunCompletedEvent - | RulesChangedEvent; + | RulesChangedEvent + | LifecycleStateChangedEvent + | ConfigUpdatedEvent; const MAX_BACKOFF = 30000; // 30 seconds const INITIAL_BACKOFF = 1000; // 1 second function getWebSocketURL(): string { @@ -342,7 +404,28 @@ export function useWebSocket() { addRun, updateRun, setCurrentRunId, + setOrchestratorState, + // Optimistic update confirmations + confirmPause, + confirmResume, + // Config updates + updateRunConfigFromServer, } = useStateStore(); + + // Helper to check if an event is stale (occurred before the last state:sync snapshot) + // Events with sequence <= the snapshot's eventSequence are already reflected in the state + const isStaleEvent = useCallback((eventSequence: number | undefined): boolean => { + if (eventSequence === undefined || eventSequence === 0) { + // No sequence number - process the event (backwards compatibility) + return false; + } + const snapshotSequence = useStateStore.getState().eventSequence; + if (snapshotSequence === 0) { + // No snapshot sequence yet - process the event + return false; + } + return eventSequence <= snapshotSequence; + }, []); const connect = useCallback(() => { // Don't reconnect if manually closed if (isManuallyClosedRef.current) { @@ -365,8 +448,18 @@ export function useWebSocket() { }; ws.onmessage = (event) => { try { - const message: EventType = JSON.parse(event.data); - console.log('[WebSocket] Received:', message.type, message); + // Parse as WebSocketEvent first to get the sequence number + const rawMessage: WebSocketEvent = JSON.parse(event.data); + const message: EventType = rawMessage as unknown as EventType; + console.log('[WebSocket] Received:', message.type, 'seq:', rawMessage.sequence ?? 'none'); + + // For events other than state:sync, check if the event is stale + // (occurred before the last snapshot we received) + if (message.type !== 'state:sync' && isStaleEvent(rawMessage.sequence)) { + console.log('[WebSocket] Discarding stale event:', message.type, 'seq:', rawMessage.sequence, '<= snapshot seq:', useStateStore.getState().eventSequence); + return; + } + switch (message.type) { case 'state:sync': { // Check if this is a repo switch event (partial sync) @@ -426,6 +519,11 @@ export function useWebSocket() { ...(agent.validation_error && { validation_error: agent.validation_error }), ...(agent.repair_attempts !== undefined && { repair_attempts: agent.repair_attempts }), ...(agent.last_repair_output && { last_repair_output: agent.last_repair_output }), + // Lifecycle state (new unified state machine) + ...(agent.lifecycle_state && { lifecycle_state: agent.lifecycle_state as import('../stores/stateStore').LifecycleState }), + // Retry information + ...(agent.attempt !== undefined && agent.attempt > 0 && { attempt: agent.attempt }), + ...(agent.max_retries !== undefined && { max_retries: agent.max_retries }), }; } // Merge dual-source tasks: runtime overlays persistent for display @@ -484,6 +582,10 @@ export function useWebSocket() { completed_tasks: 0, failed_tasks: 0, running_tasks: 0, + total_input_tokens: 0, + total_output_tokens: 0, + total_cache_creation_tokens: 0, + total_cache_read_tokens: 0, total_tokens: 0, total_cost_usd: 0, total_duration: 0, @@ -498,18 +600,22 @@ export function useWebSocket() { pause_state: backendState.pause_state ?? 'running', start_time: backendState.start_time, current_run_id: backendState.current_run_id ?? '', + orchestrator_state: backendState.orchestrator_state ?? 'off', + active_agent_count: backendState.active_agent_count ?? 0, + // Event sequence for discarding stale events + event_sequence: backendState.event_sequence ?? 0, }); - console.log('[WebSocket] State synced with', Object.keys(transformedAgents).length, 'agents'); + console.log('[WebSocket] State synced with', Object.keys(transformedAgents).length, 'agents, event_sequence:', backendState.event_sequence ?? 0); break; } case 'agent:started': { - const { agent_id, run_id, task_id, task_title, task_description, parent_agent_id } = message.payload; - // Create new agent entry + const { agent_id, run_id, task_id, task_title, task_description, parent_agent_id, attempt, max_retries } = message.payload; + // Create new agent entry with starting status updateAgent(agent_id, { id: agent_id, task_id, task_title, - status: 'running', + status: 'starting', start_time: message.timestamp, end_time: null, duration: 0, @@ -530,6 +636,25 @@ export function useWebSocket() { ...(run_id && { run_id }), ...(task_description && { task_description }), ...(parent_agent_id && { parent_agent_id }), + // Retry information + ...(attempt !== undefined && { attempt }), + ...(max_retries !== undefined && { max_retries }), + }); + break; + } + case 'agent:running': { + const payload = message.payload as { + agent_id: string; + lifecycle_state?: string; + previous_state?: string; + event?: string; + }; + const { agent_id, lifecycle_state } = payload; + // Transition agent from starting to running, include lifecycle_state if present + // The lifecycle_state field comes from lifecycle transition callbacks + updateAgent(agent_id, { + status: 'running', + ...(lifecycle_state && { lifecycle_state: lifecycle_state as import('../stores/stateStore').LifecycleState }), }); break; } @@ -570,7 +695,7 @@ export function useWebSocket() { break; } case 'agent:completed': { - const { agent_id, error, exit_code, duration, input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens, cost_usd, files_changed, commits_created } = message.payload; + const { agent_id, error, exit_code, duration, input_tokens, output_tokens, cache_creation_input_tokens, cache_read_input_tokens, cost_usd, files_changed, commits_created } = message.payload; updateAgent(agent_id, { status: error ? 'failed' : 'completed', end_time: message.timestamp, @@ -585,8 +710,8 @@ export function useWebSocket() { total_tokens: input_tokens + output_tokens, cost_usd, // Optional cache token fields - only set if defined (exactOptionalPropertyTypes compliance) - ...(cache_creation_tokens !== undefined && { cache_creation_input_tokens: cache_creation_tokens }), - ...(cache_read_tokens !== undefined && { cache_read_input_tokens: cache_read_tokens }), + ...(cache_creation_input_tokens !== undefined && { cache_creation_input_tokens }), + ...(cache_read_input_tokens !== undefined && { cache_read_input_tokens }), }, }); break; @@ -597,6 +722,7 @@ export function useWebSocket() { merge_status, queue_pos, error, + commits_applied, validation_status, validation_steps, validation_duration_ms, @@ -604,12 +730,13 @@ export function useWebSocket() { repair_attempts, last_repair_output, } = message.payload; - console.log('[WebSocket] Agent merge status:', agent_id, merge_status, 'pos:', queue_pos, 'validation:', validation_status); + console.log('[WebSocket] Agent merge status:', agent_id, merge_status, 'pos:', queue_pos, 'commits:', commits_applied, 'validation:', validation_status); updateAgentMergeStatus( agent_id, merge_status, queue_pos, error, + commits_applied, validation_status as import('../stores/stateStore').ValidationStatus | undefined, validation_steps as import('../stores/stateStore').ValidationStep[] | undefined, validation_duration_ms, @@ -639,15 +766,27 @@ export function useWebSocket() { case 'orch:paused': { const { is_paused, is_paused_by_user, is_paused_by_agent, pause_state } = message.payload; console.log('[WebSocket] Orchestrator paused:', pause_state); + // Use confirmPause to reconcile optimistic update with actual state + confirmPause(pause_state); + // Also set the full pause state for cases where we didn't have an optimistic update setPauseState(is_paused, is_paused_by_user, is_paused_by_agent, pause_state); break; } case 'orch:resumed': { const { is_paused, is_paused_by_user, is_paused_by_agent, pause_state } = message.payload; console.log('[WebSocket] Orchestrator resumed:', pause_state); + // Use confirmResume to reconcile optimistic update with actual state + confirmResume(pause_state); + // Also set the full pause state for cases where we didn't have an optimistic update setPauseState(is_paused, is_paused_by_user, is_paused_by_agent, pause_state); break; } + case 'orch:state_changed': { + const { state, active_agent_count } = message.payload; + console.log('[WebSocket] Orchestrator state changed:', state, 'agents:', active_agent_count); + setOrchestratorState(state, active_agent_count); + break; + } case 'run:started': { const { run_id, task_count, repo_id, repo_path, repo_name } = message.payload; console.log('[WebSocket] Run started:', run_id, 'tasks:', task_count); @@ -700,6 +839,21 @@ export function useWebSocket() { // Rules changes are informational for now - UI can fetch updated rules if needed break; } + case 'config:updated': { + const { concurrency, max_priority, use_bwrap, max_retries } = message.payload; + console.log('[WebSocket] Config updated:', concurrency, max_priority, use_bwrap, max_retries); + updateRunConfigFromServer({ concurrency, max_priority, use_bwrap, max_retries }); + break; + } + case 'lifecycle:state_changed': { + const { agent_id, lifecycle_state, previous_state } = message.payload; + console.log('[WebSocket] Lifecycle state changed:', agent_id, previous_state, '->', lifecycle_state); + // Update the agent's lifecycle_state for real-time UI updates + updateAgent(agent_id, { + lifecycle_state: lifecycle_state as import('../stores/stateStore').LifecycleState, + }); + break; + } default: console.warn('[WebSocket] Unknown event type:', (message as WebSocketEvent).type); } @@ -737,7 +891,7 @@ export function useWebSocket() { }, backoffTime); } } - }, [setConnected, updateAgent, updateTask, appendOutput, appendLiveFeedEvent, appendGitCommit, syncState, setPauseState, setActiveRepo, updateAgentMergeStatus, addRun, updateRun, clearOutput, setCurrentRunId]); + }, [setConnected, updateAgent, updateTask, appendOutput, appendLiveFeedEvent, appendGitCommit, syncState, setPauseState, setActiveRepo, updateAgentMergeStatus, addRun, updateRun, clearOutput, setCurrentRunId, setOrchestratorState, confirmPause, confirmResume, isStaleEvent, updateRunConfigFromServer]); const disconnect = useCallback(() => { isManuallyClosedRef.current = true; if (reconnectTimeoutRef.current !== null) { diff --git a/web/dashboard/src/stores/stateStore.ts b/web/dashboard/src/stores/stateStore.ts index d905f6f8..09afad46 100644 --- a/web/dashboard/src/stores/stateStore.ts +++ b/web/dashboard/src/stores/stateStore.ts @@ -1,5 +1,6 @@ import { create } from 'zustand'; import type { Repository, MergeQueueState, Run, ActiveRunStatus, Rule } from '../api/client'; +import { getRunConfig } from '../api/client'; // Types based on Go backend structures @@ -15,7 +16,25 @@ export type AgentStatus = export type MergeStatus = 'pending' | 'acquiring' | 'merging' | 'resolving' | 'merged' | 'failed' | 'skipped' | 'merged_needs_repair'; // Validation status types matching Go backend (validation/executor.go) -export type ValidationStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped' | 'repairing'; +// Includes repair-related intermediate states: pending_repair (deciding to spawn), spawning_repair (creating agent), repairing (agent executing) +export type ValidationStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped' | 'pending_repair' | 'spawning_repair' | 'repairing'; + +// Lifecycle state types matching Go backend (lifecycle/state.go) +// This is the new unified state machine that replaces interpreting multiple fields +export type LifecycleState = + | 'starting' + | 'running' + | 'queued_for_merge' + | 'merging' + | 'resolving' + | 'validating' + | 'repairing' + | 'merge_failed' + | 'completed' + | 'failed' + | 'needs_attention' + | 'cancelled' + | 'timed_out'; // ValidationStep represents a single validation step result export interface ValidationStep { @@ -74,6 +93,7 @@ export interface AgentState { parent_agent_id?: string; // ID of parent agent if spawned by another agent child_agent_ids?: string[]; // IDs of child agents spawned by this agent status: AgentStatus; + lifecycle_state?: LifecycleState; // New unified lifecycle state (optional for backwards compat) start_time: string; end_time: string | null; duration: number; @@ -87,6 +107,9 @@ export interface AgentState { git_commits: GitCommit[]; result_message?: string; // Final result message from agent archived: boolean; + // Retry information + attempt?: number; // Current attempt number (1 = first try, 2 = first retry, etc.) + max_retries?: number; // Maximum retry attempts configured (0 = no retries, -1 = infinite) // Merge status fields (snake_case per API conventions) merge_status?: MergeStatus; merge_queue_pos?: number; @@ -121,6 +144,10 @@ export interface Stats { completed_tasks: number; failed_tasks: number; running_tasks: number; + total_input_tokens: number; + total_output_tokens: number; + total_cache_creation_tokens: number; + total_cache_read_tokens: number; total_tokens: number; total_cost_usd: number; total_duration: number; @@ -133,12 +160,25 @@ export interface Stats { // Pause state enum matching Go backend (ipc/protocol.go) export type PauseState = 'running' | 'paused_user' | 'paused_agent' | 'paused_both'; +// Orchestrator state enum matching Go backend (daemon/orchestrator_manager.go) +export type OrchestratorState = 'off' | 'idle' | 'active' | 'paused'; + // Run configuration for starting new runs +// RuleOverride for run-configured rules (matches Go config.CustomRule) +export interface RuleOverride { + name: string; + condition: string; + action: 'deny' | 'allow'; + enabled?: boolean; + reason?: string; +} + export interface RunConfig { concurrency: number; max_priority: number; use_bwrap: boolean; max_retries: number; + rule_overrides?: RuleOverride[]; } // Default run configuration @@ -147,8 +187,35 @@ export const DEFAULT_RUN_CONFIG: RunConfig = { max_priority: 4, use_bwrap: true, max_retries: 3, + rule_overrides: [], }; +// Optimistic update types +export type OptimisticOperationType = + | 'start_run' + | 'stop_run' + | 'pause_orch' + | 'resume_orch'; + +export interface OptimisticOperation { + id: string; + type: OptimisticOperationType; + previousState: Partial; + timestamp: number; +} + +// Snapshot of state fields that can be optimistically updated +export interface OptimisticStateSnapshot { + orchestratorState: OrchestratorState; + pauseState: PauseState; + isPaused: boolean; + isPausedByUser: boolean; + isPausedByAgent: boolean; + currentRunId: string; + isStartingRun: boolean; + isStoppingRun: boolean; +} + export interface RuntimeState { agents: Record; tasks: Record; @@ -159,6 +226,11 @@ export interface RuntimeState { pause_state: PauseState; start_time: string; current_run_id: string; + orchestrator_state: OrchestratorState; + active_agent_count: number; + // Event sequence number for ordering - events with sequence <= this are already + // reflected in the snapshot and should be discarded by the client + event_sequence?: number; } // Store interface @@ -173,6 +245,11 @@ interface StateStore { isPausedByAgent: boolean; pauseState: PauseState; currentRunId: string; // Currently active orchestrator run ID (empty if no active run) + orchestratorState: OrchestratorState; // Current orchestrator state (off/idle/active/paused) + activeAgentCount: number; // Number of currently running agents + // Event sequence number from last state:sync - used to discard stale events + // that occurred before the snapshot was taken + eventSequence: number; selectedAgentId: string | null; highlightedTaskId: string | null; selectedBeadId: string | null; // Selected bead for filtering agents @@ -196,6 +273,10 @@ interface StateStore { isRulesLoading: boolean; showAddRuleDialog: boolean; + // Optimistic update state + pendingOperations: OptimisticOperation[]; + lastError: { operation: OptimisticOperationType; message: string } | null; + // Actions setConnected: (connected: boolean) => void; updateAgent: (id: string, update: Partial) => void; @@ -223,6 +304,7 @@ interface StateStore { mergeStatus: MergeStatus, queuePos?: number, error?: string, + commitsApplied?: number, validationStatus?: ValidationStatus, validationSteps?: ValidationStep[], validationDurationMs?: number, @@ -232,8 +314,11 @@ interface StateStore { ) => void; // Current run tracking setCurrentRunId: (runId: string) => void; + // Orchestrator state + setOrchestratorState: (state: OrchestratorState, activeAgentCount: number) => void; // Run filtering actions setRuns: (runs: Run[]) => void; + mergeRuns: (runs: Run[]) => void; setActiveRunId: (runId: string) => void; setRunsLoading: (loading: boolean) => void; addRun: (run: Run) => void; @@ -253,6 +338,29 @@ interface StateStore { updateRule: (name: string, update: Partial) => void; removeRule: (name: string) => void; reorderRules: (fromIndex: number, toIndex: number) => void; + + // Optimistic update actions + startOptimisticRun: (tempRunId: string) => void; + confirmRunStarted: (runId: string) => void; + rollbackRunStart: (error: string) => void; + + startOptimisticStop: () => void; + confirmRunStopped: () => void; + rollbackRunStop: (error: string) => void; + + startOptimisticPause: () => void; + confirmPause: (pauseState: PauseState) => void; + rollbackPause: (error: string) => void; + + startOptimisticResume: () => void; + confirmResume: (pauseState: PauseState) => void; + rollbackResume: (error: string) => void; + + clearLastError: () => void; + + // Config persistence actions + loadSavedConfig: () => Promise; + updateRunConfigFromServer: (config: RunConfig) => void; } // Initial stats @@ -261,6 +369,10 @@ const initialStats: Stats = { completed_tasks: 0, failed_tasks: 0, running_tasks: 0, + total_input_tokens: 0, + total_output_tokens: 0, + total_cache_creation_tokens: 0, + total_cache_read_tokens: 0, total_tokens: 0, total_cost_usd: 0, total_duration: 0, @@ -293,6 +405,11 @@ function recalculateStats(agents: Record): Stats { break; } + // Track all token types separately + stats.total_input_tokens += agent.token_usage.input_tokens; + stats.total_output_tokens += agent.token_usage.output_tokens; + stats.total_cache_creation_tokens += agent.token_usage.cache_creation_input_tokens ?? 0; + stats.total_cache_read_tokens += agent.token_usage.cache_read_input_tokens ?? 0; stats.total_tokens += agent.token_usage.total_tokens; stats.total_cost_usd += agent.token_usage.cost_usd; stats.file_changes += agent.changes; @@ -324,6 +441,9 @@ export const useStateStore = create((set) => ({ isPausedByAgent: false, pauseState: 'running', currentRunId: '', // empty means no active orchestrator run + orchestratorState: 'off' as OrchestratorState, + activeAgentCount: 0, + eventSequence: 0, // Sequence number from last state:sync selectedAgentId: null, highlightedTaskId: null, selectedBeadId: null, @@ -347,6 +467,10 @@ export const useStateStore = create((set) => ({ isRulesLoading: false, showAddRuleDialog: false, + // Optimistic update state + pendingOperations: [], + lastError: null, + // Actions setConnected: (connected) => set({ connected }), @@ -420,6 +544,10 @@ export const useStateStore = create((set) => ({ isPausedByAgent: runtimeState.is_paused_by_agent, pauseState: runtimeState.pause_state, currentRunId: runtimeState.current_run_id ?? '', + orchestratorState: runtimeState.orchestrator_state ?? 'off', + activeAgentCount: runtimeState.active_agent_count ?? 0, + // Update event sequence from snapshot - used to discard stale events + eventSequence: runtimeState.event_sequence ?? 0, }), appendOutput: (agentId, output, isError = false) => @@ -532,7 +660,7 @@ export const useStateStore = create((set) => ({ setMergeQueue: (mergeQueue) => set({ mergeQueue }), - updateAgentMergeStatus: (agentId, mergeStatus, queuePos, error, validationStatus, validationSteps, validationDurationMs, validationError, repairAttempts, lastRepairOutput) => + updateAgentMergeStatus: (agentId, mergeStatus, queuePos, error, commitsApplied, validationStatus, validationSteps, validationDurationMs, validationError, repairAttempts, lastRepairOutput) => set((state) => { const agent = state.agents[agentId]; if (!agent) { @@ -555,6 +683,9 @@ export const useStateStore = create((set) => ({ if (error !== undefined) { updatedAgent.merge_error = error; } + if (commitsApplied !== undefined) { + updatedAgent.merge_commits_applied = commitsApplied; + } if (validationStatus !== undefined) { updatedAgent.validation_status = validationStatus; } @@ -585,9 +716,35 @@ export const useStateStore = create((set) => ({ // Current run tracking setCurrentRunId: (currentRunId) => set({ currentRunId }), + // Orchestrator state + setOrchestratorState: (orchestratorState, activeAgentCount) => + set({ orchestratorState, activeAgentCount }), + // Run filtering actions setRuns: (runs) => set({ runs }), + // Merge runs from API with existing state to avoid race conditions where + // WebSocket events add runs before the initial API fetch completes + mergeRuns: (apiRuns) => + set((state) => { + // Build a map of API runs for efficient lookup + const apiRunMap = new Map(apiRuns.map((r) => [r.id, r])); + + // Keep any existing runs that aren't in the API response (recently added via WebSocket) + // These are typically runs that were just created and haven't been persisted yet + const runsNotInApi = state.runs.filter((r) => !apiRunMap.has(r.id)); + + // Merge: API runs (fresher data) + runs not in API (WebSocket-added) + // Sort by started_at descending (most recent first) + const merged = [...apiRuns, ...runsNotInApi].sort((a, b) => { + const aTime = a.started_at ? new Date(a.started_at).getTime() : 0; + const bTime = b.started_at ? new Date(b.started_at).getTime() : 0; + return bTime - aTime; + }); + + return { runs: merged }; + }), + setActiveRunId: (activeRunId) => set({ activeRunId }), setRunsLoading: (isRunsLoading) => set({ isRunsLoading }), @@ -672,4 +829,223 @@ export const useStateStore = create((set) => ({ rulesPersistedState: false, // Reordering changes persisted state }; }), + + // Optimistic update actions - Start Run + startOptimisticRun: (tempRunId) => + set((state) => { + const operation: OptimisticOperation = { + id: `start_run_${Date.now()}`, + type: 'start_run', + previousState: { + orchestratorState: state.orchestratorState, + currentRunId: state.currentRunId, + isStartingRun: state.isStartingRun, + }, + timestamp: Date.now(), + }; + return { + pendingOperations: [...state.pendingOperations, operation], + orchestratorState: 'active' as OrchestratorState, + currentRunId: tempRunId, + isStartingRun: true, + lastError: null, + }; + }), + + confirmRunStarted: (runId) => + set((state) => ({ + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'start_run'), + currentRunId: runId, + isStartingRun: false, + })), + + rollbackRunStart: (error) => + set((state) => { + const operation = state.pendingOperations.find((op) => op.type === 'start_run'); + if (!operation) { + return { isStartingRun: false, lastError: { operation: 'start_run', message: error } }; + } + return { + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'start_run'), + orchestratorState: operation.previousState.orchestratorState ?? state.orchestratorState, + currentRunId: operation.previousState.currentRunId ?? '', + isStartingRun: false, + lastError: { operation: 'start_run', message: error }, + }; + }), + + // Optimistic update actions - Stop Run + startOptimisticStop: () => + set((state) => { + const operation: OptimisticOperation = { + id: `stop_run_${Date.now()}`, + type: 'stop_run', + previousState: { + orchestratorState: state.orchestratorState, + currentRunId: state.currentRunId, + isStoppingRun: state.isStoppingRun, + }, + timestamp: Date.now(), + }; + return { + pendingOperations: [...state.pendingOperations, operation], + orchestratorState: 'idle' as OrchestratorState, + isStoppingRun: true, + lastError: null, + }; + }), + + confirmRunStopped: () => + set((state) => ({ + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'stop_run'), + currentRunId: '', + isStoppingRun: false, + })), + + rollbackRunStop: (error) => + set((state) => { + const operation = state.pendingOperations.find((op) => op.type === 'stop_run'); + if (!operation) { + return { isStoppingRun: false, lastError: { operation: 'stop_run', message: error } }; + } + return { + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'stop_run'), + orchestratorState: operation.previousState.orchestratorState ?? state.orchestratorState, + currentRunId: operation.previousState.currentRunId ?? state.currentRunId, + isStoppingRun: false, + lastError: { operation: 'stop_run', message: error }, + }; + }), + + // Optimistic update actions - Pause + startOptimisticPause: () => + set((state) => { + const operation: OptimisticOperation = { + id: `pause_orch_${Date.now()}`, + type: 'pause_orch', + previousState: { + pauseState: state.pauseState, + isPaused: state.isPaused, + isPausedByUser: state.isPausedByUser, + isPausedByAgent: state.isPausedByAgent, + }, + timestamp: Date.now(), + }; + // Determine new pause state based on current state + const newPauseState: PauseState = state.isPausedByAgent ? 'paused_both' : 'paused_user'; + return { + pendingOperations: [...state.pendingOperations, operation], + pauseState: newPauseState, + isPaused: true, + isPausedByUser: true, + lastError: null, + }; + }), + + confirmPause: (pauseState) => + set((state) => ({ + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'pause_orch'), + pauseState, + isPaused: pauseState !== 'running', + isPausedByUser: pauseState === 'paused_user' || pauseState === 'paused_both', + isPausedByAgent: pauseState === 'paused_agent' || pauseState === 'paused_both', + })), + + rollbackPause: (error) => + set((state) => { + const operation = state.pendingOperations.find((op) => op.type === 'pause_orch'); + if (!operation) { + return { lastError: { operation: 'pause_orch', message: error } }; + } + return { + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'pause_orch'), + pauseState: operation.previousState.pauseState ?? state.pauseState, + isPaused: operation.previousState.isPaused ?? state.isPaused, + isPausedByUser: operation.previousState.isPausedByUser ?? state.isPausedByUser, + isPausedByAgent: operation.previousState.isPausedByAgent ?? state.isPausedByAgent, + lastError: { operation: 'pause_orch', message: error }, + }; + }), + + // Optimistic update actions - Resume + startOptimisticResume: () => + set((state) => { + const operation: OptimisticOperation = { + id: `resume_orch_${Date.now()}`, + type: 'resume_orch', + previousState: { + pauseState: state.pauseState, + isPaused: state.isPaused, + isPausedByUser: state.isPausedByUser, + isPausedByAgent: state.isPausedByAgent, + }, + timestamp: Date.now(), + }; + // When user resumes, only the user pause is cleared + const newPauseState: PauseState = state.isPausedByAgent ? 'paused_agent' : 'running'; + return { + pendingOperations: [...state.pendingOperations, operation], + pauseState: newPauseState, + isPaused: newPauseState !== 'running', + isPausedByUser: false, + lastError: null, + }; + }), + + confirmResume: (pauseState) => + set((state) => ({ + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'resume_orch'), + pauseState, + isPaused: pauseState !== 'running', + isPausedByUser: pauseState === 'paused_user' || pauseState === 'paused_both', + isPausedByAgent: pauseState === 'paused_agent' || pauseState === 'paused_both', + })), + + rollbackResume: (error) => + set((state) => { + const operation = state.pendingOperations.find((op) => op.type === 'resume_orch'); + if (!operation) { + return { lastError: { operation: 'resume_orch', message: error } }; + } + return { + pendingOperations: state.pendingOperations.filter((op) => op.type !== 'resume_orch'), + pauseState: operation.previousState.pauseState ?? state.pauseState, + isPaused: operation.previousState.isPaused ?? state.isPaused, + isPausedByUser: operation.previousState.isPausedByUser ?? state.isPausedByUser, + isPausedByAgent: operation.previousState.isPausedByAgent ?? state.isPausedByAgent, + lastError: { operation: 'resume_orch', message: error }, + }; + }), + + clearLastError: () => set({ lastError: null }), + + // Config persistence actions + loadSavedConfig: async () => { + try { + const response = await getRunConfig(); + if (!response.error) { + set({ + runConfig: { + concurrency: response.concurrency, + max_priority: response.max_priority, + use_bwrap: response.use_bwrap, + max_retries: response.max_retries, + }, + }); + } + } catch (error) { + console.error('Failed to load saved run config:', error); + // Keep default config on error + } + }, + + updateRunConfigFromServer: (config) => + set({ + runConfig: { + concurrency: config.concurrency, + max_priority: config.max_priority, + use_bwrap: config.use_bwrap, + max_retries: config.max_retries, + }, + }), }));