From 10db5f4b549953b6201243f383994140cba81280 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 28 Jul 2026 15:53:56 -0300 Subject: [PATCH 1/3] research: validate reproducible evidence from agent history Add a content-free Codex evidence scanner and a TDD-built, path-safe ordered patch replayer. Validate repository and turn-level evidence across the local corpus, exercise verification inside a locked-down Docker container, and document the critical finding that legacy sessions omit the initial dirty-worktree baseline required for deterministic replay. --- _roadmap.md | 3 + internal/evidence/codex.go | 216 ++++++++++++++++++ internal/evidence/codex_test.go | 158 +++++++++++++ internal/evidence/replay.go | 371 +++++++++++++++++++++++++++++++ internal/evidence/replay_test.go | 210 +++++++++++++++++ scripts/evidence-audit/main.go | 73 ++++++ 6 files changed, 1031 insertions(+) create mode 100644 internal/evidence/codex.go create mode 100644 internal/evidence/codex_test.go create mode 100644 internal/evidence/replay.go create mode 100644 internal/evidence/replay_test.go create mode 100644 scripts/evidence-audit/main.go diff --git a/_roadmap.md b/_roadmap.md index 47c9edc..ae183d9 100644 --- a/_roadmap.md +++ b/_roadmap.md @@ -38,4 +38,7 @@ engineering portfolio. - [x] Propagate cancellation through agent-run command process trees and report local-model telemetry without false precision. - [x] Remove the disconnected legacy `monitor` and imperative `release` command implementations while preserving public-surface compatibility tests. - [ ] Isolate Live as an optional local observability protocol and retire unused training and experimental command surfaces. +- [x] Build a content-free Codex history scanner and validate turn-level Git, patch, verification, and completion evidence against the local corpus. +- [x] Build and container-validate a path-safe historical patch replayer, and establish that legacy Codex sessions lack the dirty-worktree baseline needed for deterministic replay. +- [ ] Capture an initial base diff or snapshot in new sessions, prove deterministic replay end to end, then add an OpenCode importer to challenge the vendor-neutral evidence model. - [ ] Raise coverage in legacy workflow packages without presenting the public fixture as repository-wide coverage. diff --git a/internal/evidence/codex.go b/internal/evidence/codex.go new file mode 100644 index 0000000..eae4034 --- /dev/null +++ b/internal/evidence/codex.go @@ -0,0 +1,216 @@ +package evidence + +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" +) + +const maxEventBytes = 16 << 20 + +var verificationCommand = regexp.MustCompile( + `(?i)(^|[[:space:];|&])(go test|go vet|make verify|mix test|mix credo|mix format|` + + `npm test|npm run lint|pytest|ruff check|cargo test|bundle exec|golangci-lint|` + + `staticcheck|eslint|tsc)([[:space:];|&]|$)`, +) + +// CodexSummary contains aggregate evidence counts only. It intentionally +// excludes prompts, outputs, paths, repository URLs, commit hashes, commands, +// session identifiers, and other potentially sensitive values. +type CodexSummary struct { + Sessions int `json:"sessions"` + SessionsWithGit int `json:"sessions_with_git"` + Commands int `json:"commands"` + SuccessfulCommands int `json:"successful_commands"` + FailedCommands int `json:"failed_commands"` + VerificationCommands int `json:"verification_commands"` + SuccessfulVerifications int `json:"successful_verifications"` + Patches int `json:"patches"` + CompletedTasks int `json:"completed_tasks"` + Level3CandidateTurns int `json:"level3_candidate_turns"` +} + +type codexEvent struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` +} + +type sessionEvidence struct { + hasMetadata bool + hasGit bool + turns map[string]*turnEvidence +} + +type turnEvidence struct { + hasSuccessfulPatch bool + hasSuccessfulVerify bool + hasCompletion bool +} + +// ScanCodex reads Codex JSONL session files and returns content-free aggregate +// evidence metrics. It never returns or persists raw event values. +func ScanCodex(root string) (CodexSummary, error) { + var summary CodexSummary + + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".jsonl" { + return nil + } + + session, err := scanCodexSession(path, &summary) + if err != nil { + return err + } + if session.hasMetadata { + summary.Sessions++ + } + if session.hasGit { + summary.SessionsWithGit++ + } + if session.hasGit { + for _, turn := range session.turns { + if turn.hasSuccessfulPatch && + turn.hasSuccessfulVerify && + turn.hasCompletion { + summary.Level3CandidateTurns++ + } + } + } + return nil + }) + if err != nil { + return CodexSummary{}, fmt.Errorf("scanning Codex sessions: %w", err) + } + + return summary, nil +} + +func scanCodexSession(path string, summary *CodexSummary) (sessionEvidence, error) { + file, err := os.Open(path) + if err != nil { + return sessionEvidence{}, fmt.Errorf("opening session file: %w", err) + } + defer file.Close() + + session := sessionEvidence{turns: make(map[string]*turnEvidence)} + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), maxEventBytes) + + for line := 1; scanner.Scan(); line++ { + var event codexEvent + if err := json.Unmarshal(scanner.Bytes(), &event); err != nil { + return sessionEvidence{}, fmt.Errorf("decoding session event at line %d: %w", line, err) + } + if err := countCodexEvent(event, summary, &session); err != nil { + return sessionEvidence{}, fmt.Errorf("reading session event at line %d: %w", line, err) + } + } + if err := scanner.Err(); err != nil { + return sessionEvidence{}, fmt.Errorf("reading session file: %w", err) + } + + return session, nil +} + +func countCodexEvent(event codexEvent, summary *CodexSummary, session *sessionEvidence) error { + var payload map[string]any + if len(event.Payload) > 0 { + if err := json.Unmarshal(event.Payload, &payload); err != nil { + return err + } + } + + if event.Type == "session_meta" { + session.hasMetadata = true + _, session.hasGit = payload["git"].(map[string]any) + return nil + } + + eventType, _ := payload["type"].(string) + turn := session.turn(payload) + switch eventType { + case "exec_command_end": + countCommand(payload, summary, turn) + case "patch_apply_end": + if success, _ := payload["success"].(bool); success { + summary.Patches++ + if turn != nil { + turn.hasSuccessfulPatch = true + } + } + case "task_complete": + summary.CompletedTasks++ + if turn != nil { + turn.hasCompletion = true + } + } + return nil +} + +func countCommand(payload map[string]any, summary *CodexSummary, turn *turnEvidence) { + summary.Commands++ + + exitCode, hasExitCode := numberAsInt(payload["exit_code"]) + if hasExitCode && exitCode == 0 { + summary.SuccessfulCommands++ + } else { + summary.FailedCommands++ + } + + command := commandText(payload["command"]) + if !verificationCommand.MatchString(command) { + return + } + + summary.VerificationCommands++ + if hasExitCode && exitCode == 0 { + summary.SuccessfulVerifications++ + if turn != nil { + turn.hasSuccessfulVerify = true + } + } +} + +func (session *sessionEvidence) turn(payload map[string]any) *turnEvidence { + turnID, _ := payload["turn_id"].(string) + if turnID == "" { + return nil + } + if session.turns[turnID] == nil { + session.turns[turnID] = &turnEvidence{} + } + return session.turns[turnID] +} + +func commandText(value any) string { + switch command := value.(type) { + case string: + return command + case []any: + parts := make([]string, 0, len(command)) + for _, part := range command { + if text, ok := part.(string); ok { + parts = append(parts, text) + } + } + return strings.Join(parts, " ") + default: + return "" + } +} + +func numberAsInt(value any) (int, bool) { + number, ok := value.(float64) + if !ok { + return 0, false + } + return int(number), true +} diff --git a/internal/evidence/codex_test.go b/internal/evidence/codex_test.go new file mode 100644 index 0000000..b76cc58 --- /dev/null +++ b/internal/evidence/codex_test.go @@ -0,0 +1,158 @@ +package evidence + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestScanCodexSummarizesEvidenceWithoutLeakingContent(t *testing.T) { + t.Parallel() + + root := t.TempDir() + writeSession(t, root, "candidate.jsonl", []map[string]any{ + event("session_meta", map[string]any{ + "id": "session-sensitive-id", + "cwd": "/private/customer/repository", + "git": map[string]any{ + "branch": "private-branch", + "commit_hash": "abc123", + "repository_url": "git@example.com:customer/private.git", + }, + }), + event("event_msg", map[string]any{ + "type": "exec_command_end", + "turn_id": "turn-1", + "command": []string{"/bin/zsh", "-lc", "go test ./..."}, + "exit_code": 0, + "stdout": "SECRET_OUTPUT", + }), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "turn-1", + "success": true, + "changes": []any{map[string]any{"path": "secret.go"}}, + }), + event("event_msg", map[string]any{"type": "task_complete", "turn_id": "turn-1"}), + }) + writeSession(t, root, "incomplete.jsonl", []map[string]any{ + event("session_meta", map[string]any{ + "id": "another-sensitive-id", + "cwd": "/private/other", + }), + event("event_msg", map[string]any{ + "type": "exec_command_end", + "command": "rm -rf build", + "exit_code": 1, + "stderr": "SECRET_ERROR", + }), + }) + writeSession(t, root, "uncorrelated.jsonl", []map[string]any{ + event("session_meta", map[string]any{ + "id": "uncorrelated-sensitive-id", + "cwd": "/private/uncorrelated", + "git": map[string]any{"commit_hash": "def456"}, + }), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "turn-a", + "success": true, + }), + event("event_msg", map[string]any{ + "type": "exec_command_end", + "turn_id": "turn-b", + "command": "go test ./...", + "exit_code": 0, + }), + event("event_msg", map[string]any{"type": "task_complete", "turn_id": "turn-c"}), + }) + + summary, err := ScanCodex(root) + if err != nil { + t.Fatalf("ScanCodex() error = %v", err) + } + + if summary.Sessions != 3 { + t.Errorf("Sessions = %d, want 3", summary.Sessions) + } + if summary.SessionsWithGit != 2 { + t.Errorf("SessionsWithGit = %d, want 2", summary.SessionsWithGit) + } + if summary.Commands != 3 || summary.SuccessfulCommands != 2 || summary.FailedCommands != 1 { + t.Errorf( + "command counts = (%d, %d, %d), want (3, 2, 1)", + summary.Commands, + summary.SuccessfulCommands, + summary.FailedCommands, + ) + } + if summary.VerificationCommands != 2 || summary.SuccessfulVerifications != 2 { + t.Errorf( + "verification counts = (%d, %d), want (2, 2)", + summary.VerificationCommands, + summary.SuccessfulVerifications, + ) + } + if summary.Patches != 2 || summary.CompletedTasks != 2 { + t.Errorf("result counts = (%d, %d), want (2, 2)", summary.Patches, summary.CompletedTasks) + } + if summary.Level3CandidateTurns != 1 { + t.Errorf("Level3CandidateTurns = %d, want 1", summary.Level3CandidateTurns) + } + + encoded, err := json.Marshal(summary) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + for _, secret := range []string{ + "customer", + "private-branch", + "abc123", + "SECRET_OUTPUT", + "SECRET_ERROR", + "session-sensitive-id", + } { + if strings.Contains(string(encoded), secret) { + t.Errorf("summary leaked sensitive value %q", secret) + } + } +} + +func TestScanCodexRejectsMalformedJSONL(t *testing.T) { + t.Parallel() + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "broken.jsonl"), []byte("{broken\n"), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := ScanCodex(root); err == nil { + t.Fatal("ScanCodex() error = nil, want malformed JSON error") + } +} + +func event(kind string, payload map[string]any) map[string]any { + return map[string]any{ + "type": kind, + "payload": payload, + } +} + +func writeSession(t *testing.T, root, name string, events []map[string]any) { + t.Helper() + + var lines []byte + for _, item := range events { + encoded, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + lines = append(lines, encoded...) + lines = append(lines, '\n') + } + if err := os.WriteFile(filepath.Join(root, name), lines, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/evidence/replay.go b/internal/evidence/replay.go new file mode 100644 index 0000000..224faaa --- /dev/null +++ b/internal/evidence/replay.go @@ -0,0 +1,371 @@ +package evidence + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// FileChange is one ordered filesystem mutation reconstructed from a recorded +// patch event. +type FileChange struct { + Path string `json:"-"` + Type string `json:"-"` + Content string `json:"-"` + UnifiedDiff string `json:"-"` + MovePath string `json:"-"` +} + +// ReplayPlan contains sensitive local reconstruction inputs and is explicitly +// excluded from JSON serialization. +type ReplayPlan struct { + RepositoryRoot string `json:"-"` + BaseCommit string `json:"-"` + Changes []FileChange `json:"-"` +} + +type replayPayload struct { + Type string `json:"type"` + TurnID string `json:"turn_id"` + Success bool `json:"success"` + CWD string `json:"cwd"` + Git replayGit `json:"git"` + Changes json.RawMessage `json:"changes"` +} + +type replayGit struct { + CommitHash string `json:"commit_hash"` +} + +type recordedChange struct { + Type string `json:"type"` + Content string `json:"content"` + UnifiedDiff string `json:"unified_diff"` + MovePath string `json:"move_path"` +} + +// ExtractReplayPlan reconstructs the ordered filesystem mutations from the +// session start through completion of targetTurn. +func ExtractReplayPlan(sessionPath, targetTurn string) (ReplayPlan, error) { + if targetTurn == "" { + return ReplayPlan{}, errors.New("target turn is required") + } + + file, err := os.Open(sessionPath) + if err != nil { + return ReplayPlan{}, fmt.Errorf("opening session file: %w", err) + } + defer file.Close() + + var plan ReplayPlan + foundTarget := false + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64<<10), maxEventBytes) + for line := 1; scanner.Scan(); line++ { + var event codexEvent + if err := json.Unmarshal(scanner.Bytes(), &event); err != nil { + return ReplayPlan{}, fmt.Errorf("decoding session event at line %d: %w", line, err) + } + + var payload replayPayload + if err := json.Unmarshal(event.Payload, &payload); err != nil { + return ReplayPlan{}, fmt.Errorf("decoding replay event at line %d: %w", line, err) + } + if event.Type == "session_meta" { + plan.RepositoryRoot = payload.CWD + plan.BaseCommit = payload.Git.CommitHash + continue + } + if payload.Type == "patch_apply_end" && payload.Success { + changes, err := decodeRecordedChanges(payload.Changes) + if err != nil { + return ReplayPlan{}, fmt.Errorf("decoding changes at line %d: %w", line, err) + } + changes, err = normalizeRecordedChanges(plan.RepositoryRoot, changes) + if err != nil { + return ReplayPlan{}, fmt.Errorf("normalizing changes at line %d: %w", line, err) + } + plan.Changes = append(plan.Changes, changes...) + } + if payload.Type == "task_complete" && payload.TurnID == targetTurn { + foundTarget = true + break + } + } + if err := scanner.Err(); err != nil { + return ReplayPlan{}, fmt.Errorf("reading session file: %w", err) + } + if !foundTarget { + return ReplayPlan{}, errors.New("target turn completion was not found") + } + if plan.RepositoryRoot == "" || plan.BaseCommit == "" { + return ReplayPlan{}, errors.New("session is missing repository root or base commit") + } + + return plan, nil +} + +func decodeRecordedChanges(raw json.RawMessage) ([]FileChange, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil { + return nil, err + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' { + return nil, errors.New("changes must be a JSON object") + } + + var changes []FileChange + for decoder.More() { + pathToken, err := decoder.Token() + if err != nil { + return nil, err + } + path, ok := pathToken.(string) + if !ok { + return nil, errors.New("change path must be a string") + } + var recorded recordedChange + if err := decoder.Decode(&recorded); err != nil { + return nil, err + } + changes = append(changes, FileChange{ + Path: path, + Type: recorded.Type, + Content: recorded.Content, + UnifiedDiff: recorded.UnifiedDiff, + MovePath: recorded.MovePath, + }) + } + if _, err := decoder.Token(); err != nil { + return nil, err + } + return changes, nil +} + +func normalizeRecordedChanges(root string, changes []FileChange) ([]FileChange, error) { + if root == "" { + return nil, errors.New("repository root must precede patch events") + } + absoluteRoot, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("resolving repository root: %w", err) + } + + normalized := make([]FileChange, 0, len(changes)) + for _, change := range changes { + change.Path, err = recordedRelativePath(absoluteRoot, change.Path) + if err != nil { + return nil, err + } + if change.MovePath != "" { + change.MovePath, err = recordedRelativePath(absoluteRoot, change.MovePath) + if err != nil { + return nil, fmt.Errorf("normalizing move destination: %w", err) + } + } + normalized = append(normalized, change) + } + return normalized, nil +} + +func recordedRelativePath(root, recorded string) (string, error) { + if recorded == "" { + return "", errors.New("recorded path is empty") + } + if !filepath.IsAbs(recorded) { + return filepath.Clean(recorded), nil + } + + relative, err := filepath.Rel(root, filepath.Clean(recorded)) + if err != nil { + return "", fmt.Errorf("relativizing recorded path: %w", err) + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("recorded path is outside repository root") + } + return relative, nil +} + +// ApplyChanges applies a previously validated, ordered patch sequence inside +// root. The operation rejects paths that can escape root or alter Git metadata. +func ApplyChanges(ctx context.Context, root string, changes []FileChange) error { + absoluteRoot, err := filepath.Abs(root) + if err != nil { + return fmt.Errorf("resolving replay root: %w", err) + } + + for index, change := range changes { + if err := validateChange(absoluteRoot, change); err != nil { + return fmt.Errorf("validating change %d: %w", index+1, err) + } + } + + for index, change := range changes { + if err := applyChange(ctx, absoluteRoot, change); err != nil { + return fmt.Errorf("applying change %d: %w", index+1, err) + } + } + return nil +} + +func validateChange(root string, change FileChange) error { + if _, err := safePath(root, change.Path); err != nil { + return err + } + if change.MovePath != "" { + if _, err := safePath(root, change.MovePath); err != nil { + return fmt.Errorf("invalid move destination: %w", err) + } + } + + switch change.Type { + case "add": + if change.UnifiedDiff != "" { + return errors.New("add change must not include a unified diff") + } + case "delete": + case "update": + if change.UnifiedDiff == "" { + return errors.New("update change is missing a unified diff") + } + if err := validateDiffPaths(root, change.UnifiedDiff); err != nil { + return err + } + default: + return fmt.Errorf("unsupported change type %q", change.Type) + } + return nil +} + +func applyChange(ctx context.Context, root string, change FileChange) error { + path, err := safePath(root, change.Path) + if err != nil { + return err + } + + switch change.Type { + case "add": + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("creating parent directory: %w", err) + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("creating added file: %w", err) + } + if _, err := file.WriteString(change.Content); err != nil { + file.Close() + return fmt.Errorf("writing added file: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("closing added file: %w", err) + } + case "delete": + if err := os.Remove(path); err != nil { + return fmt.Errorf("deleting file: %w", err) + } + case "update": + command := exec.CommandContext( + ctx, + "git", + "-C", + root, + "apply", + "--whitespace=nowarn", + "-", + ) + command.Stdin = strings.NewReader(change.UnifiedDiff) + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Run(); err != nil { + return fmt.Errorf("applying unified diff: %w: %s", err, strings.TrimSpace(stderr.String())) + } + if change.MovePath != "" { + destination, err := safePath(root, change.MovePath) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return fmt.Errorf("creating move destination directory: %w", err) + } + if err := os.Rename(path, destination); err != nil { + return fmt.Errorf("moving updated file: %w", err) + } + } + } + return nil +} + +func safePath(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) { + return "", errors.New("path must be a non-empty relative path") + } + + clean := filepath.Clean(relative) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", errors.New("path escapes replay root") + } + if clean == ".git" || strings.HasPrefix(clean, ".git"+string(filepath.Separator)) { + return "", errors.New("Git metadata paths are not allowed") + } + + target := filepath.Join(root, clean) + relativeToRoot, err := filepath.Rel(root, target) + if err != nil || relativeToRoot == ".." || + strings.HasPrefix(relativeToRoot, ".."+string(filepath.Separator)) { + return "", errors.New("path escapes replay root") + } + + current := root + for _, component := range strings.Split(filepath.Dir(clean), string(filepath.Separator)) { + if component == "." || component == "" { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if os.IsNotExist(err) { + break + } + if err != nil { + return "", fmt.Errorf("inspecting path component: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return "", errors.New("symlinked path components are not allowed") + } + } + + return target, nil +} + +func validateDiffPaths(root, diff string) error { + for line := range strings.SplitSeq(diff, "\n") { + if line == "new file mode 120000" || line == "new mode 120000" { + return errors.New("unified diff may not create symlinks") + } + if !strings.HasPrefix(line, "--- ") && !strings.HasPrefix(line, "+++ ") { + continue + } + path := strings.TrimSpace(line[4:]) + path = strings.SplitN(path, "\t", 2)[0] + if path == "/dev/null" { + continue + } + path = strings.TrimPrefix(path, "a/") + path = strings.TrimPrefix(path, "b/") + if _, err := safePath(root, path); err != nil { + return fmt.Errorf("unsafe unified diff path: %w", err) + } + } + return nil +} diff --git a/internal/evidence/replay_test.go b/internal/evidence/replay_test.go new file mode 100644 index 0000000..fd098ff --- /dev/null +++ b/internal/evidence/replay_test.go @@ -0,0 +1,210 @@ +package evidence + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestExtractReplayPlanIncludesOrderedChangesThroughTargetTurn(t *testing.T) { + t.Parallel() + + root := t.TempDir() + sessionPath := filepath.Join(root, "session.jsonl") + writeSession(t, root, "session.jsonl", []map[string]any{ + event("session_meta", map[string]any{ + "cwd": "/private/repository", + "git": map[string]any{"commit_hash": "base123"}, + }), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "prior", + "success": true, + "changes": map[string]any{ + "/private/repository/first.txt": map[string]any{ + "type": "add", "content": "first\n", + }, + }, + }), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "target", + "success": true, + "changes": map[string]any{ + "/private/repository/second.txt": map[string]any{ + "type": "add", "content": "second\n", + }, + }, + }), + event("event_msg", map[string]any{"type": "task_complete", "turn_id": "target"}), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "later", + "success": true, + "changes": map[string]any{ + "later.txt": map[string]any{"type": "add", "content": "later\n"}, + }, + }), + }) + + plan, err := ExtractReplayPlan(sessionPath, "target") + if err != nil { + t.Fatalf("ExtractReplayPlan() error = %v", err) + } + + if plan.BaseCommit != "base123" { + t.Errorf("BaseCommit = %q, want base123", plan.BaseCommit) + } + if plan.RepositoryRoot != "/private/repository" { + t.Errorf("RepositoryRoot = %q, want /private/repository", plan.RepositoryRoot) + } + if len(plan.Changes) != 2 { + t.Fatalf("len(Changes) = %d, want 2", len(plan.Changes)) + } + if plan.Changes[0].Path != "first.txt" || plan.Changes[1].Path != "second.txt" { + t.Errorf("change order = %q, %q", plan.Changes[0].Path, plan.Changes[1].Path) + } + + encoded, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + if string(encoded) != "{}" { + t.Fatalf("encoded plan leaked reconstruction inputs: %s", encoded) + } +} + +func TestExtractReplayPlanRejectsRecordedPathOutsideRepository(t *testing.T) { + t.Parallel() + + root := t.TempDir() + sessionPath := filepath.Join(root, "session.jsonl") + writeSession(t, root, "session.jsonl", []map[string]any{ + event("session_meta", map[string]any{ + "cwd": "/private/repository", + "git": map[string]any{"commit_hash": "base123"}, + }), + event("event_msg", map[string]any{ + "type": "patch_apply_end", + "turn_id": "target", + "success": true, + "changes": map[string]any{ + "/private/other/escape.txt": map[string]any{ + "type": "add", "content": "escape\n", + }, + }, + }), + event("event_msg", map[string]any{"type": "task_complete", "turn_id": "target"}), + }) + + if _, err := ExtractReplayPlan(sessionPath, "target"); err == nil { + t.Fatal("ExtractReplayPlan() error = nil, want outside-repository rejection") + } +} + +func TestApplyChangesReconstructsOrderedFileState(t *testing.T) { + t.Parallel() + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "existing.txt"), []byte("old\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "obsolete.txt"), []byte("remove\n"), 0o600); err != nil { + t.Fatal(err) + } + + changes := []FileChange{ + { + Path: "existing.txt", + Type: "update", + UnifiedDiff: "diff --git a/existing.txt b/existing.txt\n" + + "--- a/existing.txt\n" + + "+++ b/existing.txt\n" + + "@@ -1 +1 @@\n" + + "-old\n" + + "+new\n", + }, + {Path: "nested/added.txt", Type: "add", Content: "created\n"}, + {Path: "obsolete.txt", Type: "delete"}, + } + + if err := ApplyChanges(context.Background(), root, changes); err != nil { + t.Fatalf("ApplyChanges() error = %v", err) + } + + assertFileContent(t, filepath.Join(root, "existing.txt"), "new\n") + assertFileContent(t, filepath.Join(root, "nested/added.txt"), "created\n") + if _, err := os.Stat(filepath.Join(root, "obsolete.txt")); !os.IsNotExist(err) { + t.Fatalf("obsolete file still exists or stat failed unexpectedly: %v", err) + } +} + +func TestApplyChangesRejectsUnsafePathsBeforeWriting(t *testing.T) { + t.Parallel() + + root := t.TempDir() + sentinel := filepath.Join(root, "sentinel.txt") + if err := os.WriteFile(sentinel, []byte("unchanged\n"), 0o600); err != nil { + t.Fatal(err) + } + + changes := []FileChange{ + {Path: "safe.txt", Type: "add", Content: "must not be written\n"}, + {Path: "../escape.txt", Type: "add", Content: "escaped\n"}, + } + + if err := ApplyChanges(context.Background(), root, changes); err == nil { + t.Fatal("ApplyChanges() error = nil, want unsafe path rejection") + } + if _, err := os.Stat(filepath.Join(root, "safe.txt")); !os.IsNotExist(err) { + t.Fatalf("safe change was applied before validation completed: %v", err) + } + assertFileContent(t, sentinel, "unchanged\n") +} + +func TestApplyChangesRejectsGitMetadata(t *testing.T) { + t.Parallel() + + root := t.TempDir() + err := ApplyChanges(context.Background(), root, []FileChange{ + {Path: ".git/config", Type: "add", Content: "malicious\n"}, + }) + if err == nil { + t.Fatal("ApplyChanges() error = nil, want .git path rejection") + } +} + +func TestApplyChangesRejectsDiffThatCreatesSymlink(t *testing.T) { + t.Parallel() + + root := t.TempDir() + err := ApplyChanges(context.Background(), root, []FileChange{ + { + Path: "escape", + Type: "update", + UnifiedDiff: "diff --git a/escape b/escape\n" + + "new file mode 120000\n" + + "--- /dev/null\n" + + "+++ b/escape\n" + + "@@ -0,0 +1 @@\n" + + "+../../outside\n", + }, + }) + if err == nil { + t.Fatal("ApplyChanges() error = nil, want symlink creation rejection") + } +} + +func assertFileContent(t *testing.T, path, want string) { + t.Helper() + + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(content) != want { + t.Errorf("%s content = %q, want %q", filepath.Base(path), content, want) + } +} diff --git a/scripts/evidence-audit/main.go b/scripts/evidence-audit/main.go new file mode 100644 index 0000000..237b30a --- /dev/null +++ b/scripts/evidence-audit/main.go @@ -0,0 +1,73 @@ +// Command evidence-audit produces content-free aggregate metrics from local +// agent history. It is a research utility, not part of the public GPTCode CLI. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/jadercorrea/gptcode/internal/evidence" +) + +func main() { + root := flag.String("codex-root", "", "path to the Codex sessions directory") + replaySession := flag.String("replay-session", "", "Codex JSONL session to replay") + replayTurn := flag.String("replay-turn", "", "turn ID to replay through") + replayRoot := flag.String("replay-root", "", "detached worktree receiving the replay") + flag.Parse() + + if *replaySession != "" || *replayTurn != "" || *replayRoot != "" { + replay(context.Background(), *replaySession, *replayTurn, *replayRoot) + return + } + if *root == "" { + fmt.Fprintln(os.Stderr, "usage: evidence-audit -codex-root ") + os.Exit(2) + } + + summary, err := evidence.ScanCodex(*root) + if err != nil { + fmt.Fprintf(os.Stderr, "evidence audit failed: %v\n", err) + os.Exit(1) + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(summary); err != nil { + fmt.Fprintf(os.Stderr, "writing evidence audit: %v\n", err) + os.Exit(1) + } +} + +func replay(ctx context.Context, sessionPath, turnID, root string) { + if sessionPath == "" || turnID == "" || root == "" { + fmt.Fprintln( + os.Stderr, + "replay requires -replay-session, -replay-turn, and -replay-root", + ) + os.Exit(2) + } + + plan, err := evidence.ExtractReplayPlan(sessionPath, turnID) + if err != nil { + fmt.Fprintln(os.Stderr, "extracting replay plan failed safely") + os.Exit(1) + } + if err := evidence.ApplyChanges(ctx, root, plan.Changes); err != nil { + fmt.Fprintln(os.Stderr, "applying replay plan failed safely") + os.Exit(1) + } + + result := struct { + AppliedChanges int `json:"applied_changes"` + }{ + AppliedChanges: len(plan.Changes), + } + if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { + fmt.Fprintf(os.Stderr, "writing replay result: %v\n", err) + os.Exit(1) + } +} From 8fe858b81c72625c214308a8aa72f6d25d14c2cb Mon Sep 17 00:00:00 2001 From: Jader Correa Date: Tue, 28 Jul 2026 17:20:47 -0300 Subject: [PATCH 2/3] research: make local agents measurable Local agent runs were not credible evidence: historical sessions lacked reproducible baselines, fresh runs hid progress until completion, Ollama used a fixed timeout, and retry budgets could be exhausted without measurable improvement. The same gaps made local execution slower, opaque, and easy to overstate. Build deterministic, replayable evaluation bundles around clean fixtures; ground planning and retries in repository contracts; stream verbose stages while preserving captured output; tune Ollama timeout and context for available hardware; and stop equivalent verification plateaus early. Strengthened fixtures retain false positives and failures, while reviewed Qwen cache and Devstral filesystem results demonstrate bounded, zero-API-cost capability. General reliability still requires repeatability across more fixtures. The Devstral filesystem result is one reviewed contract pass, and its generated patch is not claimed as a race-free filesystem primitive against an actively hostile concurrent process. --- _roadmap.md | 12 +- benchmarks/README.md | 106 ++++ .../fixtures/go-expiring-cache/cache.go | 53 ++ .../fixtures/go-expiring-cache/cache_test.go | 67 +++ benchmarks/fixtures/go-expiring-cache/go.mod | 3 + .../go-expiring-cache/quality_test.go | 22 + benchmarks/fixtures/go-expiring-cache/task.md | 16 + .../fixtures/go-ledger/concurrency_test.go | 68 +++ benchmarks/fixtures/go-ledger/go.mod | 3 + benchmarks/fixtures/go-ledger/ledger.go | 38 ++ benchmarks/fixtures/go-ledger/ledger_test.go | 33 ++ benchmarks/fixtures/go-ledger/quality_test.go | 41 ++ benchmarks/fixtures/go-ledger/task.md | 17 + benchmarks/fixtures/go-safe-store/go.mod | 3 + .../fixtures/go-safe-store/quality_test.go | 22 + benchmarks/fixtures/go-safe-store/store.go | 22 + .../fixtures/go-safe-store/store_test.go | 109 ++++ benchmarks/fixtures/go-safe-store/task.md | 16 + benchmarks/results/2026-07-28-local-smoke.md | 116 ++++ .../results/2026-07-29-local-filesystem.md | 93 ++++ .../local-devstral-small-2-safe-store.json | 43 ++ benchmarks/suites/local-gpt-oss.json | 69 +++ .../suites/local-qwen3-coder-cache.json | 39 ++ benchmarks/suites/local-qwen3-coder.json | 69 +++ internal/agents/editor.go | 37 +- internal/agents/editor_test.go | 110 +++- internal/agents/query.go | 12 + internal/agents/query_test.go | 30 + internal/autonomous/analyzer.go | 52 +- internal/autonomous/movement_test.go | 38 ++ internal/config/model_selector.go | 60 +- internal/config/model_selector_test.go | 64 +++ internal/evidence/bundle.go | 511 ++++++++++++++++++ internal/evidence/bundle_test.go | 232 ++++++++ internal/evidence/suite.go | 309 +++++++++++ internal/evidence/suite_test.go | 224 ++++++++ internal/llm/ollama.go | 47 +- internal/llm/ollama_test.go | 68 +++ internal/maestro/conductor.go | 104 +++- internal/maestro/formatter.go | 63 +++ internal/maestro/formatter_test.go | 67 +++ internal/maestro/repository_context.go | 66 +++ internal/maestro/repository_context_test.go | 49 ++ internal/maestro/verification_progress.go | 55 ++ .../maestro/verification_progress_test.go | 57 ++ internal/modes/model_selection.go | 7 + internal/modes/model_selection_test.go | 28 + internal/modes/research.go | 100 +++- internal/modes/research_test.go | 49 ++ internal/modes/review.go | 51 +- internal/modes/review_test.go | 33 ++ internal/observability/observer.go | 15 +- internal/observability/observer_test.go | 23 + scripts/evidence-run/main.go | 48 ++ scripts/evidence-suite/main.go | 65 +++ 55 files changed, 3659 insertions(+), 95 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/fixtures/go-expiring-cache/cache.go create mode 100644 benchmarks/fixtures/go-expiring-cache/cache_test.go create mode 100644 benchmarks/fixtures/go-expiring-cache/go.mod create mode 100644 benchmarks/fixtures/go-expiring-cache/quality_test.go create mode 100644 benchmarks/fixtures/go-expiring-cache/task.md create mode 100644 benchmarks/fixtures/go-ledger/concurrency_test.go create mode 100644 benchmarks/fixtures/go-ledger/go.mod create mode 100644 benchmarks/fixtures/go-ledger/ledger.go create mode 100644 benchmarks/fixtures/go-ledger/ledger_test.go create mode 100644 benchmarks/fixtures/go-ledger/quality_test.go create mode 100644 benchmarks/fixtures/go-ledger/task.md create mode 100644 benchmarks/fixtures/go-safe-store/go.mod create mode 100644 benchmarks/fixtures/go-safe-store/quality_test.go create mode 100644 benchmarks/fixtures/go-safe-store/store.go create mode 100644 benchmarks/fixtures/go-safe-store/store_test.go create mode 100644 benchmarks/fixtures/go-safe-store/task.md create mode 100644 benchmarks/results/2026-07-28-local-smoke.md create mode 100644 benchmarks/results/2026-07-29-local-filesystem.md create mode 100644 benchmarks/suites/local-devstral-small-2-safe-store.json create mode 100644 benchmarks/suites/local-gpt-oss.json create mode 100644 benchmarks/suites/local-qwen3-coder-cache.json create mode 100644 benchmarks/suites/local-qwen3-coder.json create mode 100644 internal/agents/query_test.go create mode 100644 internal/autonomous/movement_test.go create mode 100644 internal/evidence/bundle.go create mode 100644 internal/evidence/bundle_test.go create mode 100644 internal/evidence/suite.go create mode 100644 internal/evidence/suite_test.go create mode 100644 internal/llm/ollama_test.go create mode 100644 internal/maestro/formatter.go create mode 100644 internal/maestro/formatter_test.go create mode 100644 internal/maestro/verification_progress.go create mode 100644 internal/maestro/verification_progress_test.go create mode 100644 internal/modes/model_selection.go create mode 100644 internal/modes/model_selection_test.go create mode 100644 scripts/evidence-run/main.go create mode 100644 scripts/evidence-suite/main.go diff --git a/_roadmap.md b/_roadmap.md index ae183d9..258f7a1 100644 --- a/_roadmap.md +++ b/_roadmap.md @@ -40,5 +40,15 @@ engineering portfolio. - [ ] Isolate Live as an optional local observability protocol and retire unused training and experimental command surfaces. - [x] Build a content-free Codex history scanner and validate turn-level Git, patch, verification, and completion evidence against the local corpus. - [x] Build and container-validate a path-safe historical patch replayer, and establish that legacy Codex sessions lack the dirty-worktree baseline needed for deterministic replay. -- [ ] Capture an initial base diff or snapshot in new sessions, prove deterministic replay end to end, then add an OpenCode importer to challenge the vendor-neutral evidence model. +- [x] Capture complete initial and final snapshots for new agent experiments and prove deterministic bundle restoration end to end. +- [x] Run a real local-model evaluation against a concurrent Go fixture and use its failures to fix local routing, repository-grounded planning, and retry context. +- [x] Expand the evaluation corpus across concurrency, temporal semantics, and filesystem containment, with failure-inclusive suite aggregation and per-run time budgets. +- [x] Compare GPT-OSS and Qwen3-Coder on the same smoke corpus, review the apparent pass, and strengthen the contract after identifying a false positive. +- [x] Identify a local configuration that produces a human-reviewed fixture pass and measure three-run repeatability on the strengthened cache contract. +- [x] Find a fully GPU-backed local configuration that passes the strengthened safe-store contract, while retaining Qwen's extended-budget failure as negative evidence. +- [x] Stream verbose agent stages from the evidence suite without sacrificing the replayable output bundle. +- [x] Measure Devstral safe-store repeatability and retain the 0/3 timeout result alongside the earlier reviewed capability pass. +- [ ] Improve local safe-store convergence; neither Qwen nor Devstral currently supports a reliability claim on the strengthened contract. +- [ ] Extend reviewed repeatability beyond one fixture before publishing a general coding-agent success-rate claim. +- [ ] Add an OpenCode importer to challenge the vendor-neutral evidence model. - [ ] Raise coverage in legacy workflow packages without presenting the public fixture as repository-wide coverage. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..a3da318 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,106 @@ +# GPTCode agent evaluation + +The evaluation harness runs coding agents against the same clean Git fixture +and records an inspectable evidence bundle. It is designed to answer a narrow +question: did the agent produce a change that satisfies executable, +task-specific checks without losing the initial repository state? + +An experiment configuration names the agent process explicitly. Commands are +executed directly, without an implicit shell. The evidence directory must live +outside the evaluated repository so the run cannot accidentally measure its +own artifacts. + +```json +{ + "id": "go-ledger-deadlock", + "repository": "/tmp/go-ledger", + "output": "/tmp/go-ledger-evidence", + "agent": { + "name": "gptcode-local", + "args": [ + "/Users/jadercorrea/bin/gt", + "do", + "Fix the opposing-transfer deadlock without changing the public API" + ] + }, + "verifications": [ + { + "name": "tests-and-race-detector", + "args": ["go", "test", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] +} +``` + +Run it with: + +```bash +go run ./scripts/evidence-run -config /tmp/experiment.json +``` + +Every result retains failures as evidence. A successful agent process does not +make a run pass: every verification command must also exit successfully. + +## Repeatability suites + +The suite runner creates a fresh Git repository for every fixture repetition +and aggregates all outcomes. It runs sequentially so local CPU and memory +contention do not bias model comparisons. + +```bash +go run ./scripts/evidence-suite \ + -verbose \ + -config benchmarks/suites/local-gpt-oss.json \ + -output /tmp/gptcode-go-core-quality +``` + +`-verbose` streams the agent's inspectable stages while retaining identical +stdout and stderr in the evidence bundle. It exposes model selection, planning, +tool execution, retries, and deterministic checks; it does not print private +model chain-of-thought. + +Use `benchmarks/suites/local-qwen3-coder.json` for the equivalent Qwen suite. +Each committed local suite records and executes its required GPTCode profile +selection in `setup.json`; the agent name is not merely a user-supplied label. +`benchmarks/suites/local-qwen3-coder-cache.json` preserves the first +human-reviewed three-run result on the strengthened cache contract. +`benchmarks/suites/local-devstral-small-2-safe-store.json` records the +hardware-tuned 16k Devstral configuration that produced the first reviewed +pass on strengthened filesystem containment. + +The initial corpus covers three distinct failure classes: + +- lock ordering and deterministic concurrency; +- TTL boundary semantics under concurrent cache access; +- path traversal and symlink containment at a filesystem boundary. + +The committed suite uses three repetitions. During harness development, +`-repetitions 1` provides a smoke test without presenting it as a consistency +measurement. `run_timeout_seconds` is a hard agent budget; verification still +runs after a timeout so the resulting repository state remains evidence. +Fixtures with `require_failing_baseline` also write `baseline.json` and abort +the suite if every check already passes before the agent runs. + +Development results are recorded under `benchmarks/results/`. They distinguish +system defects from model failures and never promote a one-run smoke test to a +repeatability claim. + +## Bundle contract + +The generated directory contains: + +- `manifest.json`: schema, base commit, timestamps, and snapshot hashes. +- `initial.tar` and `final.tar`: deterministic repository snapshots excluding + Git metadata. +- `initial.patch`, `agent.patch`, and `final.patch`: binary-capable Git diffs. +- `events.jsonl`: lifecycle events. +- `commands.jsonl`: exact commands, outputs, exit codes, and durations. +- `verification.json`: the machine-readable pass/fail decision. +- `report.md`: a compact human-readable summary. + +Snapshots are intentionally complete. Only use controlled public fixtures: +real repositories may contain credentials or proprietary untracked files. diff --git a/benchmarks/fixtures/go-expiring-cache/cache.go b/benchmarks/fixtures/go-expiring-cache/cache.go new file mode 100644 index 0000000..71564b0 --- /dev/null +++ b/benchmarks/fixtures/go-expiring-cache/cache.go @@ -0,0 +1,53 @@ +package expiringcache + +import ( + "sync" + "time" +) + +type entry struct { + value string + expiresAt time.Time +} + +type Cache struct { + mu sync.RWMutex + entries map[string]entry + now func() time.Time +} + +func New() *Cache { + return newWithClock(time.Now) +} + +func newWithClock(now func() time.Time) *Cache { + return &Cache{ + entries: make(map[string]entry), + now: now, + } +} + +func (c *Cache) Set(key, value string, ttl time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.entries[key] = entry{ + value: value, + expiresAt: c.now().Add(ttl), + } +} + +func (c *Cache) Get(key string) (string, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + item, found := c.entries[key] + if !found { + return "", false + } + return item.value, true +} + +func (c *Cache) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + return len(c.entries) +} diff --git a/benchmarks/fixtures/go-expiring-cache/cache_test.go b/benchmarks/fixtures/go-expiring-cache/cache_test.go new file mode 100644 index 0000000..b65ef28 --- /dev/null +++ b/benchmarks/fixtures/go-expiring-cache/cache_test.go @@ -0,0 +1,67 @@ +package expiringcache + +import ( + "sync" + "testing" + "time" +) + +func TestGetReturnsLiveEntry(t *testing.T) { + now := time.Date(2026, time.July, 28, 12, 0, 0, 0, time.UTC) + cache := newWithClock(func() time.Time { return now }) + cache.Set("session", "active", time.Minute) + + value, found := cache.Get("session") + if !found || value != "active" { + t.Fatalf("Get() = %q, %v, want active, true", value, found) + } +} + +func TestGetEvictsExpiredEntry(t *testing.T) { + now := time.Date(2026, time.July, 28, 12, 0, 0, 0, time.UTC) + cache := newWithClock(func() time.Time { return now }) + cache.Set("session", "expired", time.Minute) + now = now.Add(time.Minute) + + if value, found := cache.Get("session"); found || value != "" { + t.Fatalf("Get() = %q, %v, want empty, false", value, found) + } + if length := cache.Len(); length != 0 { + t.Fatalf("Len() = %d, want expired entry removed", length) + } +} + +func TestLenPurgesExpiredEntriesWithoutLookup(t *testing.T) { + now := time.Date(2026, time.July, 28, 12, 0, 0, 0, time.UTC) + cache := newWithClock(func() time.Time { return now }) + cache.Set("expired", "stale", time.Minute) + cache.Set("live", "current", 2*time.Minute) + now = now.Add(time.Minute) + + if length := cache.Len(); length != 1 { + t.Fatalf("Len() = %d, want only the live entry", length) + } + if value, found := cache.Get("live"); !found || value != "current" { + t.Fatalf("Get(live) = %q, %v, want current, true", value, found) + } +} + +func TestCacheSupportsConcurrentReadersAndWriters(t *testing.T) { + cache := New() + var workers sync.WaitGroup + for worker := range 8 { + workers.Add(1) + go func() { + defer workers.Done() + for iteration := range 250 { + key := string(rune('a' + worker)) + cache.Set(key, "value", time.Minute) + cache.Get(key) + if iteration%10 == 0 { + cache.Len() + } + } + }() + } + workers.Wait() +} diff --git a/benchmarks/fixtures/go-expiring-cache/go.mod b/benchmarks/fixtures/go-expiring-cache/go.mod new file mode 100644 index 0000000..6926ffb --- /dev/null +++ b/benchmarks/fixtures/go-expiring-cache/go.mod @@ -0,0 +1,3 @@ +module example.com/expiringcache + +go 1.22 diff --git a/benchmarks/fixtures/go-expiring-cache/quality_test.go b/benchmarks/fixtures/go-expiring-cache/quality_test.go new file mode 100644 index 0000000..39f8ea9 --- /dev/null +++ b/benchmarks/fixtures/go-expiring-cache/quality_test.go @@ -0,0 +1,22 @@ +package expiringcache + +import ( + "bytes" + "go/format" + "os" + "testing" +) + +func TestCacheImplementationIsFormatted(t *testing.T) { + source, err := os.ReadFile("cache.go") + if err != nil { + t.Fatal(err) + } + formatted, err := format.Source(source) + if err != nil { + t.Fatalf("format.Source() error = %v", err) + } + if !bytes.Equal(source, formatted) { + t.Fatal("cache.go is not gofmt-formatted") + } +} diff --git a/benchmarks/fixtures/go-expiring-cache/task.md b/benchmarks/fixtures/go-expiring-cache/task.md new file mode 100644 index 0000000..b2ae0ef --- /dev/null +++ b/benchmarks/fixtures/go-expiring-cache/task.md @@ -0,0 +1,16 @@ +# Enforce cache expiration + +`Cache.Get` currently returns entries after their TTL has elapsed, and `Len` +continues to count those stale entries. + +Make expiration authoritative while preserving the exported API. A lookup at +the exact expiration instant is expired and must remove the stale entry. +Maintain race-free concurrent reads and writes; do not introduce background +goroutines or wall-clock sleeps. + +The implementation must pass: + +```text +go test -race ./... +go vet ./... +``` diff --git a/benchmarks/fixtures/go-ledger/concurrency_test.go b/benchmarks/fixtures/go-ledger/concurrency_test.go new file mode 100644 index 0000000..b87f599 --- /dev/null +++ b/benchmarks/fixtures/go-ledger/concurrency_test.go @@ -0,0 +1,68 @@ +package ledger + +import ( + "sync" + "testing" + "time" +) + +func TestSelfTransferCompletes(t *testing.T) { + account := NewAccount(1_000) + completed := make(chan error, 1) + go func() { + completed <- Transfer(account, account, 10) + }() + + select { + case err := <-completed: + if err != nil { + t.Fatalf("Transfer() error = %v", err) + } + case <-time.After(250 * time.Millisecond): + t.Fatal("self-transfer deadlocked") + } + + if balance := account.Balance(); balance != 1_000 { + t.Errorf("balance = %d, want 1000", balance) + } +} + +func TestOpposingTransfersComplete(t *testing.T) { + left := NewAccount(1_000) + right := NewAccount(1_000) + start := make(chan struct{}) + var workers sync.WaitGroup + workers.Add(2) + + go func() { + defer workers.Done() + <-start + for range 1_000 { + _ = Transfer(left, right, 1) + } + }() + go func() { + defer workers.Done() + <-start + for range 1_000 { + _ = Transfer(right, left, 1) + } + }() + + close(start) + completed := make(chan struct{}) + go func() { + workers.Wait() + close(completed) + }() + + select { + case <-completed: + case <-time.After(2 * time.Second): + t.Fatal("opposing transfers deadlocked") + } + + if total := left.Balance() + right.Balance(); total != 2_000 { + t.Errorf("total balance = %d, want 2000", total) + } +} diff --git a/benchmarks/fixtures/go-ledger/go.mod b/benchmarks/fixtures/go-ledger/go.mod new file mode 100644 index 0000000..4bb9136 --- /dev/null +++ b/benchmarks/fixtures/go-ledger/go.mod @@ -0,0 +1,3 @@ +module example.com/ledger + +go 1.22 diff --git a/benchmarks/fixtures/go-ledger/ledger.go b/benchmarks/fixtures/go-ledger/ledger.go new file mode 100644 index 0000000..3598890 --- /dev/null +++ b/benchmarks/fixtures/go-ledger/ledger.go @@ -0,0 +1,38 @@ +package ledger + +import ( + "errors" + "sync" +) + +var ErrInsufficientFunds = errors.New("insufficient funds") + +type Account struct { + mu sync.Mutex + balance int +} + +func NewAccount(balance int) *Account { + return &Account{balance: balance} +} + +func (a *Account) Balance() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.balance +} + +// Transfer moves funds while preserving the total held by both accounts. +func Transfer(from, to *Account, amount int) error { + from.mu.Lock() + defer from.mu.Unlock() + to.mu.Lock() + defer to.mu.Unlock() + + if from.balance < amount { + return ErrInsufficientFunds + } + from.balance -= amount + to.balance += amount + return nil +} diff --git a/benchmarks/fixtures/go-ledger/ledger_test.go b/benchmarks/fixtures/go-ledger/ledger_test.go new file mode 100644 index 0000000..251e47d --- /dev/null +++ b/benchmarks/fixtures/go-ledger/ledger_test.go @@ -0,0 +1,33 @@ +package ledger + +import ( + "errors" + "testing" +) + +func TestTransfer(t *testing.T) { + from := NewAccount(100) + to := NewAccount(25) + + if err := Transfer(from, to, 40); err != nil { + t.Fatalf("Transfer() error = %v", err) + } + if got := from.Balance(); got != 60 { + t.Errorf("source balance = %d, want 60", got) + } + if got := to.Balance(); got != 65 { + t.Errorf("destination balance = %d, want 65", got) + } +} + +func TestTransferRejectsInsufficientFunds(t *testing.T) { + from := NewAccount(10) + to := NewAccount(20) + + if err := Transfer(from, to, 11); !errors.Is(err, ErrInsufficientFunds) { + t.Fatalf("Transfer() error = %v, want ErrInsufficientFunds", err) + } + if got := from.Balance() + to.Balance(); got != 30 { + t.Errorf("total balance = %d, want 30", got) + } +} diff --git a/benchmarks/fixtures/go-ledger/quality_test.go b/benchmarks/fixtures/go-ledger/quality_test.go new file mode 100644 index 0000000..998c6cd --- /dev/null +++ b/benchmarks/fixtures/go-ledger/quality_test.go @@ -0,0 +1,41 @@ +package ledger + +import ( + "bytes" + "go/format" + "go/parser" + "go/token" + "os" + "strconv" + "testing" +) + +func TestLedgerImplementationIsFormatted(t *testing.T) { + source, err := os.ReadFile("ledger.go") + if err != nil { + t.Fatal(err) + } + formatted, err := format.Source(source) + if err != nil { + t.Fatalf("format.Source() error = %v", err) + } + if !bytes.Equal(source, formatted) { + t.Fatal("ledger.go is not gofmt-formatted") + } +} + +func TestLedgerImplementationAvoidsUnsafe(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "ledger.go", nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatal(err) + } + if path == "unsafe" { + t.Fatal("ledger.go must not depend on package unsafe for lock ordering") + } + } +} diff --git a/benchmarks/fixtures/go-ledger/task.md b/benchmarks/fixtures/go-ledger/task.md new file mode 100644 index 0000000..8d39fd4 --- /dev/null +++ b/benchmarks/fixtures/go-ledger/task.md @@ -0,0 +1,17 @@ +# Eliminate opposing-transfer deadlocks + +Two goroutines transferring funds in opposite directions can each hold one +account lock while waiting forever for the other. A transfer to the same +account deterministically attempts to lock the same mutex twice. + +Fix both deadlocks while preserving the existing exported API, error semantics, +and total balance. Keep the implementation idiomatic, `gofmt`-formatted, and +do not use package `unsafe`. The provided regression and quality tests must +pass: + +```text +go test -race ./... +go vet ./... +``` + +Do not weaken or remove synchronization. diff --git a/benchmarks/fixtures/go-safe-store/go.mod b/benchmarks/fixtures/go-safe-store/go.mod new file mode 100644 index 0000000..a05a534 --- /dev/null +++ b/benchmarks/fixtures/go-safe-store/go.mod @@ -0,0 +1,3 @@ +module example.com/safestore + +go 1.22 diff --git a/benchmarks/fixtures/go-safe-store/quality_test.go b/benchmarks/fixtures/go-safe-store/quality_test.go new file mode 100644 index 0000000..c368717 --- /dev/null +++ b/benchmarks/fixtures/go-safe-store/quality_test.go @@ -0,0 +1,22 @@ +package safestore + +import ( + "bytes" + "go/format" + "os" + "testing" +) + +func TestStoreImplementationIsFormatted(t *testing.T) { + source, err := os.ReadFile("store.go") + if err != nil { + t.Fatal(err) + } + formatted, err := format.Source(source) + if err != nil { + t.Fatalf("format.Source() error = %v", err) + } + if !bytes.Equal(source, formatted) { + t.Fatal("store.go is not gofmt-formatted") + } +} diff --git a/benchmarks/fixtures/go-safe-store/store.go b/benchmarks/fixtures/go-safe-store/store.go new file mode 100644 index 0000000..5c02463 --- /dev/null +++ b/benchmarks/fixtures/go-safe-store/store.go @@ -0,0 +1,22 @@ +package safestore + +import ( + "os" + "path/filepath" +) + +type Store struct { + root string +} + +func New(root string) *Store { + return &Store{root: root} +} + +func (s *Store) Write(name string, content []byte) error { + path := filepath.Join(s.root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + return os.WriteFile(path, content, 0o600) +} diff --git a/benchmarks/fixtures/go-safe-store/store_test.go b/benchmarks/fixtures/go-safe-store/store_test.go new file mode 100644 index 0000000..5564b7a --- /dev/null +++ b/benchmarks/fixtures/go-safe-store/store_test.go @@ -0,0 +1,109 @@ +package safestore + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestWriteStoresNestedFileInsideRoot(t *testing.T) { + root := t.TempDir() + store := New(root) + + if err := store.Write("reports/summary.txt", []byte("verified\n")); err != nil { + t.Fatalf("Write() error = %v", err) + } + content, err := os.ReadFile(filepath.Join(root, "reports", "summary.txt")) + if err != nil { + t.Fatal(err) + } + if string(content) != "verified\n" { + t.Fatalf("content = %q", content) + } +} + +func TestWriteAllowsDotsInsidePathSegment(t *testing.T) { + root := t.TempDir() + store := New(root) + + name := "reports/v1..v2.txt" + if err := store.Write(name, []byte("valid\n")); err != nil { + t.Fatalf("Write() error = %v for legitimate name", err) + } + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(name))) + if err != nil { + t.Fatal(err) + } + if string(content) != "valid\n" { + t.Fatalf("content = %q", content) + } +} + +func TestWriteRejectsTraversal(t *testing.T) { + parent := t.TempDir() + parent, err := filepath.EvalSymlinks(parent) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(parent, "store") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store := New(root) + + err = store.Write("../escaped.txt", []byte("escape")) + if err == nil { + t.Fatal("Write() error = nil, want traversal rejection") + } + if _, statErr := os.Stat(filepath.Join(parent, "escaped.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("file escaped root: %v", statErr) + } +} + +func TestWriteRejectsTraversalAfterNormalizingSegments(t *testing.T) { + parent := t.TempDir() + parent, err := filepath.EvalSymlinks(parent) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(parent, "store") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store := New(root) + + err = store.Write("reports/../../escaped.txt", []byte("escape")) + if err == nil { + t.Fatal("Write() error = nil, want normalized traversal rejection") + } + if _, statErr := os.Stat(filepath.Join(parent, "escaped.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("file escaped root: %v", statErr) + } +} + +func TestWriteRejectsAbsolutePath(t *testing.T) { + root := t.TempDir() + store := New(root) + + if err := store.Write(filepath.Join(t.TempDir(), "escaped.txt"), []byte("escape")); err == nil { + t.Fatal("Write() error = nil, want absolute path rejection") + } +} + +func TestWriteRejectsSymlinkedParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Fatal(err) + } + store := New(root) + + err := store.Write("linked/escaped.txt", []byte("escape")) + if err == nil { + t.Fatal("Write() error = nil, want symlink rejection") + } + if _, statErr := os.Stat(filepath.Join(outside, "escaped.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("file escaped through symlink: %v", statErr) + } +} diff --git a/benchmarks/fixtures/go-safe-store/task.md b/benchmarks/fixtures/go-safe-store/task.md new file mode 100644 index 0000000..60ee6ca --- /dev/null +++ b/benchmarks/fixtures/go-safe-store/task.md @@ -0,0 +1,16 @@ +# Contain writes inside the store root + +`Store.Write` accepts attacker-controlled names. Traversal, absolute paths, and +symlinked parent directories can currently write outside the configured root. + +Contain every write beneath the root while preserving the exported API and +support for legitimate nested paths. Reject unsafe input before creating +directories or files. Do not resolve the problem by deleting symlinks, changing +the process working directory, or weakening file permissions. + +The implementation must pass: + +```text +go test -race ./... +go vet ./... +``` diff --git a/benchmarks/results/2026-07-28-local-smoke.md b/benchmarks/results/2026-07-28-local-smoke.md new file mode 100644 index 0000000..cd13283 --- /dev/null +++ b/benchmarks/results/2026-07-28-local-smoke.md @@ -0,0 +1,116 @@ +# Local GPT-OSS smoke evaluation — 2026-07-28 + +This is a development smoke test, not a benchmark claim. Each fixture ran once; +repeatability requires the three repetitions declared by the suite. + +## Corpus + +| Fixture | Failure class | Deterministic contract | +| --- | --- | --- | +| `ledger-deadlock` | lock ordering and self-deadlock | tests, race detector, formatting, no `unsafe`, vet | +| `cache-expiration` | TTL boundary and stale eviction | tests, race detector, formatting, vet | +| `safe-store` | traversal and symlink containment | tests, race detector, formatting, vet | + +## First smoke + +The first run exposed harness and CLI defects before it could measure the model +fairly: + +- a planner stage escaped the local profile and attempted OpenRouter; +- structured `required_files` output could not be decoded; +- Go formatting remained probabilistic. + +Result: **0/3 passed**, median duration **1m57s**. These runs remain preserved +locally, but are not treated as model-performance evidence. + +## Corrected smoke + +The CLI was rebuilt with: + +- a hard prohibition on cloud backends in local mode; +- support for string and structured movement file references; +- deterministic `gofmt` on modified Go files before verification. + +| Fixture | Result | Duration | Final failure | +| --- | ---: | ---: | --- | +| `ledger-deadlock` | failed | 7m38s | invalid imports and unresolved `unsafe` reference | +| `cache-expiration` | failed | 5m50s | `Len` still counted the expired entry | +| `safe-store` | failed | 5m31s | malformed final source after retries | + +Corrected result: **0/3 passed**, median duration **5m50s**. + +No corrected run contacted OpenRouter. The model identified each root cause and +made relevant edits, but did not converge to a verified implementation within +three attempts. The current evidence therefore supports local exercisability, +not acceptable coding-agent reliability. + +## Next measurement + +Do not publish a comparative success-rate claim yet. First: + +1. enforce the ten-minute per-run budget in the suite; +2. test a faster local model on the same corpus; +3. run all three repetitions only for configurations that pass at least one + smoke fixture; +4. publish every run, including failures and timeouts. + +## Qwen3-Coder comparison + +The same corrected 1×3 smoke was run with `qwen3-coder:latest`. + +| Fixture | Initial result | Duration | +| --- | ---: | ---: | +| `ledger-deadlock` | failed | 1m38s | +| `cache-expiration` | failed | 1m41s | +| `safe-store` | passed contract | 1m51s | + +Initial aggregate: **1/3 passed**, median **1m41s**. This was substantially +faster than GPT-OSS, but human review found that the safe-store implementation +rejected every name containing `..`, including the legitimate +`reports/v1..v2.txt`. + +The contract was strengthened to preserve dots inside ordinary path segments. +A new Qwen safe-store run then failed in **1m35s** after three attempts. +Therefore the earlier pass is classified as a contract false positive, not a +quality-confirmed success. + +That conclusion changed after closing the editor feedback loop. + +## Grounded closed-loop Qwen result + +The CLI previously stopped the editor as soon as every planned file had been +touched. It also omitted tests and `task.md` from planning and retry context. +The corrected loop now: + +- grounds the initial plan and retries in implementation, tests, and task + constraints; +- lets the editor run verification and repair the same file within one turn; +- permits up to twenty tool calls for inspect → edit → verify → repair; +- rejects empty local-model answers and propagates Ollama HTTP errors. + +Human review found two more fixture weaknesses before accepting a result: + +- safe-store traversal appeared to pass only because `/var` is a symlink on + macOS; the strengthened test resolves the temporary root first; +- cache `Len` was not required to purge expired entries unless `Get` had + already been called; the strengthened test now checks independent cleanup. + +Qwen did not solve the strengthened safe-store contract within the ten-minute +budget. It did solve the strengthened cache contract, and three clean +repetitions produced: + +| Repetition | Result | Duration | +| --- | ---: | ---: | +| 1 | passed | 4m07s | +| 2 | passed | 3m56s | +| 3 | passed | 4m46s | + +Reviewed aggregate: **3/3 passed**, median **4m07s**, with `go test -race ./...` +and `go vet ./...` passing after every agent run. The three patches used +different locking strategies but all preserved the exported API, treated the +exact TTL boundary as expired, and purged stale entries through both `Get` and +`Len`. + +This is evidence of repeatability for one bounded temporal/concurrency task, +not a general coding-agent success-rate claim. Filesystem containment remains +an observed failure class. diff --git a/benchmarks/results/2026-07-29-local-filesystem.md b/benchmarks/results/2026-07-29-local-filesystem.md new file mode 100644 index 0000000..5d116e9 --- /dev/null +++ b/benchmarks/results/2026-07-29-local-filesystem.md @@ -0,0 +1,93 @@ +# Local filesystem-containment evaluation — 2026-07-29 + +This development evaluation compares local configurations on the strengthened +`safe-store` fixture. It is a single-run capability check, not a repeatability +or general success-rate claim. + +## Hardware and execution + +- Apple M1 Pro, 16 GPU cores, 32 GB unified memory. +- Ollama served every measured inference locally. +- `ollama ps` reported Qwen at 24 GB and **100% GPU** with a 65,536-token + context. +- Devstral Small 2 used 25 GB at 65,536 tokens and offloaded 8% to CPU. +- Setting `GPTCODE_OLLAMA_CONTEXT_LENGTH=16384` reduced Devstral to 17 GB and + kept it at **100% GPU**. + +Kimi was not evaluated locally. Current Kimi releases do not fit this hardware +as a comparable fully local coding-agent configuration. + +## Adaptive retry budget + +The CLI now stops after three equivalent deterministic-verification failures. +Timing noise is normalized before comparison, while a changed failure resets +the counter. This permits larger local-model budgets without spending every +attempt on a demonstrated plateau. + +Qwen received 30 minutes and eight allowed attempts. It stopped after six +attempts in **16m27s** when the symlink-parent failure repeated three times. +More time did not resolve the strengthened contract. + +## Devstral calibration + +The first Devstral run exposed a fixed 120-second Ollama client timeout before +the model could edit. The provider now defaults to ten minutes per request and +supports: + +- `GPTCODE_OLLAMA_TIMEOUT`, parsed as a Go duration; +- `GPTCODE_OLLAMA_CONTEXT_LENGTH`, sent to Ollama as `num_ctx`. + +At 32k, Devstral remained fully GPU-backed but timed out at 30 minutes after +repeating an incorrect `EvalSymlinks` strategy and entering a no-diff tool +loop. + +At 16k, Devstral completed in **26m08s**: + +| Gate | Result | +| --- | ---: | +| Agent exit | passed | +| `go test -count=1 -race ./...` | passed | +| `go vet ./...` | passed | +| Files modified | 1 | +| API cost | $0 | + +The first edit failed only the symlinked-parent test. The second edit passed +the deterministic contract. The evidence suite streamed the verbose execution +while preserving the same stdout and stderr in `commands.jsonl`. + +## Human review + +The Devstral patch correctly rejects the fixture's traversal, absolute-path, +and pre-existing symlink cases while permitting nested paths and ordinary dots +inside path segments. + +It is not presented as a general production filesystem primitive. Its +symlink check and subsequent write are separate operations, leaving a TOCTOU +window under a concurrently hostile filesystem. The result proves the bounded +fixture contract, not race-free containment against an active attacker. + +One reviewed pass justifies further repetition. It does not justify a general +model reliability claim. + +## Three-run repeatability result + +The same committed 16k configuration was then run three times from clean +fixtures with a 30-minute budget per run. + +| Repetition | Result | Duration | Final observed failure | +| --- | ---: | ---: | --- | +| 1 | timed out | 30m01s | legitimate nested paths rejected | +| 2 | timed out | 30m01s | legitimate nested paths rejected | +| 3 | timed out | 30m01s | absolute path and symlink escape accepted | + +Repeatability result: **0/3 passed**, median **30m01s**. + +All three runs remained fully local and GPU-backed. They produced relevant +edits, but did not converge before the task budget. The earlier reviewed pass +remains evidence that this configuration can solve the bounded contract; the +repeatability suite is stronger evidence that it cannot yet do so reliably. + +The runs also exposed nondeterministic Go temporary-directory IDs in otherwise +equivalent test failures. The plateau detector now normalizes those paths so +future local runs do not mistake ephemeral filesystem names for engineering +progress. diff --git a/benchmarks/suites/local-devstral-small-2-safe-store.json b/benchmarks/suites/local-devstral-small-2-safe-store.json new file mode 100644 index 0000000..7f3a08c --- /dev/null +++ b/benchmarks/suites/local-devstral-small-2-safe-store.json @@ -0,0 +1,43 @@ +{ + "id": "go-safe-store-devstral-small-2", + "output": "/tmp/gptcode-devstral-small-2-safe-store", + "repetitions": 1, + "run_timeout_seconds": 1800, + "setup": [ + { + "name": "select-local-devstral-profile", + "args": ["gt", "profiles", "use", "local.devstral"] + } + ], + "agent": { + "name": "gptcode-local-devstral-small-2-16k", + "args": [ + "/usr/bin/env", + "GPTCODE_OLLAMA_CONTEXT_LENGTH=16384", + "GPTCODE_OLLAMA_TIMEOUT=6m", + "gt", + "do", + "Read task.md, implement the requested fix, and meet every regression and quality constraint in the repository.", + "--verbose", + "--max-attempts", + "8" + ] + }, + "fixtures": [ + { + "id": "safe-store", + "source": "benchmarks/fixtures/go-safe-store", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + } + ] +} diff --git a/benchmarks/suites/local-gpt-oss.json b/benchmarks/suites/local-gpt-oss.json new file mode 100644 index 0000000..1251e37 --- /dev/null +++ b/benchmarks/suites/local-gpt-oss.json @@ -0,0 +1,69 @@ +{ + "id": "go-core-quality", + "output": "/tmp/gptcode-go-core-quality", + "repetitions": 3, + "run_timeout_seconds": 600, + "setup": [ + { + "name": "select-local-gpt-oss-profile", + "args": ["gt", "profiles", "use", "local.gptoss"] + } + ], + "agent": { + "name": "gptcode-local-gpt-oss", + "args": [ + "gt", + "do", + "Read task.md, implement the requested fix, and meet every regression and quality constraint in the repository.", + "--max-attempts", + "3" + ] + }, + "fixtures": [ + { + "id": "ledger-deadlock", + "source": "benchmarks/fixtures/go-ledger", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + }, + { + "id": "cache-expiration", + "source": "benchmarks/fixtures/go-expiring-cache", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + }, + { + "id": "safe-store", + "source": "benchmarks/fixtures/go-safe-store", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + } + ] +} diff --git a/benchmarks/suites/local-qwen3-coder-cache.json b/benchmarks/suites/local-qwen3-coder-cache.json new file mode 100644 index 0000000..f84d261 --- /dev/null +++ b/benchmarks/suites/local-qwen3-coder-cache.json @@ -0,0 +1,39 @@ +{ + "id": "go-cache-qwen3-coder-reviewed", + "output": "/tmp/gptcode-qwen-cache-reviewed", + "repetitions": 3, + "run_timeout_seconds": 600, + "setup": [ + { + "name": "select-local-qwen3-coder-profile", + "args": ["gt", "profiles", "use", "local.evaluation"] + } + ], + "agent": { + "name": "gptcode-local-qwen3-coder", + "args": [ + "gt", + "do", + "Read task.md, implement the requested fix, and meet every regression and quality constraint in the repository.", + "--max-attempts", + "3" + ] + }, + "fixtures": [ + { + "id": "cache-expiration", + "source": "benchmarks/fixtures/go-expiring-cache", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + } + ] +} diff --git a/benchmarks/suites/local-qwen3-coder.json b/benchmarks/suites/local-qwen3-coder.json new file mode 100644 index 0000000..17a460d --- /dev/null +++ b/benchmarks/suites/local-qwen3-coder.json @@ -0,0 +1,69 @@ +{ + "id": "go-core-quality-qwen3-coder", + "output": "/tmp/gptcode-go-core-quality-qwen3-coder", + "repetitions": 3, + "run_timeout_seconds": 600, + "setup": [ + { + "name": "select-local-qwen3-coder-profile", + "args": ["gt", "profiles", "use", "local.evaluation"] + } + ], + "agent": { + "name": "gptcode-local-qwen3-coder", + "args": [ + "gt", + "do", + "Read task.md, implement the requested fix, and meet every regression and quality constraint in the repository.", + "--max-attempts", + "3" + ] + }, + "fixtures": [ + { + "id": "ledger-deadlock", + "source": "benchmarks/fixtures/go-ledger", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + }, + { + "id": "cache-expiration", + "source": "benchmarks/fixtures/go-expiring-cache", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + }, + { + "id": "safe-store", + "source": "benchmarks/fixtures/go-safe-store", + "require_failing_baseline": true, + "verifications": [ + { + "name": "tests-race-and-quality-contract", + "args": ["go", "test", "-count=1", "-race", "./..."] + }, + { + "name": "static-analysis", + "args": ["go", "vet", "./..."] + } + ] + } + ] +} diff --git a/internal/agents/editor.go b/internal/agents/editor.go index 6d9d82b..f03d89e 100644 --- a/internal/agents/editor.go +++ b/internal/agents/editor.go @@ -103,7 +103,9 @@ WORKFLOW: 1. For file reading: Call read_file to get current content 2. For shell commands: Call run_command (e.g., "gh pr list", "go test", "npm run lint") 3. For file modification: Call apply_patch for small changes, or write_file for new files/large rewrites -4. **WHEN DONE**: Stop immediately. Do NOT call tools again. Return success message. +4. Run the relevant repository verification after editing. If it fails, use the + output to repair the implementation before returning. +5. **WHEN DONE**: Return a brief success message. CRITICAL RULES: - Use run_command for ANY shell operation (git, gh, tests, linters, etc) @@ -113,7 +115,7 @@ CRITICAL RULES: - NEVER use placeholders like "[previous content]" or "[rest of file]" - NEVER create fake/placeholder files instead of using run_command - **IDEMPOTENCY**: Before modifying, check if change already exists. Don't apply same patch twice -- **ONE CHANGE PER FILE**: After modifying a file, do NOT modify it again in same turn +- A failed command is actionable feedback: repair the same file when necessary - **GO PACKAGE NAMES**: When editing Go files, NEVER change the package declaration unless explicitly asked. If main.go has "package main", ALL files in the same directory MUST use "package main". Do NOT infer package names from filenames (e.g., utils.go should NOT have "package utils" if it's in a package main directory) EXAMPLE 1 - Using run_command (for shell operations): @@ -327,8 +329,10 @@ func (e *EditorAgent) Execute(ctx context.Context, history []llm.ChatMessage, st // Tool call processing loop: handles multiple sequential tool calls within a single editor run. // The outer retry logic (errors, validation failures) is controlled by Maestro's LoopDetector. // This internal loop is for processing a chain of tool calls (discovery → read → write). - // Set to 10 to allow complex tasks: 3-4 discovery calls + 2-3 reads + 2-3 writes - maxToolChainDepth := 10 + // Local models commonly need a longer inspect → edit → verify → repair chain + // than hosted models. Keep the outer conductor responsible for retries, but + // do not truncate a productive in-turn tool chain before validation. + maxToolChainDepth := 20 for iteration := 0; iteration < maxToolChainDepth; iteration++ { llmStart := time.Now() resp, err := e.provider.Chat(ctx, llm.ChatRequest{ @@ -461,9 +465,6 @@ func (e *EditorAgent) Execute(ctx context.Context, history []llm.ChatMessage, st fmt.Fprintf(os.Stderr, "[EDITOR] Executed %s: %s\n", tc.Name, result.Result[:min(50, len(result.Result))]) } } - if e.allExpectedFilesModified(modifiedFiles) { - return "All planned files changed; awaiting deterministic validation", modifiedFiles, nil - } continue } if editRequested(messages) && len(modifiedFiles) == 0 { @@ -565,9 +566,6 @@ func (e *EditorAgent) Execute(ctx context.Context, history []llm.ChatMessage, st fmt.Fprintf(os.Stderr, "[EDITOR] Executed %s: %s\n", tc.Name, result.Result[:min(50, len(result.Result))]) } } - if e.allExpectedFilesModified(modifiedFiles) { - return "All planned files changed; awaiting deterministic validation", modifiedFiles, nil - } } if editRequested(messages) && len(modifiedFiles) == 0 { @@ -577,25 +575,6 @@ func (e *EditorAgent) Execute(ctx context.Context, history []llm.ChatMessage, st return "Editor reached max iterations", modifiedFiles, nil } -func (e *EditorAgent) allExpectedFilesModified(modifiedFiles []string) bool { - if len(e.allowedFiles) == 0 { - return false - } - for _, expected := range e.allowedFiles { - found := false - for _, modified := range modifiedFiles { - if modified == expected || strings.HasSuffix(modified, expected) || strings.HasSuffix(expected, modified) { - found = true - break - } - } - if !found { - return false - } - } - return true -} - func min(a, b int) int { if a < b { return a diff --git a/internal/agents/editor_test.go b/internal/agents/editor_test.go index 634cbca..9cc12e8 100644 --- a/internal/agents/editor_test.go +++ b/internal/agents/editor_test.go @@ -2,6 +2,7 @@ package agents import ( "context" + "fmt" "os" "path/filepath" "strings" @@ -525,16 +526,19 @@ func TestEditor_EditTask_ModifiesEveryPlannedFileBeforeCompleting(t *testing.T) } } -func TestEditor_EditTask_StopsWhenEveryExpectedFileWasModified(t *testing.T) { +func TestEditor_EditTask_WaitsForModelCompletionAfterExpectedFilesChange(t *testing.T) { tmpDir := t.TempDir() mock := &mockProvider{ - responses: []llm.ChatResponse{{ - ToolCalls: []llm.ChatToolCall{{ - ID: "1", - Name: "write_file", - Arguments: `{"path":"counter.go","content":"package counter"}`, - }}, - }}, + responses: []llm.ChatResponse{ + { + ToolCalls: []llm.ChatToolCall{{ + ID: "1", + Name: "write_file", + Arguments: `{"path":"counter.go","content":"package counter"}`, + }}, + }, + {Text: "Implemented counter.go; ready for deterministic validation."}, + }, } editor := NewEditor(mock, tmpDir, "test-model") @@ -547,14 +551,98 @@ func TestEditor_EditTask_StopsWhenEveryExpectedFileWasModified(t *testing.T) { if err != nil { t.Fatalf("expected edit to complete: %v", err) } - if result != "All planned files changed; awaiting deterministic validation" { + if result != "Implemented counter.go; ready for deterministic validation." { t.Fatalf("unexpected completion: %q", result) } if len(modifiedFiles) != 1 || modifiedFiles[0] != "counter.go" { t.Fatalf("unexpected modified files: %v", modifiedFiles) } - if mock.callCount != 1 { - t.Fatalf("expected no extra model completion call, got %d", mock.callCount) + if mock.callCount != 2 { + t.Fatalf("expected an explicit completion after the write, got %d calls", mock.callCount) + } +} + +func TestEditor_EditTask_CanRepairSameFileAfterCommandFailure(t *testing.T) { + tmpDir := t.TempDir() + mock := &mockProvider{ + responses: []llm.ChatResponse{ + {ToolCalls: []llm.ChatToolCall{{ + ID: "1", + Name: "write_file", + Arguments: `{"path":"counter.go","content":"package counter\n\nvar Value = broken"}`, + }}}, + {ToolCalls: []llm.ChatToolCall{{ + ID: "2", + Name: "run_command", + Arguments: `{"command":"test \"$(tail -n 1 counter.go)\" = \"var Value = 1\""}`, + }}}, + {ToolCalls: []llm.ChatToolCall{{ + ID: "3", + Name: "apply_patch", + Arguments: `{"path":"counter.go","search":"var Value = broken","replace":"var Value = 1"}`, + }}}, + {Text: "Verification failure repaired."}, + }, + } + + editor := NewEditor(mock, tmpDir, "test-model") + editor.SetExpectedFiles([]string{"counter.go"}) + result, modifiedFiles, err := editor.Execute(context.Background(), []llm.ChatMessage{{ + Role: "user", + Content: "Fix counter.go and verify the result.", + }}, nil) + + if err != nil { + t.Fatalf("expected in-turn repair to complete: %v", err) + } + if result != "Verification failure repaired." { + t.Fatalf("unexpected completion: %q", result) + } + content, err := os.ReadFile(filepath.Join(tmpDir, "counter.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(content), "var Value = 1") { + t.Fatalf("same-file repair was not applied: %s", content) + } + if len(modifiedFiles) != 2 { + t.Fatalf("expected both writes to be recorded, got %v", modifiedFiles) + } +} + +func TestEditor_EditTask_AllowsGroundedToolChainsBeyondTenCalls(t *testing.T) { + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "counter.go"), []byte("package counter\n"), 0o644); err != nil { + t.Fatal(err) + } + + responses := make([]llm.ChatResponse, 0, 14) + for i := 0; i < 11; i++ { + responses = append(responses, llm.ChatResponse{ToolCalls: []llm.ChatToolCall{{ + ID: fmt.Sprintf("read-%d", i), + Name: "read_file", + Arguments: `{"path":"counter.go"}`, + }}}) + } + responses = append(responses, + llm.ChatResponse{ToolCalls: []llm.ChatToolCall{{ + ID: "write", + Name: "apply_patch", + Arguments: `{"path":"counter.go","search":"package counter","replace":"package counter\n\nvar Value = 1"}`, + }}}, + llm.ChatResponse{Text: "Grounded implementation complete."}, + ) + + editor := NewEditor(&mockProvider{responses: responses}, tmpDir, "test-model") + result, modifiedFiles, err := editor.Execute(context.Background(), []llm.ChatMessage{{ + Role: "user", + Content: "Fix counter.go after inspecting the repository evidence.", + }}, nil) + if err != nil { + t.Fatalf("expected a long grounded tool chain to complete: %v", err) + } + if result != "Grounded implementation complete." || len(modifiedFiles) != 1 { + t.Fatalf("unexpected result %q with files %v", result, modifiedFiles) } } diff --git a/internal/agents/query.go b/internal/agents/query.go index 5c1e2ca..7fe3275 100644 --- a/internal/agents/query.go +++ b/internal/agents/query.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/jadercorrea/gptcode/internal/llm" "github.com/jadercorrea/gptcode/internal/tools" @@ -135,6 +136,14 @@ func (q *QueryAgent) Execute(ctx context.Context, history []llm.ChatMessage, sta } if len(resp.ToolCalls) == 0 { + if strings.TrimSpace(resp.Text) == "" { + messages = append(messages, llm.ChatMessage{ + Role: "user", + Content: "The previous answer was empty. Answer the original question using the supplied " + + "repository evidence. Include concrete file paths and verification commands.", + }) + continue + } return resp.Text, nil } @@ -193,6 +202,9 @@ func (q *QueryAgent) Execute(ctx context.Context, history []llm.ChatMessage, sta if err != nil { return "", err } + if strings.TrimSpace(finalResp.Text) == "" { + return "", fmt.Errorf("query returned an empty answer after %d attempts", maxIterations+1) + } return finalResp.Text, nil } diff --git a/internal/agents/query_test.go b/internal/agents/query_test.go new file mode 100644 index 0000000..feae82b --- /dev/null +++ b/internal/agents/query_test.go @@ -0,0 +1,30 @@ +package agents + +import ( + "context" + "testing" + + "github.com/jadercorrea/gptcode/internal/llm" +) + +func TestQueryRetriesAnEmptyAnswer(t *testing.T) { + provider := &mockProvider{responses: []llm.ChatResponse{ + {Text: ""}, + {Text: "The cache uses an RWMutex and verifies behavior with go test -race ./..."}, + }} + query := NewQuery(provider, t.TempDir(), "test-model") + + result, err := query.Execute(context.Background(), []llm.ChatMessage{{ + Role: "user", + Content: "Explain cache concurrency.", + }}, nil) + if err != nil { + t.Fatalf("expected empty answer recovery: %v", err) + } + if result == "" { + t.Fatal("expected a grounded non-empty answer") + } + if provider.callCount != 2 { + t.Fatalf("expected one retry after the empty answer, got %d calls", provider.callCount) + } +} diff --git a/internal/autonomous/analyzer.go b/internal/autonomous/analyzer.go index ec825e6..fb46341 100644 --- a/internal/autonomous/analyzer.go +++ b/internal/autonomous/analyzer.go @@ -141,17 +141,51 @@ type TaskAnalysis struct { Movements []Movement `json:"movements,omitempty"` } +// FileReferences accepts the compact string format requested by the prompt +// and the structured path objects commonly emitted by local models. +type FileReferences []string + +func (references *FileReferences) UnmarshalJSON(data []byte) error { + var values []json.RawMessage + if err := json.Unmarshal(data, &values); err != nil { + return err + } + parsed := make(FileReferences, 0, len(values)) + for index, value := range values { + var path string + if err := json.Unmarshal(value, &path); err == nil { + if path == "" { + return fmt.Errorf("file reference %d is empty", index) + } + parsed = append(parsed, path) + continue + } + var structured struct { + Path string `json:"path"` + } + if err := json.Unmarshal(value, &structured); err != nil { + return fmt.Errorf("decoding file reference %d: %w", index, err) + } + if structured.Path == "" { + return fmt.Errorf("file reference %d is missing path", index) + } + parsed = append(parsed, structured.Path) + } + *references = parsed + return nil +} + // Movement represents a single phase in a complex task type Movement struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Goal string `json:"goal"` - Dependencies []string `json:"dependencies"` - RequiredFiles []string `json:"required_files"` - OutputFiles []string `json:"output_files"` - SuccessCriteria []string `json:"success_criteria"` - Status string `json:"status"` // "pending", "executing", "completed", "failed" + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Goal string `json:"goal"` + Dependencies []string `json:"dependencies"` + RequiredFiles FileReferences `json:"required_files"` + OutputFiles FileReferences `json:"output_files"` + SuccessCriteria []string `json:"success_criteria"` + Status string `json:"status"` // "pending", "executing", "completed", "failed" } // TaskAnalyzer analyzes tasks and decomposes them into movements if complex diff --git a/internal/autonomous/movement_test.go b/internal/autonomous/movement_test.go new file mode 100644 index 0000000..d8780ba --- /dev/null +++ b/internal/autonomous/movement_test.go @@ -0,0 +1,38 @@ +package autonomous + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestMovementAcceptsStringAndStructuredFileReferences(t *testing.T) { + var movement Movement + err := json.Unmarshal([]byte(`{ + "id": "movement-1", + "required_files": [ + "task.md", + {"path": "cache.go", "content_type": "read"} + ], + "output_files": [ + {"path": "cache.go"} + ] + }`), &movement) + if err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if want := (FileReferences{"task.md", "cache.go"}); !reflect.DeepEqual(movement.RequiredFiles, want) { + t.Fatalf("RequiredFiles = %v, want %v", movement.RequiredFiles, want) + } + if want := (FileReferences{"cache.go"}); !reflect.DeepEqual(movement.OutputFiles, want) { + t.Fatalf("OutputFiles = %v, want %v", movement.OutputFiles, want) + } +} + +func TestMovementRejectsStructuredFileWithoutPath(t *testing.T) { + var movement Movement + err := json.Unmarshal([]byte(`{"required_files":[{"content_type":"read"}]}`), &movement) + if err == nil { + t.Fatal("json.Unmarshal() error = nil, want missing path rejection") + } +} diff --git a/internal/config/model_selector.go b/internal/config/model_selector.go index f065996..7dfda79 100644 --- a/internal/config/model_selector.go +++ b/internal/config/model_selector.go @@ -381,6 +381,20 @@ func (ms *ModelSelector) SelectModel(action ActionType, language string, complex mode := ms.setup.Defaults.Mode defaultBackend := ms.setup.Defaults.Backend + // An explicit model in the active backend/profile is the user's routing + // decision. Approved models are fallbacks and must not silently move a + // local execution to a cloud backend. + if backendCfg, ok := ms.setup.Backend[defaultBackend]; ok { + configuredModel := explicitlyConfiguredModelForAction(backendCfg, ms.setup.Defaults.Profile, action) + if configuredModel != "" { + if os.Getenv("GPTCODE_DEBUG") == "1" { + fmt.Fprintf(os.Stderr, "[MODEL_SELECTOR] Using configured model for action=%s: %s/%s\n", + action, defaultBackend, configuredModel) + } + return defaultBackend, configuredModel, nil + } + } + // First, try approved models for this action approvedModels := ms.setup.GetApprovedModelsForAction(string(action)) if len(approvedModels) > 0 { @@ -389,6 +403,9 @@ func (ms *ModelSelector) SelectModel(action ActionType, language string, complex } for _, approved := range approvedModels { backend, model, err := ms.trySelectApprovedModel(approved.Model, action, language, complexity) + if err == nil && !modeAllowsBackend(mode, backend, ms.setup) { + err = fmt.Errorf("backend %q is not allowed in %s mode", backend, mode) + } if err == nil { if os.Getenv("GPTCODE_DEBUG") == "1" { fmt.Fprintf(os.Stderr, "[MODEL_SELECTOR] Approved model selected: %s/%s\n", backend, model) @@ -408,17 +425,6 @@ func (ms *ModelSelector) SelectModel(action ActionType, language string, complex return "", "", fmt.Errorf("all approved models failed for action=%s (check setup.yaml and models_catalog.json)", action) } - if backendCfg, ok := ms.setup.Backend[defaultBackend]; ok { - configuredModel := configuredModelForAction(backendCfg, ms.setup.Defaults.Profile, action) - if configuredModel != "" { - if os.Getenv("GPTCODE_DEBUG") == "1" { - fmt.Fprintf(os.Stderr, "[MODEL_SELECTOR] Using configured model for action=%s: %s/%s\n", - action, defaultBackend, configuredModel) - } - return defaultBackend, configuredModel, nil - } - } - type scoredModel struct { backend string model string @@ -504,6 +510,38 @@ func (ms *ModelSelector) SelectModel(action ActionType, language string, complex return best.backend, best.model, nil } +func modeAllowsBackend(mode, backendName string, setup *Setup) bool { + backend, configured := setup.Backend[backendName] + isLocal := backendName == "ollama" || (configured && backend.Type == "ollama") + switch mode { + case "local": + return isLocal + case "cloud": + return !isLocal + default: + return true + } +} + +func explicitlyConfiguredModelForAction(backend BackendConfig, profile string, action ActionType) string { + models := backend.AgentModels + if profile != "" && profile != "default" { + if configuredProfile, ok := backend.Profiles[profile]; ok { + models = configuredProfile.AgentModels + } + } + switch action { + case ActionEdit: + return models.Editor + case ActionResearch: + return models.Research + case ActionPlan, ActionReview, ActionRoute: + return models.Query + default: + return "" + } +} + func configuredModelForAction(backend BackendConfig, profile string, action ActionType) string { switch action { case ActionEdit: diff --git a/internal/config/model_selector_test.go b/internal/config/model_selector_test.go index 6cb9a80..f7f0e6b 100644 --- a/internal/config/model_selector_test.go +++ b/internal/config/model_selector_test.go @@ -55,6 +55,70 @@ func TestSelectModelPrefersConfiguredEditor(t *testing.T) { } } +func TestSelectModelPrefersExplicitLocalProfileOverApprovedCloudModel(t *testing.T) { + setup := &Setup{ + Backend: map[string]BackendConfig{ + "local": { + Type: "ollama", + Profiles: map[string]ProfileConfig{ + "evaluation": { + AgentModels: AgentModels{ + Query: "qwen3-coder:latest", + }, + }, + }, + }, + "openrouter": { + Type: "openai", + }, + }, + ApprovedModels: []ApprovedModel{ + {Model: "openai/o3-mini", ForActions: []string{string(ActionPlan)}}, + }, + } + setup.Defaults.Backend = "local" + setup.Defaults.Profile = "evaluation" + setup.Defaults.Mode = "local" + + selector := &ModelSelector{setup: setup} + backend, model, err := selector.SelectModel(ActionPlan, "go", "complex") + if err != nil { + t.Fatal(err) + } + if backend != "local" || model != "qwen3-coder:latest" { + t.Fatalf("selected %s/%s, want explicit local profile", backend, model) + } +} + +func TestSelectModelNeverRoutesLocalModeToApprovedCloudBackend(t *testing.T) { + setup := &Setup{ + Backend: map[string]BackendConfig{ + "local": { + Type: "ollama", + }, + "openrouter": { + Type: "openai", + }, + }, + ApprovedModels: []ApprovedModel{ + {Model: "google/gemini-2.5-pro", ForActions: []string{string(ActionPlan)}}, + }, + } + setup.Defaults.Backend = "local" + setup.Defaults.Mode = "local" + + selector := &ModelSelector{ + setup: setup, + catalog: map[string][]ModelInfo{ + "openrouter": {{ID: "google/gemini-2.5-pro"}}, + }, + } + backend, model, err := selector.SelectModel(ActionPlan, "go", "complex") + if err == nil { + t.Fatalf("SelectModel() = %s/%s, want local-mode cloud rejection", backend, model) + } +} + func TestApprovedOpenRouterModelUsesConfiguredBackend(t *testing.T) { setup := &Setup{ Backend: map[string]BackendConfig{ diff --git a/internal/evidence/bundle.go b/internal/evidence/bundle.go new file mode 100644 index 0000000..dd3cd75 --- /dev/null +++ b/internal/evidence/bundle.go @@ -0,0 +1,511 @@ +package evidence + +import ( + "archive/tar" + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +const EvidenceSchemaVersion = "gptcode.evidence/v1" + +type Command struct { + Name string `json:"name"` + Args []string `json:"args"` +} + +type Experiment struct { + ID string `json:"id"` + Repository string `json:"repository"` + Output string `json:"output"` + Agent Command `json:"agent"` + Verifications []Command `json:"verifications"` + AgentTimeout time.Duration `json:"-"` + Progress io.Writer `json:"-"` +} + +type Snapshot struct { + FileCount int `json:"file_count"` + SHA256 string `json:"sha256"` +} + +type Manifest struct { + SchemaVersion string `json:"schema_version"` + ExperimentID string `json:"experiment_id"` + BaseCommit string `json:"base_commit"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + InitialSnapshot Snapshot `json:"initial_snapshot"` + FinalSnapshot Snapshot `json:"final_snapshot"` +} + +type CommandResult struct { + Name string `json:"name"` + Args []string `json:"args"` + StartedAt time.Time `json:"started_at"` + Duration time.Duration `json:"duration"` + ExitCode int `json:"exit_code"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` +} + +type VerificationReport struct { + Passed bool `json:"passed"` + Commands []CommandResult `json:"commands"` +} + +type ExperimentResult struct { + Passed bool + TimedOut bool +} + +type bundleEvent struct { + Type string `json:"type"` + At time.Time `json:"at"` +} + +// RunExperiment executes one agent command and its deterministic checks in a +// clean Git fixture, preserving enough state to inspect and replay the run. +func RunExperiment(ctx context.Context, experiment Experiment) (ExperimentResult, error) { + if err := validateExperiment(experiment); err != nil { + return ExperimentResult{}, err + } + + baseCommit, err := gitOutput(ctx, experiment.Repository, "rev-parse", "HEAD") + if err != nil { + return ExperimentResult{}, fmt.Errorf("reading base commit: %w", err) + } + status, err := gitOutput(ctx, experiment.Repository, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return ExperimentResult{}, fmt.Errorf("checking repository state: %w", err) + } + if status != "" { + return ExperimentResult{}, errors.New("repository must be clean before an experiment") + } + if _, err := os.Stat(experiment.Output); err == nil { + return ExperimentResult{}, errors.New("evidence directory already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return ExperimentResult{}, fmt.Errorf("checking evidence directory: %w", err) + } + if err := os.Mkdir(experiment.Output, 0o700); err != nil { + return ExperimentResult{}, fmt.Errorf("creating evidence directory: %w", err) + } + + started := time.Now().UTC() + events, err := os.Create(filepath.Join(experiment.Output, "events.jsonl")) + if err != nil { + return ExperimentResult{}, fmt.Errorf("creating event log: %w", err) + } + defer events.Close() + eventEncoder := json.NewEncoder(events) + + initial, err := writeSnapshot(experiment.Repository, filepath.Join(experiment.Output, "initial.tar")) + if err != nil { + return ExperimentResult{}, fmt.Errorf("capturing initial snapshot: %w", err) + } + if err := writeGitDiff(ctx, experiment.Repository, filepath.Join(experiment.Output, "initial.patch")); err != nil { + return ExperimentResult{}, err + } + if err := eventEncoder.Encode(bundleEvent{Type: "agent_started", At: time.Now().UTC()}); err != nil { + return ExperimentResult{}, fmt.Errorf("recording agent start: %w", err) + } + + agentContext := ctx + cancelAgent := func() {} + if experiment.AgentTimeout > 0 { + agentContext, cancelAgent = context.WithTimeout(ctx, experiment.AgentTimeout) + } + var progress []io.Writer + if experiment.Progress != nil { + progress = append(progress, experiment.Progress) + } + agentResult := runCommand(agentContext, experiment.Repository, experiment.Agent, progress...) + timedOut := errors.Is(agentContext.Err(), context.DeadlineExceeded) + cancelAgent() + if err := appendJSONLine(filepath.Join(experiment.Output, "commands.jsonl"), agentResult); err != nil { + return ExperimentResult{}, err + } + if err := eventEncoder.Encode(bundleEvent{Type: "agent_completed", At: time.Now().UTC()}); err != nil { + return ExperimentResult{}, fmt.Errorf("recording agent completion: %w", err) + } + if err := writeGitDiff(ctx, experiment.Repository, filepath.Join(experiment.Output, "agent.patch")); err != nil { + return ExperimentResult{}, err + } + + verification := VerificationReport{Passed: agentResult.ExitCode == 0} + for _, check := range experiment.Verifications { + result := runCommand(ctx, experiment.Repository, check) + verification.Commands = append(verification.Commands, result) + if result.ExitCode != 0 { + verification.Passed = false + } + if err := appendJSONLine(filepath.Join(experiment.Output, "commands.jsonl"), result); err != nil { + return ExperimentResult{}, err + } + } + if err := writeJSON(filepath.Join(experiment.Output, "verification.json"), verification); err != nil { + return ExperimentResult{}, err + } + if err := eventEncoder.Encode(bundleEvent{Type: "verification_completed", At: time.Now().UTC()}); err != nil { + return ExperimentResult{}, fmt.Errorf("recording verification completion: %w", err) + } + + finalSnapshot, err := writeSnapshot(experiment.Repository, filepath.Join(experiment.Output, "final.tar")) + if err != nil { + return ExperimentResult{}, fmt.Errorf("capturing final snapshot: %w", err) + } + if err := writeGitDiff(ctx, experiment.Repository, filepath.Join(experiment.Output, "final.patch")); err != nil { + return ExperimentResult{}, err + } + + manifest := Manifest{ + SchemaVersion: EvidenceSchemaVersion, + ExperimentID: experiment.ID, + BaseCommit: strings.TrimSpace(baseCommit), + StartedAt: started, + CompletedAt: time.Now().UTC(), + InitialSnapshot: initial, + FinalSnapshot: finalSnapshot, + } + if err := writeJSON(filepath.Join(experiment.Output, "manifest.json"), manifest); err != nil { + return ExperimentResult{}, err + } + if err := writeReport(filepath.Join(experiment.Output, "report.md"), manifest, agentResult, verification); err != nil { + return ExperimentResult{}, err + } + + return ExperimentResult{Passed: verification.Passed, TimedOut: timedOut}, nil +} + +func validateExperiment(experiment Experiment) error { + if experiment.ID == "" { + return errors.New("experiment ID is required") + } + if experiment.Repository == "" || experiment.Output == "" { + return errors.New("repository and output paths are required") + } + if len(experiment.Agent.Args) == 0 || experiment.Agent.Name == "" { + return errors.New("agent name and executable are required") + } + if len(experiment.Verifications) == 0 { + return errors.New("at least one deterministic verification is required") + } + repository, err := filepath.Abs(experiment.Repository) + if err != nil { + return fmt.Errorf("resolving repository: %w", err) + } + output, err := filepath.Abs(experiment.Output) + if err != nil { + return fmt.Errorf("resolving output: %w", err) + } + relative, err := filepath.Rel(repository, output) + if err != nil { + return fmt.Errorf("comparing repository and output: %w", err) + } + if relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return errors.New("evidence output must be outside the evaluated repository") + } + for _, command := range experiment.Verifications { + if command.Name == "" || len(command.Args) == 0 { + return errors.New("verification name and executable are required") + } + } + return nil +} + +func runCommand(ctx context.Context, directory string, command Command, mirrors ...io.Writer) CommandResult { + started := time.Now().UTC() + process := exec.CommandContext(ctx, command.Args[0], command.Args[1:]...) + process.Dir = directory + var stdout, stderr strings.Builder + stdoutWriters := []io.Writer{&stdout} + stderrWriters := []io.Writer{&stderr} + if len(mirrors) > 0 { + progress := &synchronizedWriter{writer: io.MultiWriter(mirrors...)} + stdoutWriters = append(stdoutWriters, progress) + stderrWriters = append(stderrWriters, progress) + } + process.Stdout = io.MultiWriter(stdoutWriters...) + process.Stderr = io.MultiWriter(stderrWriters...) + err := process.Run() + + exitCode := 0 + if err != nil { + exitCode = -1 + var exitError *exec.ExitError + if errors.As(err, &exitError) { + exitCode = exitError.ExitCode() + } + } + return CommandResult{ + Name: command.Name, + Args: append([]string(nil), command.Args...), + StartedAt: started, + Duration: time.Since(started), + ExitCode: exitCode, + Stdout: stdout.String(), + Stderr: stderr.String(), + } +} + +type synchronizedWriter struct { + mu sync.Mutex + writer io.Writer +} + +func (w *synchronizedWriter) Write(content []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.writer.Write(content) +} + +func writeSnapshot(root, destination string) (Snapshot, error) { + paths, err := snapshotPaths(root) + if err != nil { + return Snapshot{}, err + } + file, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return Snapshot{}, err + } + hash := sha256.New() + writer := tar.NewWriter(io.MultiWriter(file, hash)) + for _, relative := range paths { + path := filepath.Join(root, relative) + info, err := os.Lstat(path) + if err != nil { + writer.Close() + file.Close() + return Snapshot{}, err + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + writer.Close() + file.Close() + return Snapshot{}, err + } + header.Name = filepath.ToSlash(relative) + header.ModTime = time.Unix(0, 0) + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + header.Uid, header.Gid = 0, 0 + header.Uname, header.Gname = "", "" + if err := writer.WriteHeader(header); err != nil { + writer.Close() + file.Close() + return Snapshot{}, err + } + if info.Mode().IsRegular() { + source, err := os.Open(path) + if err != nil { + writer.Close() + file.Close() + return Snapshot{}, err + } + _, copyErr := io.Copy(writer, source) + closeErr := source.Close() + if copyErr != nil { + writer.Close() + file.Close() + return Snapshot{}, copyErr + } + if closeErr != nil { + writer.Close() + file.Close() + return Snapshot{}, closeErr + } + } + } + if err := writer.Close(); err != nil { + file.Close() + return Snapshot{}, err + } + if err := file.Close(); err != nil { + return Snapshot{}, err + } + return Snapshot{FileCount: len(paths), SHA256: hex.EncodeToString(hash.Sum(nil))}, nil +} + +func snapshotPaths(root string) ([]string, error) { + var paths []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + if relative == "." { + return nil + } + if relative == ".git" { + return filepath.SkipDir + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() && !info.IsDir() { + return fmt.Errorf("unsupported snapshot file type at %q", relative) + } + paths = append(paths, relative) + return nil + }) + sort.Strings(paths) + return paths, err +} + +// RestoreSnapshot safely restores a bundle snapshot into an empty directory. +func RestoreSnapshot(snapshotPath, destination string) error { + file, err := os.Open(snapshotPath) + if err != nil { + return err + } + defer file.Close() + reader := tar.NewReader(bufio.NewReader(file)) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + path, err := safePath(destination, filepath.FromSlash(header.Name)) + if err != nil { + return fmt.Errorf("unsafe snapshot entry: %w", err) + } + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, header.FileInfo().Mode().Perm()); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + output, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, header.FileInfo().Mode().Perm()) + if err != nil { + return err + } + _, copyErr := io.Copy(output, reader) + closeErr := output.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("unsupported snapshot entry type %d", header.Typeflag) + } + } +} + +func gitOutput(ctx context.Context, root string, args ...string) (string, error) { + command := exec.CommandContext(ctx, "git", args...) + command.Dir = root + command.Env = isolatedGitEnvironment() + output, err := command.CombinedOutput() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, output) + } + return strings.TrimSpace(string(output)), nil +} + +func isolatedGitEnvironment() []string { + blocked := map[string]struct{}{ + "GIT_ALTERNATE_OBJECT_DIRECTORIES": {}, + "GIT_COMMON_DIR": {}, + "GIT_DIR": {}, + "GIT_INDEX_FILE": {}, + "GIT_OBJECT_DIRECTORY": {}, + "GIT_PREFIX": {}, + "GIT_WORK_TREE": {}, + } + environment := make([]string, 0, len(os.Environ())) + for _, variable := range os.Environ() { + name, _, _ := strings.Cut(variable, "=") + if _, found := blocked[name]; !found { + environment = append(environment, variable) + } + } + return environment +} + +func writeGitDiff(ctx context.Context, root, destination string) error { + diff, err := gitOutput(ctx, root, "diff", "--binary", "--no-ext-diff", "HEAD") + if err != nil { + return fmt.Errorf("capturing Git diff: %w", err) + } + if diff != "" { + diff += "\n" + } + if err := os.WriteFile(destination, []byte(diff), 0o600); err != nil { + return fmt.Errorf("writing Git diff: %w", err) + } + return nil +} + +func appendJSONLine(path string, value any) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("opening command log: %w", err) + } + defer file.Close() + if err := json.NewEncoder(file).Encode(value); err != nil { + return fmt.Errorf("writing command log: %w", err) + } + return nil +} + +func writeJSON(path string, value any) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + file.Close() + return err + } + return file.Close() +} + +func writeReport(path string, manifest Manifest, agent CommandResult, verification VerificationReport) error { + status := "FAILED" + if verification.Passed { + status = "PASSED" + } + report := fmt.Sprintf( + "# Evidence report: %s\n\n"+ + "- Result: **%s**\n"+ + "- Base commit: `%s`\n"+ + "- Agent exit code: `%d`\n"+ + "- Verification commands: `%d`\n"+ + "- Initial snapshot: `%s`\n"+ + "- Final snapshot: `%s`\n", + manifest.ExperimentID, + status, + manifest.BaseCommit, + agent.ExitCode, + len(verification.Commands), + manifest.InitialSnapshot.SHA256, + manifest.FinalSnapshot.SHA256, + ) + return os.WriteFile(path, []byte(report), 0o600) +} diff --git a/internal/evidence/bundle_test.go b/internal/evidence/bundle_test.go new file mode 100644 index 0000000..fe4982b --- /dev/null +++ b/internal/evidence/bundle_test.go @@ -0,0 +1,232 @@ +package evidence + +import ( + "bytes" + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestRunCommandMirrorsOutputWhilePreservingEvidence(t *testing.T) { + t.Parallel() + + var progress bytes.Buffer + result := runCommand( + context.Background(), + t.TempDir(), + Command{ + Name: "visible-agent", + Args: []string{"/bin/sh", "-c", "printf 'stage: analyze\\n'; printf 'stage: verify\\n' >&2"}, + }, + &progress, + ) + + if result.ExitCode != 0 { + t.Fatalf("exit code = %d, want 0", result.ExitCode) + } + if result.Stdout != "stage: analyze\n" { + t.Fatalf("stdout = %q", result.Stdout) + } + if result.Stderr != "stage: verify\n" { + t.Fatalf("stderr = %q", result.Stderr) + } + if got := progress.String(); !strings.Contains(got, "stage: analyze") || !strings.Contains(got, "stage: verify") { + t.Fatalf("mirrored output = %q", got) + } +} + +func TestRunExperimentProducesReplayableEvidenceBundle(t *testing.T) { + t.Parallel() + + repository := newFixtureRepository(t) + output := filepath.Join(t.TempDir(), "evidence") + + result, err := RunExperiment(context.Background(), Experiment{ + ID: "fix-greeting", + Repository: repository, + Output: output, + Agent: Command{ + Name: "fixture-agent", + Args: []string{"/bin/sh", "-c", "printf 'hello, evidence\\n' > greeting.txt"}, + }, + Verifications: []Command{ + { + Name: "greeting-test", + Args: []string{"/bin/sh", "-c", "test \"$(cat greeting.txt)\" = 'hello, evidence'"}, + }, + }, + }) + if err != nil { + t.Fatalf("RunExperiment() error = %v", err) + } + if !result.Passed { + t.Fatal("RunExperiment() Passed = false, want true") + } + + for _, name := range []string{ + "manifest.json", + "initial.tar", + "initial.patch", + "events.jsonl", + "commands.jsonl", + "agent.patch", + "verification.json", + "final.tar", + "final.patch", + "report.md", + } { + if _, err := os.Stat(filepath.Join(output, name)); err != nil { + t.Errorf("%s was not created: %v", name, err) + } + } + + var manifest Manifest + readJSONFile(t, filepath.Join(output, "manifest.json"), &manifest) + if manifest.SchemaVersion != EvidenceSchemaVersion { + t.Errorf("SchemaVersion = %q, want %q", manifest.SchemaVersion, EvidenceSchemaVersion) + } + if manifest.InitialSnapshot.SHA256 == "" || manifest.FinalSnapshot.SHA256 == "" { + t.Error("snapshot hashes must be recorded") + } + if manifest.InitialSnapshot.SHA256 == manifest.FinalSnapshot.SHA256 { + t.Error("initial and final snapshot hashes unexpectedly match") + } + + replay := t.TempDir() + if err := RestoreSnapshot(filepath.Join(output, "final.tar"), replay); err != nil { + t.Fatalf("RestoreSnapshot() error = %v", err) + } + content, err := os.ReadFile(filepath.Join(replay, "greeting.txt")) + if err != nil { + t.Fatal(err) + } + if string(content) != "hello, evidence\n" { + t.Errorf("replayed greeting = %q", content) + } +} + +func TestRunExperimentRejectsDirtyRepository(t *testing.T) { + t.Parallel() + + repository := newFixtureRepository(t) + if err := os.WriteFile(filepath.Join(repository, "dirty.txt"), []byte("not captured\n"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := RunExperiment(context.Background(), Experiment{ + ID: "dirty", + Repository: repository, + Output: filepath.Join(t.TempDir(), "evidence"), + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + }) + if err == nil { + t.Fatal("RunExperiment() error = nil, want dirty repository rejection") + } +} + +func TestRunExperimentRejectsExistingEvidenceDirectory(t *testing.T) { + t.Parallel() + + repository := newFixtureRepository(t) + output := filepath.Join(t.TempDir(), "evidence") + if err := os.Mkdir(output, 0o700); err != nil { + t.Fatal(err) + } + + _, err := RunExperiment(context.Background(), Experiment{ + ID: "existing-output", + Repository: repository, + Output: output, + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + }) + if err == nil { + t.Fatal("RunExperiment() error = nil, want existing output rejection") + } +} + +func TestRunExperimentRequiresDeterministicVerification(t *testing.T) { + t.Parallel() + + repository := newFixtureRepository(t) + _, err := RunExperiment(context.Background(), Experiment{ + ID: "no-verification", + Repository: repository, + Output: filepath.Join(t.TempDir(), "evidence"), + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + }) + if err == nil { + t.Fatal("RunExperiment() error = nil, want missing verification rejection") + } +} + +func TestRunExperimentRecordsFailedVerificationWithoutDiscardingEvidence(t *testing.T) { + t.Parallel() + + repository := newFixtureRepository(t) + output := filepath.Join(t.TempDir(), "evidence") + + result, err := RunExperiment(context.Background(), Experiment{ + ID: "failed-check", + Repository: repository, + Output: output, + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + Verifications: []Command{ + {Name: "expected-failure", Args: []string{"/usr/bin/false"}}, + }, + }) + if err != nil { + t.Fatalf("RunExperiment() error = %v", err) + } + if result.Passed { + t.Fatal("RunExperiment() Passed = true, want false") + } + + var verification VerificationReport + readJSONFile(t, filepath.Join(output, "verification.json"), &verification) + if verification.Passed || len(verification.Commands) != 1 || verification.Commands[0].ExitCode == 0 { + t.Fatalf("unexpected verification report: %+v", verification) + } +} + +func newFixtureRepository(t *testing.T) string { + t.Helper() + + root := t.TempDir() + runTestCommand(t, root, "git", "init", "-q") + runTestCommand(t, root, "git", "config", "user.email", "evidence@example.test") + runTestCommand(t, root, "git", "config", "user.name", "Evidence Test") + if err := os.WriteFile(filepath.Join(root, "greeting.txt"), []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + runTestCommand(t, root, "git", "add", "greeting.txt") + runTestCommand(t, root, "git", "commit", "-qm", "fixture") + return root +} + +func runTestCommand(t *testing.T, directory, name string, args ...string) { + t.Helper() + command := exec.Command(name, args...) + command.Dir = directory + if name == "git" { + command.Env = isolatedGitEnvironment() + } + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", name, err, output) + } +} + +func readJSONFile(t *testing.T, path string, destination any) { + t.Helper() + + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(content, destination); err != nil { + t.Fatal(err) + } +} diff --git a/internal/evidence/suite.go b/internal/evidence/suite.go new file mode 100644 index 0000000..8065cb4 --- /dev/null +++ b/internal/evidence/suite.go @@ -0,0 +1,309 @@ +package evidence + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "time" +) + +const SuiteSchemaVersion = "gptcode.evidence-suite/v1" + +var suiteIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +type Suite struct { + ID string `json:"id"` + Output string `json:"output"` + Repetitions int `json:"repetitions"` + RunTimeoutSeconds int `json:"run_timeout_seconds"` + Setup []Command `json:"setup,omitempty"` + Agent Command `json:"agent"` + Fixtures []SuiteFixture `json:"fixtures"` + Progress io.Writer `json:"-"` +} + +type SuiteFixture struct { + ID string `json:"id"` + Source string `json:"source"` + RequireFailingBaseline bool `json:"require_failing_baseline"` + Verifications []Command `json:"verifications"` +} + +type SuiteRun struct { + FixtureID string `json:"fixture_id"` + Repetition int `json:"repetition"` + Passed bool `json:"passed"` + TimedOut bool `json:"timed_out"` + Duration time.Duration `json:"duration"` + EvidencePath string `json:"evidence_path"` +} + +type SuiteSummary struct { + SchemaVersion string `json:"schema_version"` + SuiteID string `json:"suite_id"` + TotalRuns int `json:"total_runs"` + PassedRuns int `json:"passed_runs"` + PassRate float64 `json:"pass_rate"` + MedianDuration time.Duration `json:"median_duration"` + Runs []SuiteRun `json:"runs"` +} + +// RunSuite executes every fixture and repetition sequentially. Sequential +// execution keeps local-model resource contention from biasing comparisons. +func RunSuite(ctx context.Context, suite Suite) (SuiteSummary, error) { + if err := validateSuite(suite); err != nil { + return SuiteSummary{}, err + } + if _, err := os.Stat(suite.Output); err == nil { + return SuiteSummary{}, errors.New("suite output already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return SuiteSummary{}, fmt.Errorf("checking suite output: %w", err) + } + if err := os.Mkdir(suite.Output, 0o700); err != nil { + return SuiteSummary{}, fmt.Errorf("creating suite output: %w", err) + } + if len(suite.Setup) > 0 { + setup := runVerification(ctx, ".", suite.Setup) + if err := writeJSON(filepath.Join(suite.Output, "setup.json"), setup); err != nil { + return SuiteSummary{}, fmt.Errorf("writing suite setup evidence: %w", err) + } + if !setup.Passed { + return SuiteSummary{}, errors.New("suite setup command failed") + } + } + + summary := SuiteSummary{ + SchemaVersion: SuiteSchemaVersion, + SuiteID: suite.ID, + } + var durations []time.Duration + for _, fixture := range suite.Fixtures { + for repetition := 1; repetition <= suite.Repetitions; repetition++ { + if err := ctx.Err(); err != nil { + return summary, err + } + runDirectory := filepath.Join( + suite.Output, + "runs", + fixture.ID, + fmt.Sprintf("%03d", repetition), + ) + workspace := filepath.Join(runDirectory, "workspace") + evidencePath := filepath.Join(runDirectory, "evidence") + if err := os.MkdirAll(runDirectory, 0o700); err != nil { + return summary, fmt.Errorf("creating run directory: %w", err) + } + if err := copyFixture(fixture.Source, workspace); err != nil { + return summary, fmt.Errorf("copying fixture %q: %w", fixture.ID, err) + } + if err := initializeFixtureRepository(ctx, workspace); err != nil { + return summary, fmt.Errorf("initializing fixture %q: %w", fixture.ID, err) + } + baseline := runVerification(ctx, workspace, fixture.Verifications) + if err := writeJSON(filepath.Join(runDirectory, "baseline.json"), baseline); err != nil { + return summary, fmt.Errorf("writing fixture baseline: %w", err) + } + if fixture.RequireFailingBaseline && baseline.Passed { + return summary, fmt.Errorf( + "fixture %q repetition %d already passes its baseline", + fixture.ID, + repetition, + ) + } + + started := time.Now() + result, err := RunExperiment(ctx, Experiment{ + ID: fmt.Sprintf( + "%s-%s-%03d", + suite.ID, + fixture.ID, + repetition, + ), + Repository: workspace, + Output: evidencePath, + Agent: suite.Agent, + Verifications: fixture.Verifications, + AgentTimeout: time.Duration(suite.RunTimeoutSeconds) * time.Second, + Progress: suite.Progress, + }) + duration := time.Since(started) + if err != nil { + return summary, fmt.Errorf( + "running fixture %q repetition %d: %w", + fixture.ID, + repetition, + err, + ) + } + + relativeEvidence, err := filepath.Rel(suite.Output, evidencePath) + if err != nil { + return summary, fmt.Errorf("relativizing evidence path: %w", err) + } + run := SuiteRun{ + FixtureID: fixture.ID, + Repetition: repetition, + Passed: result.Passed, + TimedOut: result.TimedOut, + Duration: duration, + EvidencePath: filepath.ToSlash(relativeEvidence), + } + summary.Runs = append(summary.Runs, run) + summary.TotalRuns++ + if result.Passed { + summary.PassedRuns++ + } + durations = append(durations, duration) + } + } + if summary.TotalRuns > 0 { + summary.PassRate = float64(summary.PassedRuns) / float64(summary.TotalRuns) + summary.MedianDuration = medianDuration(durations) + } + if err := writeJSON(filepath.Join(suite.Output, "summary.json"), summary); err != nil { + return summary, fmt.Errorf("writing suite summary: %w", err) + } + return summary, nil +} + +func runVerification(ctx context.Context, root string, commands []Command) VerificationReport { + report := VerificationReport{Passed: true} + for _, command := range commands { + result := runCommand(ctx, root, command) + report.Commands = append(report.Commands, result) + if result.ExitCode != 0 { + report.Passed = false + } + } + return report +} + +func validateSuite(suite Suite) error { + if !suiteIDPattern.MatchString(suite.ID) { + return errors.New("suite ID must contain lowercase letters, numbers, or hyphens") + } + if suite.Output == "" { + return errors.New("suite output is required") + } + if suite.Repetitions < 1 { + return errors.New("suite repetitions must be positive") + } + if suite.RunTimeoutSeconds < 1 { + return errors.New("suite run timeout must be positive") + } + if suite.Agent.Name == "" || len(suite.Agent.Args) == 0 { + return errors.New("suite agent name and executable are required") + } + if len(suite.Fixtures) == 0 { + return errors.New("suite must contain at least one fixture") + } + seen := make(map[string]struct{}, len(suite.Fixtures)) + for _, fixture := range suite.Fixtures { + if !suiteIDPattern.MatchString(fixture.ID) { + return fmt.Errorf("invalid fixture ID %q", fixture.ID) + } + if _, found := seen[fixture.ID]; found { + return fmt.Errorf("duplicate fixture ID %q", fixture.ID) + } + seen[fixture.ID] = struct{}{} + if fixture.Source == "" { + return fmt.Errorf("fixture %q source is required", fixture.ID) + } + if len(fixture.Verifications) == 0 { + return fmt.Errorf("fixture %q requires deterministic verification", fixture.ID) + } + } + return nil +} + +func copyFixture(source, destination string) error { + sourceInfo, err := os.Stat(source) + if err != nil { + return err + } + if !sourceInfo.IsDir() { + return errors.New("fixture source must be a directory") + } + return filepath.WalkDir(source, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(source, path) + if err != nil { + return err + } + if relative == "." { + return os.Mkdir(destination, 0o700) + } + if relative == ".git" { + return filepath.SkipDir + } + info, err := entry.Info() + if err != nil { + return err + } + target := filepath.Join(destination, relative) + if info.IsDir() { + return os.Mkdir(target, info.Mode().Perm()) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("unsupported fixture file type at %q", relative) + } + input, err := os.Open(path) + if err != nil { + return err + } + output, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()) + if err != nil { + input.Close() + return err + } + _, copyErr := io.Copy(output, input) + inputCloseErr := input.Close() + outputCloseErr := output.Close() + if copyErr != nil { + return copyErr + } + if inputCloseErr != nil { + return inputCloseErr + } + return outputCloseErr + }) +} + +func initializeFixtureRepository(ctx context.Context, root string) error { + commands := [][]string{ + {"init", "--quiet"}, + {"add", "."}, + { + "-c", "user.name=GPTCode Evidence", + "-c", "user.email=evidence@gptcode.dev", + "-c", "core.hooksPath=/dev/null", + "commit", "--quiet", "-m", "fixture: initial state", + }, + } + for _, args := range commands { + if _, err := gitOutput(ctx, root, args...); err != nil { + return err + } + } + return nil +} + +func medianDuration(durations []time.Duration) time.Duration { + ordered := append([]time.Duration(nil), durations...) + sort.Slice(ordered, func(left, right int) bool { + return ordered[left] < ordered[right] + }) + middle := len(ordered) / 2 + if len(ordered)%2 == 1 { + return ordered[middle] + } + return (ordered[middle-1] + ordered[middle]) / 2 +} diff --git a/internal/evidence/suite_test.go b/internal/evidence/suite_test.go new file mode 100644 index 0000000..bce36c6 --- /dev/null +++ b/internal/evidence/suite_test.go @@ -0,0 +1,224 @@ +package evidence + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestRunSuiteExecutesEveryRepetitionAndAggregatesResults(t *testing.T) { + t.Parallel() + + fixture := t.TempDir() + if err := os.WriteFile(filepath.Join(fixture, "value.txt"), []byte("before\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(t.TempDir(), "suite") + + summary, err := RunSuite(context.Background(), Suite{ + ID: "repeatability", + Output: output, + Repetitions: 2, + RunTimeoutSeconds: 5, + Agent: Command{ + Name: "fixture-agent", + Args: []string{"/bin/sh", "-c", "printf 'after\\n' > value.txt"}, + }, + Fixtures: []SuiteFixture{ + { + ID: "text-change", + Source: fixture, + RequireFailingBaseline: true, + Verifications: []Command{ + { + Name: "content", + Args: []string{"/bin/sh", "-c", "test \"$(cat value.txt)\" = after"}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("RunSuite() error = %v", err) + } + if summary.TotalRuns != 2 || summary.PassedRuns != 2 || summary.PassRate != 1 { + t.Fatalf("unexpected summary: %+v", summary) + } + if len(summary.Runs) != 2 || summary.Runs[0].Repetition != 1 || summary.Runs[1].Repetition != 2 { + t.Fatalf("unexpected run records: %+v", summary.Runs) + } + for _, run := range summary.Runs { + if _, err := os.Stat(filepath.Join(output, run.EvidencePath, "manifest.json")); err != nil { + t.Errorf("run evidence missing: %v", err) + } + } + + var persisted SuiteSummary + content, err := os.ReadFile(filepath.Join(output, "summary.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(content, &persisted); err != nil { + t.Fatal(err) + } + if persisted.TotalRuns != summary.TotalRuns || persisted.PassRate != summary.PassRate { + t.Fatalf("persisted summary = %+v, want %+v", persisted, summary) + } +} + +func TestRunSuiteExecutesAndRecordsSetupCommands(t *testing.T) { + t.Parallel() + + fixture := t.TempDir() + if err := os.WriteFile(filepath.Join(fixture, "value.txt"), []byte("before\n"), 0o600); err != nil { + t.Fatal(err) + } + marker := filepath.Join(t.TempDir(), "configured") + output := filepath.Join(t.TempDir(), "suite") + + _, err := RunSuite(context.Background(), Suite{ + ID: "explicit-setup", + Output: output, + Repetitions: 1, + RunTimeoutSeconds: 5, + Setup: []Command{ + {Name: "select-profile", Args: []string{"/usr/bin/touch", marker}}, + }, + Agent: Command{ + Name: "fixture-agent", + Args: []string{"/bin/sh", "-c", "printf 'after\\n' > value.txt"}, + }, + Fixtures: []SuiteFixture{ + { + ID: "text-change", + Source: fixture, + RequireFailingBaseline: true, + Verifications: []Command{ + {Name: "content", Args: []string{"/bin/sh", "-c", "test \"$(cat value.txt)\" = after"}}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("RunSuite() error = %v", err) + } + if _, err := os.Stat(marker); err != nil { + t.Fatalf("setup command did not execute: %v", err) + } + var setup VerificationReport + readJSONFile(t, filepath.Join(output, "setup.json"), &setup) + if !setup.Passed || len(setup.Commands) != 1 || setup.Commands[0].Name != "select-profile" { + t.Fatalf("setup evidence = %+v", setup) + } +} + +func TestRunSuiteRetainsFailedRunsInAggregate(t *testing.T) { + t.Parallel() + + fixture := t.TempDir() + if err := os.WriteFile(filepath.Join(fixture, "value.txt"), []byte("unchanged\n"), 0o600); err != nil { + t.Fatal(err) + } + + summary, err := RunSuite(context.Background(), Suite{ + ID: "failure-is-evidence", + Output: filepath.Join(t.TempDir(), "suite"), + Repetitions: 1, + RunTimeoutSeconds: 5, + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + Fixtures: []SuiteFixture{ + { + ID: "failing-check", + Source: fixture, + RequireFailingBaseline: true, + Verifications: []Command{ + {Name: "failure", Args: []string{"/usr/bin/false"}}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("RunSuite() error = %v", err) + } + if summary.TotalRuns != 1 || summary.PassedRuns != 0 || summary.PassRate != 0 { + t.Fatalf("failed run was not retained: %+v", summary) + } + if len(summary.Runs) != 1 || summary.Runs[0].Passed { + t.Fatalf("failed run record missing: %+v", summary.Runs) + } +} + +func TestRunSuiteRejectsFixtureThatAlreadyPassesRequiredBaseline(t *testing.T) { + t.Parallel() + + fixture := t.TempDir() + if err := os.WriteFile(filepath.Join(fixture, "value.txt"), []byte("already valid\n"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := RunSuite(context.Background(), Suite{ + ID: "green-baseline", + Output: filepath.Join(t.TempDir(), "suite"), + Repetitions: 1, + RunTimeoutSeconds: 5, + Agent: Command{Name: "noop", Args: []string{"/usr/bin/true"}}, + Fixtures: []SuiteFixture{ + { + ID: "already-passing", + Source: fixture, + RequireFailingBaseline: true, + Verifications: []Command{ + {Name: "passes", Args: []string{"/usr/bin/true"}}, + }, + }, + }, + }) + if err == nil { + t.Fatal("RunSuite() error = nil, want green baseline rejection") + } +} + +func TestRunSuiteRecordsAgentTimeoutAndContinuesVerification(t *testing.T) { + t.Parallel() + + fixture := t.TempDir() + if err := os.WriteFile(filepath.Join(fixture, "value.txt"), []byte("unchanged\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(t.TempDir(), "suite") + + summary, err := RunSuite(context.Background(), Suite{ + ID: "bounded-run", + Output: output, + Repetitions: 1, + RunTimeoutSeconds: 1, + Agent: Command{Name: "slow", Args: []string{"/bin/sleep", "5"}}, + Fixtures: []SuiteFixture{ + { + ID: "timeout", + Source: fixture, + Verifications: []Command{ + {Name: "repository-still-valid", Args: []string{"/usr/bin/true"}}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("RunSuite() error = %v", err) + } + if len(summary.Runs) != 1 || !summary.Runs[0].TimedOut || summary.Runs[0].Passed { + t.Fatalf("timeout not represented as failed evidence: %+v", summary) + } + + var verification VerificationReport + readJSONFile( + t, + filepath.Join(output, summary.Runs[0].EvidencePath, "verification.json"), + &verification, + ) + if len(verification.Commands) != 1 || verification.Commands[0].ExitCode != 0 { + t.Fatalf("verification did not run after timeout: %+v", verification) + } +} diff --git a/internal/llm/ollama.go b/internal/llm/ollama.go index 696f430..68219fd 100644 --- a/internal/llm/ollama.go +++ b/internal/llm/ollama.go @@ -6,9 +6,11 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "os" "regexp" + "strconv" "strings" "time" ) @@ -32,6 +34,11 @@ type ollamaReq struct { Messages []ollamaMessage `json:"messages"` Stream bool `json:"stream"` Tools []interface{} `json:"tools,omitempty"` + Options ollamaOptions `json:"options,omitempty"` +} + +type ollamaOptions struct { + NumCtx int `json:"num_ctx,omitempty"` } type ollamaMessage struct { @@ -82,6 +89,7 @@ func (o *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, callba Messages: messages, Stream: true, Tools: req.Tools, + Options: configuredOllamaOptions(), } b, _ := json.Marshal(body) @@ -93,6 +101,10 @@ func (o *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, callba return err } defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + return fmt.Errorf("ollama returned %s: %s", resp.Status, strings.TrimSpace(string(body))) + } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { @@ -145,18 +157,23 @@ func (o *OllamaProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon Messages: messages, Stream: false, Tools: req.Tools, + Options: configuredOllamaOptions(), } b, _ := json.Marshal(body) httpReq, _ := http.NewRequestWithContext(ctx, "POST", o.BaseURL, bytes.NewReader(b)) httpReq.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 120 * time.Second} + client := &http.Client{Timeout: configuredOllamaTimeout()} resp, err := client.Do(httpReq) if err != nil { return nil, err } defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + return nil, fmt.Errorf("ollama returned %s: %s", resp.Status, strings.TrimSpace(string(body))) + } var or ollamaResp if err := json.NewDecoder(resp.Body).Decode(&or); err != nil { @@ -194,6 +211,34 @@ func (o *OllamaProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon return response, nil } +func configuredOllamaOptions() ollamaOptions { + raw := strings.TrimSpace(os.Getenv("GPTCODE_OLLAMA_CONTEXT_LENGTH")) + if raw == "" { + return ollamaOptions{} + } + + numCtx, err := strconv.Atoi(raw) + if err != nil || numCtx <= 0 { + return ollamaOptions{} + } + return ollamaOptions{NumCtx: numCtx} +} + +func configuredOllamaTimeout() time.Duration { + const defaultTimeout = 10 * time.Minute + + raw := strings.TrimSpace(os.Getenv("GPTCODE_OLLAMA_TIMEOUT")) + if raw == "" { + return defaultTimeout + } + + timeout, err := time.ParseDuration(raw) + if err != nil || timeout <= 0 { + return defaultTimeout + } + return timeout +} + func parseXMLToolCalls(text string) []ChatToolCall { var calls []ChatToolCall diff --git a/internal/llm/ollama_test.go b/internal/llm/ollama_test.go new file mode 100644 index 0000000..10a8fee --- /dev/null +++ b/internal/llm/ollama_test.go @@ -0,0 +1,68 @@ +package llm + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestOllamaChatReturnsServerErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"model is required"}`)) + })) + defer server.Close() + + provider := NewOllama(server.URL) + _, err := provider.Chat(context.Background(), ChatRequest{}) + if err == nil || !strings.Contains(err.Error(), "model is required") { + t.Fatalf("expected Ollama server error, got %v", err) + } +} + +func TestOllamaChatUsesConfiguredContextWindow(t *testing.T) { + t.Setenv("GPTCODE_OLLAMA_CONTEXT_LENGTH", "32768") + + var request struct { + Options struct { + NumCtx int `json:"num_ctx"` + } `json:"options"` + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode request: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"message":{"content":"ok"}}`)) + })) + defer server.Close() + + provider := NewOllama(server.URL) + if _, err := provider.Chat(context.Background(), ChatRequest{Model: "local"}); err != nil { + t.Fatalf("Chat() error = %v", err) + } + if request.Options.NumCtx != 32768 { + t.Fatalf("num_ctx = %d, want 32768", request.Options.NumCtx) + } +} + +func TestOllamaChatUsesConfiguredTimeout(t *testing.T) { + t.Setenv("GPTCODE_OLLAMA_TIMEOUT", "10ms") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(100 * time.Millisecond) + _, _ = w.Write([]byte(`{"message":{"content":"late"}}`)) + })) + defer server.Close() + + provider := NewOllama(server.URL) + _, err := provider.Chat(context.Background(), ChatRequest{Model: "local"}) + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") { + t.Fatalf("expected configured timeout, got %v", err) + } +} diff --git a/internal/maestro/conductor.go b/internal/maestro/conductor.go index 7ac660c..bf1b760 100644 --- a/internal/maestro/conductor.go +++ b/internal/maestro/conductor.go @@ -236,7 +236,7 @@ func (c *Conductor) ReportCompleteWithPR(success bool, summary string, prURL str } // ExecuteTask orchestrates the execution of a task -func (c *Conductor) ExecuteTask(ctx context.Context, task string, complexity string) error { +func (c *Conductor) ExecuteTask(ctx context.Context, task string, complexity string) (taskErr error) { if os.Getenv("GPTCODE_DEBUG") == "1" { fmt.Fprintf(os.Stderr, "[MAESTRO] ExecuteTask called: task=%s complexity=%s lang=%s\n", task, complexity, c.language) } @@ -254,7 +254,7 @@ func (c *Conductor) ExecuteTask(ctx context.Context, task string, complexity str sessionID := uuid.New().String() if c.Tracer != nil { _ = c.Tracer.Begin(sessionID, task) - defer func() { _ = c.Tracer.End(true) }() // End with success status (will be updated on error) + defer func() { _ = c.Tracer.End(taskErr == nil) }() } // Select model for planning @@ -282,11 +282,15 @@ func (c *Conductor) ExecuteTask(ctx context.Context, task string, complexity str // Create planner with selected model planProvider := c.createProvider(planBackend) planner := agents.NewPlanner(planProvider, planModel) + repositoryContext, err := buildPlanningRepositoryContext(c.cwd, c.language, 32*1024) + if err != nil { + return fmt.Errorf("build planning repository context: %w", err) + } fmt.Println("Creating plan...") c.ReportProgress("planning", "Creating plan") start := time.Now() - plan, err := planner.CreatePlan(ctx, task, "", nil) + plan, err := planner.CreatePlan(ctx, task, repositoryContext, nil) elapsed := time.Since(start) c.ReportProgress("planning", "Plan created") c.selector.RecordUsage(planBackend, planModel, err == nil, errorMsg(err)) @@ -304,10 +308,6 @@ func (c *Conductor) ExecuteTask(ctx context.Context, task string, complexity str } // Build conversation history - repositoryContext, err := buildEditorRepositoryContext(c.cwd, c.language, 32*1024) - if err != nil { - return fmt.Errorf("build editor repository context: %w", err) - } history := []llm.ChatMessage{ { Role: "user", @@ -339,6 +339,7 @@ Do not add, remove, or rename exported symbols when API stability is required.`, } consecutiveErrors := 0 + verificationProgress := newVerificationProgress(3) for { // Check if we should continue (intent-aware limits + loop detection) @@ -429,9 +430,13 @@ Do not add, remove, or rename exported symbols when API stability is required.`, _ = c.Tracer.RecordDecision("RecoverySystem", decision) } + retryMessage, retryErr := buildRetryMessage(c.cwd, c.language, advancedPrompt) + if retryErr != nil { + return fmt.Errorf("build execution retry context: %w", retryErr) + } history = append(history, llm.ChatMessage{ Role: "user", - Content: advancedPrompt, + Content: retryMessage, }) continue } @@ -447,6 +452,26 @@ Do not add, remove, or rename exported symbols when API stability is required.`, _ = c.Tracer.RecordMetrics("EditorAgent", metrics) } + if strings.EqualFold(c.language, "go") && len(modifiedFiles) > 0 { + formatOutput, formatErr := formatModifiedGoFiles(ctx, c.cwd, modifiedFiles) + if formatErr != nil { + feedback := fmt.Sprintf( + "Deterministic Go formatting failed. Fix the current implementation.\nOutput:\n%s\nError: %v", + formatOutput, + formatErr, + ) + retryMessage, retryErr := buildRetryMessage(c.cwd, c.language, feedback) + if retryErr != nil { + return fmt.Errorf("build formatter retry context: %w", retryErr) + } + history = append(history, llm.ChatMessage{ + Role: "user", + Content: retryMessage, + }) + continue + } + } + // Check if this is a query-only task (no validation needed) if c.isQueryTask(task, plan, modifiedFiles) { c.recordFeedback(editBackend, editModel, "editor", task, true, "") @@ -458,13 +483,10 @@ Do not add, remove, or rename exported symbols when API stability is required.`, // Print detailed execution summary if c.Observer != nil { + c.Observer.SetOutcome(true) c.Observer.PrintSummary() } - // Record success metrics - if c.Tracer != nil { - _ = c.Tracer.End(true) - } return nil } @@ -476,10 +498,17 @@ Do not add, remove, or rename exported symbols when API stability is required.`, if changes := publicAPIChanges(publicAPIBaseline, currentPublicAPI); len(changes) > 0 { issues := strings.Join(changes, "\n") fmt.Printf("[WARNING] Public API contract changed:\n%s\n", issues) + retryMessage, retryErr := buildRetryMessage( + c.cwd, + c.language, + "Deterministic public API validation failed. Restore the original exported API while preserving the requested internal fix:\n"+issues, + ) + if retryErr != nil { + return fmt.Errorf("build public API retry context: %w", retryErr) + } history = append(history, llm.ChatMessage{ - Role: "user", - Content: "Deterministic public API validation failed. Restore the original exported API while preserving the requested internal fix:\n" + - issues, + Role: "user", + Content: retryMessage, }) continue } @@ -497,10 +526,32 @@ Do not add, remove, or rename exported symbols when API stability is required.`, verificationOutput, verificationErr := runRequestedVerification(ctx, c.cwd, verificationCommand) if verificationErr != nil { fmt.Printf("[WARNING] Deterministic verification failed:\n%s\n", verificationOutput) - history = append(history, llm.ChatMessage{ - Role: "user", - Content: fmt.Sprintf("Deterministic verification failed. Fix the implementation and rerun the required check.\nCommand: %s\nOutput:\n%s", + if verificationProgress.Observe(verificationOutput) { + c.recordFeedback( + editBackend, + editModel, + "editor", + task, + false, + "verification plateau: "+verificationOutput, + ) + return fmt.Errorf( + "verification plateau after %d equivalent failures; stopping without spending the remaining attempt budget", + verificationProgress.Consecutive(), + ) + } + retryMessage, retryErr := buildRetryMessage( + c.cwd, + c.language, + fmt.Sprintf("Deterministic verification failed. Fix the implementation and rerun the required check.\nCommand: %s\nOutput:\n%s", strings.Join(verificationCommand, " "), verificationOutput), + ) + if retryErr != nil { + return fmt.Errorf("build verification retry context: %w", retryErr) + } + history = append(history, llm.ChatMessage{ + Role: "user", + Content: retryMessage, }) continue } @@ -580,9 +631,13 @@ Output: _ = c.Tracer.RecordDecision("RecoverySystem", decision) } + retryMessage, retryErr := buildRetryMessage(c.cwd, c.language, advancedPrompt) + if retryErr != nil { + return fmt.Errorf("build validation retry context: %w", retryErr) + } history = append(history, llm.ChatMessage{ Role: "user", - Content: advancedPrompt, + Content: retryMessage, }) continue } @@ -621,9 +676,13 @@ Output: _ = c.Tracer.RecordDecision("RecoverySystem", decision) } + retryMessage, retryErr := buildRetryMessage(c.cwd, c.language, advancedPrompt) + if retryErr != nil { + return fmt.Errorf("build review retry context: %w", retryErr) + } history = append(history, llm.ChatMessage{ Role: "user", - Content: advancedPrompt, + Content: retryMessage, }) continue } @@ -648,13 +707,10 @@ Output: // Print detailed execution summary if c.Observer != nil { + c.Observer.SetOutcome(true) c.Observer.PrintSummary() } - // Record success metrics - if c.Tracer != nil { - _ = c.Tracer.End(true) - } return nil } diff --git a/internal/maestro/formatter.go b/internal/maestro/formatter.go new file mode 100644 index 0000000..ef52ecf --- /dev/null +++ b/internal/maestro/formatter.go @@ -0,0 +1,63 @@ +package maestro + +import ( + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strings" +) + +func formatModifiedGoFiles(ctx context.Context, root string, modifiedFiles []string) (string, error) { + absoluteRoot, err := filepath.Abs(root) + if err != nil { + return "", fmt.Errorf("resolving repository root: %w", err) + } + absoluteRoot, err = filepath.EvalSymlinks(absoluteRoot) + if err != nil { + return "", fmt.Errorf("resolving repository root symlinks: %w", err) + } + var paths []string + seen := make(map[string]struct{}, len(modifiedFiles)) + for _, modified := range modifiedFiles { + path := modified + if !filepath.IsAbs(path) { + path = filepath.Join(absoluteRoot, path) + } + path, err = filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolving modified path: %w", err) + } + path, err = filepath.EvalSymlinks(path) + if err != nil { + return "", fmt.Errorf("resolving modified path symlinks: %w", err) + } + relative, err := filepath.Rel(absoluteRoot, path) + if err != nil { + return "", fmt.Errorf("relativizing modified path: %w", err) + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("modified file is outside repository root") + } + if filepath.Ext(relative) != ".go" { + continue + } + if _, found := seen[relative]; found { + continue + } + seen[relative] = struct{}{} + paths = append(paths, relative) + } + if len(paths) == 0 { + return "", nil + } + + command := exec.CommandContext(ctx, "gofmt", append([]string{"-w"}, paths...)...) + command.Dir = absoluteRoot + output, err := command.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("gofmt modified files: %w", err) + } + return string(output), nil +} diff --git a/internal/maestro/formatter_test.go b/internal/maestro/formatter_test.go new file mode 100644 index 0000000..1fb49ea --- /dev/null +++ b/internal/maestro/formatter_test.go @@ -0,0 +1,67 @@ +package maestro + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestFormatModifiedGoFilesFormatsOnlyRepositoryFiles(t *testing.T) { + t.Parallel() + + root := t.TempDir() + path := filepath.Join(root, "store.go") + if err := os.WriteFile(path, []byte("package store\n\nfunc Value( )int{return 1}\n"), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := formatModifiedGoFiles(context.Background(), root, []string{"store.go"}); err != nil { + t.Fatalf("formatModifiedGoFiles() error = %v", err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(content) != "package store\n\nfunc Value() int { return 1 }\n" { + t.Fatalf("formatted content = %q", content) + } +} + +func TestFormatModifiedGoFilesRejectsPathOutsideRepository(t *testing.T) { + t.Parallel() + + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.go") + if err := os.WriteFile(outside, []byte("package outside\n"), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := formatModifiedGoFiles(context.Background(), root, []string{outside}); err == nil { + t.Fatal("formatModifiedGoFiles() error = nil, want outside path rejection") + } +} + +func TestFormatModifiedGoFilesRejectsSymlinkOutsideRepository(t *testing.T) { + t.Parallel() + + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.go") + if err := os.WriteFile(outside, []byte("package outside\n\nfunc Value( )int{return 1}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "linked.go")); err != nil { + t.Fatal(err) + } + + if _, err := formatModifiedGoFiles(context.Background(), root, []string{"linked.go"}); err == nil { + t.Fatal("formatModifiedGoFiles() error = nil, want symlink escape rejection") + } + content, err := os.ReadFile(outside) + if err != nil { + t.Fatal(err) + } + if string(content) != "package outside\n\nfunc Value( )int{return 1}\n" { + t.Fatal("formatter changed file outside repository") + } +} diff --git a/internal/maestro/repository_context.go b/internal/maestro/repository_context.go index 7d488e5..a6d3c24 100644 --- a/internal/maestro/repository_context.go +++ b/internal/maestro/repository_context.go @@ -58,6 +58,72 @@ func buildEditorRepositoryContext(root, language string, maxBytes int) (string, return context.String(), err } +func buildPlanningRepositoryContext(root, language string, maxBytes int) (string, error) { + return buildRecoveryRepositoryContext(root, language, maxBytes) +} + +func buildRetryMessage(root, language, feedback string) (string, error) { + current, err := buildRecoveryRepositoryContext(root, language, 64*1024) + if err != nil { + return "", err + } + return fmt.Sprintf( + "%s\n\n## CURRENT REPOSITORY STATE AFTER THE PREVIOUS ATTEMPT\n%s\n"+ + "Base the correction on this current state, not the original snapshot.", + feedback, + current, + ), nil +} + +// buildRecoveryRepositoryContext includes the contract that failed as well as +// the implementation being repaired. Planning context intentionally omits test +// files to conserve tokens, but retrying without tests leaves the editor unable +// to reason about assertions that are only summarized by test output. +func buildRecoveryRepositoryContext(root, language string, maxBytes int) (string, error) { + if maxBytes <= 0 { + return "", nil + } + + var context strings.Builder + used := 0 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + switch entry.Name() { + case ".git", "vendor", "node_modules": + if path != root { + return filepath.SkipDir + } + } + return nil + } + + name := entry.Name() + if !isEditorSourceFile(path, language) && + name != "task.md" && + name != "AGENTS.md" { + return nil + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + if used+len(content) > maxBytes { + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + fmt.Fprintf(&context, "\n\n%s\n\n", filepath.ToSlash(relative), content) + used += len(content) + return nil + }) + return context.String(), err +} + func isEditorSourceFile(path, language string) bool { extension := strings.ToLower(filepath.Ext(path)) switch strings.ToLower(language) { diff --git a/internal/maestro/repository_context_test.go b/internal/maestro/repository_context_test.go index 81c243e..f46fbf7 100644 --- a/internal/maestro/repository_context_test.go +++ b/internal/maestro/repository_context_test.go @@ -27,3 +27,52 @@ func TestBuildEditorRepositoryContextIncludesImplementation(t *testing.T) { t.Fatalf("test file should not consume editor context: %s", context) } } + +func TestBuildPlanningRepositoryContextGroundsPlanInImplementation(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "ledger.go"), []byte("package ledger\nfunc Transfer() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "ledger_test.go"), []byte("package ledger\nfunc TestTransferPreservesTotal() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "task.md"), []byte("Keep the public API stable.\n"), 0o644); err != nil { + t.Fatal(err) + } + + context, err := buildPlanningRepositoryContext(root, "Go", 32*1024) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(context, `path="ledger.go"`) || + !strings.Contains(context, "func Transfer") || + !strings.Contains(context, "TestTransferPreservesTotal") || + !strings.Contains(context, "Keep the public API stable") { + t.Fatalf("planning evidence missing: %s", context) + } +} + +func TestBuildRetryMessageIncludesCurrentRepositoryState(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "ledger.go"), []byte("package ledger\nvar attempt = 2\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "ledger_test.go"), []byte("package ledger\nfunc TestTransferContract() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "task.md"), []byte("Preserve the exported API and pass the race detector.\n"), 0o644); err != nil { + t.Fatal(err) + } + + message, err := buildRetryMessage(root, "Go", "go test failed") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(message, "go test failed") || + !strings.Contains(message, "CURRENT REPOSITORY STATE") || + !strings.Contains(message, "var attempt = 2") || + !strings.Contains(message, "TestTransferContract") || + !strings.Contains(message, "Preserve the exported API") { + t.Fatalf("retry message lacks current evidence: %s", message) + } +} diff --git a/internal/maestro/verification_progress.go b/internal/maestro/verification_progress.go new file mode 100644 index 0000000..3e15044 --- /dev/null +++ b/internal/maestro/verification_progress.go @@ -0,0 +1,55 @@ +package maestro + +import ( + "regexp" + "strings" +) + +var ( + verificationDuration = regexp.MustCompile(`\d+(?:\.\d+)?s\b`) + goTestTemporaryPath = regexp.MustCompile( + `(?:/private)?/var/folders/[^[:space:]]+/T/Test[^/[:space:]]+[0-9]+/[0-9]+|/tmp/Test[^/[:space:]]+[0-9]+/[0-9]+`, + ) +) + +type verificationProgress struct { + threshold int + lastFailure string + consecutive int +} + +func newVerificationProgress(threshold int) *verificationProgress { + if threshold < 1 { + threshold = 1 + } + return &verificationProgress{threshold: threshold} +} + +func (p *verificationProgress) Observe(output string) bool { + failure := normalizeVerificationFailure(output) + if failure == p.lastFailure && failure != "" { + p.consecutive++ + } else { + p.lastFailure = failure + p.consecutive = 1 + } + return failure != "" && p.consecutive >= p.threshold +} + +func (p *verificationProgress) Consecutive() int { + return p.consecutive +} + +func normalizeVerificationFailure(output string) string { + output = verificationDuration.ReplaceAllString(output, "(duration)") + output = goTestTemporaryPath.ReplaceAllString(output, "(go-test-temp)") + lines := strings.Split(output, "\n") + normalized := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.Join(strings.Fields(line), " ") + if line != "" { + normalized = append(normalized, line) + } + } + return strings.Join(normalized, "\n") +} diff --git a/internal/maestro/verification_progress_test.go b/internal/maestro/verification_progress_test.go new file mode 100644 index 0000000..5140831 --- /dev/null +++ b/internal/maestro/verification_progress_test.go @@ -0,0 +1,57 @@ +package maestro + +import "testing" + +func TestVerificationProgressStopsAfterThreeEquivalentFailures(t *testing.T) { + tracker := newVerificationProgress(3) + outputs := []string{ + "--- FAIL: TestWriteRejectsSymlinkedParent (0.00s)\nFAIL example.com/safestore 0.295s\n", + "--- FAIL: TestWriteRejectsSymlinkedParent (0.01s)\nFAIL example.com/safestore 0.420s\n", + "--- FAIL: TestWriteRejectsSymlinkedParent (0.02s)\nFAIL example.com/safestore 0.362s\n", + } + + for index, output := range outputs { + plateau := tracker.Observe(output) + if index < 2 && plateau { + t.Fatalf("reported plateau after only %d observations", index+1) + } + if index == 2 && !plateau { + t.Fatal("expected three equivalent failures to report a plateau") + } + } +} + +func TestVerificationProgressResetsWhenFailureChanges(t *testing.T) { + tracker := newVerificationProgress(3) + if tracker.Observe("undefined: unsafe") { + t.Fatal("first failure cannot be a plateau") + } + if tracker.Observe("undefined: unsafe") { + t.Fatal("second failure cannot be a plateau") + } + if tracker.Observe("TestSelfTransferCompletes: self-transfer deadlocked") { + t.Fatal("a changed failure is progress, not a plateau") + } + if tracker.Consecutive() != 1 { + t.Fatalf("consecutive failures = %d, want reset to 1", tracker.Consecutive()) + } +} + +func TestVerificationProgressIgnoresGoTemporaryDirectoryIDs(t *testing.T) { + tracker := newVerificationProgress(3) + outputs := []string{ + "Write() error = lstat /private/var/folders/aa/one/T/TestWriteStoresNestedFileInsideRoot1722907079/001/reports: no such file\n", + "Write() error = lstat /private/var/folders/bb/two/T/TestWriteStoresNestedFileInsideRoot3193033618/001/reports: no such file\n", + "Write() error = lstat /private/var/folders/cc/three/T/TestWriteStoresNestedFileInsideRoot1568810318/001/reports: no such file\n", + } + + for index, output := range outputs { + plateau := tracker.Observe(output) + if index < 2 && plateau { + t.Fatalf("reported plateau after only %d observations", index+1) + } + if index == 2 && !plateau { + t.Fatal("temporary directory IDs must not disguise an equivalent failure") + } + } +} diff --git a/internal/modes/model_selection.go b/internal/modes/model_selection.go new file mode 100644 index 0000000..7b2d2ae --- /dev/null +++ b/internal/modes/model_selection.go @@ -0,0 +1,7 @@ +package modes + +import "github.com/jadercorrea/gptcode/internal/config" + +func configuredAgentModel(backend config.BackendConfig, profile, agent string) string { + return backend.GetModelForAgentWithProfile(agent, profile) +} diff --git a/internal/modes/model_selection_test.go b/internal/modes/model_selection_test.go new file mode 100644 index 0000000..c6560bd --- /dev/null +++ b/internal/modes/model_selection_test.go @@ -0,0 +1,28 @@ +package modes + +import ( + "testing" + + "github.com/jadercorrea/gptcode/internal/config" +) + +func TestConfiguredAgentModelUsesActiveBackendProfile(t *testing.T) { + backend := config.BackendConfig{ + DefaultModel: "fallback", + Profiles: map[string]config.ProfileConfig{ + "evaluation": { + AgentModels: config.AgentModels{ + Query: "qwen-query", + Research: "qwen-research", + }, + }, + }, + } + + if got := configuredAgentModel(backend, "evaluation", "query"); got != "qwen-query" { + t.Fatalf("query model = %q, want profile model", got) + } + if got := configuredAgentModel(backend, "evaluation", "research"); got != "qwen-research" { + t.Fatalf("research model = %q, want profile model", got) + } +} diff --git a/internal/modes/research.go b/internal/modes/research.go index 59db520..7e635da 100644 --- a/internal/modes/research.go +++ b/internal/modes/research.go @@ -19,6 +19,8 @@ import ( "golang.org/x/term" ) +const localModelTimeout = 2 * time.Minute + func RunResearch(args []string) error { question := "" if len(args) > 0 { @@ -78,7 +80,7 @@ func RunResearch(args []string) error { customExec = llm.NewChatCompletion(backendCfg.BaseURL, backendName) } - queryModel := backendCfg.GetModelForAgent("query") + queryModel := configuredAgentModel(backendCfg, setup.Defaults.Profile, "research") queryAgent := agents.NewQuery(customExec, cwd, queryModel) evidence, err := collectRepositoryEvidence(cwd, question) @@ -86,7 +88,7 @@ func RunResearch(args []string) error { return fmt.Errorf("collect repository evidence: %w", err) } codebasePrompt := buildCodebaseResearchPrompt(question, evidence) - researchCtx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + researchCtx, cancel := context.WithTimeout(context.Background(), localModelTimeout) defer cancel() codebaseAnalysis, err := queryAgent.Execute(researchCtx, []llm.ChatMessage{{Role: "user", Content: codebasePrompt}}, nil) @@ -110,6 +112,41 @@ override the implementation evidence.` } } + for correctionAttempt := 0; correctionAttempt < 2; correctionAttempt++ { + issues := unsupportedGoLockClaims(codebaseAnalysis, evidence) + if ciIssue := unsupportedCIClaim(codebaseAnalysis, evidence); ciIssue != "" { + issues = append(issues, ciIssue) + } + if len(issues) == 0 { + break + } + correctionCtx, correctionCancel := context.WithTimeout(context.Background(), localModelTimeout) + corrected, correctionErr := queryAgent.Execute(correctionCtx, []llm.ChatMessage{{ + Role: "user", + Content: buildCodebaseResearchPrompt(question, evidence) + fmt.Sprintf(` + +Mandatory correction: +The draft made these claims that contradict the inspected repository evidence: +- %s + +Rewrite the answer from scratch. Describe the exact lock acquired by each +method separately. Return only Findings, Evidence, and Verification; do not +include the previous draft or a corrections appendix.`, strings.Join(issues, "\n- ")), + }}, nil) + correctionCancel() + if correctionErr != nil { + return fmt.Errorf("correct unsupported lock claims: %w", correctionErr) + } + codebaseAnalysis = corrected + } + remainingIssues := unsupportedGoLockClaims(codebaseAnalysis, evidence) + if ciIssue := unsupportedCIClaim(codebaseAnalysis, evidence); ciIssue != "" { + remainingIssues = append(remainingIssues, ciIssue) + } + if len(remainingIssues) > 0 { + return fmt.Errorf("codebase analysis contradicted deterministic evidence: %s", strings.Join(remainingIssues, "; ")) + } + home, _ := os.UserHomeDir() researchDir := filepath.Join(home, ".gptcode", "research") _ = os.MkdirAll(researchDir, 0755) @@ -181,6 +218,61 @@ func contradictsUnsafeConcurrencyEvidence(analysis string, evidence repositoryEv return claimsSafe && !acknowledgesUnsafe } +func unsupportedGoLockClaims(analysis string, evidence repositoryEvidence) []string { + if !strings.EqualFold(evidence.Language, "Go") { + return nil + } + + methodUsesReadLock := make(map[string]bool) + methodPattern := regexp.MustCompile(`(?s)func\s+\([^)]*\)\s+([A-Za-z][A-Za-z0-9_]*)\([^)]*\)[^{]*\{(.*?)\n\}`) + for _, content := range evidence.Contents { + for _, match := range methodPattern.FindAllStringSubmatch(content, -1) { + if len(match) == 3 { + methodUsesReadLock[match[1]] = strings.Contains(match[2], ".RLock()") + } + } + } + + var issues []string + segments := strings.FieldsFunc(analysis, func(r rune) bool { + return r == '\n' || r == ';' + }) + for _, segment := range segments { + lower := strings.ToLower(segment) + if !strings.Contains(lower, "read lock") && !strings.Contains(lower, "rlock") { + continue + } + for method, usesReadLock := range methodUsesReadLock { + if usesReadLock { + continue + } + methodPattern := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(method) + `\b`) + if methodPattern.MatchString(segment) { + issues = append(issues, fmt.Sprintf("%s does not acquire RLock, but the draft says: %s", method, strings.TrimSpace(segment))) + } + } + } + sort.Strings(issues) + return issues +} + +func unsupportedCIClaim(analysis string, evidence repositoryEvidence) string { + mentionsCI := regexp.MustCompile(`(?i)\bCI(?:/CD)?\b`).MatchString(analysis) + if !mentionsCI { + return "" + } + for _, path := range evidence.Files { + lower := strings.ToLower(path) + if strings.HasPrefix(lower, ".github/workflows/") || + strings.Contains(lower, "gitlab-ci") || + strings.Contains(lower, "circleci") || + strings.Contains(lower, "buildkite") { + return "" + } + } + return "the draft claims CI execution, but no CI configuration exists in the inspected evidence" +} + type repositoryEvidence struct { Language string Files []string @@ -335,7 +427,7 @@ func buildCodebaseResearchPrompt(question string, evidence repositoryEvidence) s if !ok { continue } - fmt.Fprintf(&groundedFiles, "\n\n%s\n\n", path, content) + fmt.Fprintf(&groundedFiles, "\n\n%s\n\n", path, numberLines(content)) } return fmt.Sprintf(`Research this repository question: @@ -360,6 +452,8 @@ Grounding requirements: 8. Compare stated expectations with implementation details and report contradictions explicitly. 9. Concurrency claims require synchronization in the implementation (for example locks, atomics, channels, or documented confinement). A WaitGroup or goroutines in a test exercise concurrency; they do not make shared state safe. 10. If mutable shared state has no visible synchronization, state that concurrent access is unsafe and recommend running the repository's race/concurrency verification. +11. Make the answer internally consistent. Describe the lock actually acquired by each method; do not classify a method as a read operation if its implementation takes a write lock or mutates state. +12. Do not claim a command passed unless its exit status and output are included in the evidence. Distinguish commands required by the repository from commands actually executed. Return concise sections: Findings, Evidence, Verification.`, question, evidence.Language, strings.Join(evidence.Files, "\n"), groundedFiles.String()) } diff --git a/internal/modes/research_test.go b/internal/modes/research_test.go index a0bfc1c..1477445 100644 --- a/internal/modes/research_test.go +++ b/internal/modes/research_test.go @@ -71,6 +71,7 @@ func TestBuildCodebaseResearchPromptRequiresGroundedEvidence(t *testing.T) { "session/store.go", "session/store_test.go", "func (s *Store) Active", + `1 | func (s *Store) Active`, "read_file", "exact file paths", "Do not speculate", @@ -80,6 +81,8 @@ func TestBuildCodebaseResearchPromptRequiresGroundedEvidence(t *testing.T) { "report contradictions", "Concurrency claims require synchronization in the implementation", "WaitGroup or goroutines in a test exercise concurrency", + "internally consistent", + "Do not claim a command passed", } { if !strings.Contains(prompt, required) { t.Fatalf("research prompt missing %q:\n%s", required, prompt) @@ -131,6 +134,52 @@ func TestResearchDetectsUnsupportedConcurrencySafetyClaim(t *testing.T) { } } +func TestResearchDetectsUnsupportedGoMethodLockClaim(t *testing.T) { + evidence := repositoryEvidence{ + Language: "Go", + Contents: map[string]string{ + "cache.go": `package cache +func (c *Cache) Get() { + c.mu.RLock() + defer c.mu.RUnlock() +} +func (c *Cache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return 0 +}`, + }, + } + + issues := unsupportedGoLockClaims( + "Read operations (`Get`, `Len`) acquire read locks (`RLock`).", + evidence, + ) + if len(issues) != 1 || !strings.Contains(issues[0], "Len") { + t.Fatalf("expected the false Len lock claim to be rejected, got %v", issues) + } + if issues := unsupportedGoLockClaims( + "`Get` starts with RLock; `Len` takes the exclusive Lock because it deletes expired entries.", + evidence, + ); len(issues) != 0 { + t.Fatalf("expected exact per-method lock claims to pass, got %v", issues) + } +} + +func TestResearchRejectsCIClaimsWithoutCIEvidence(t *testing.T) { + evidence := repositoryEvidence{ + Files: []string{"cache.go", "cache_test.go", "task.md"}, + } + if issue := unsupportedCIClaim("These commands are executed by the repository's CI system.", evidence); issue == "" { + t.Fatal("expected unsupported CI claim to be rejected") + } + + evidence.Files = append(evidence.Files, ".github/workflows/verify.yml") + if issue := unsupportedCIClaim("These commands are executed by CI.", evidence); issue != "" { + t.Fatalf("expected CI claim with workflow evidence to pass, got %q", issue) + } +} + func TestCollectRepositoryEvidenceStaysInsidePublicRepositoryContext(t *testing.T) { repository := t.TempDir() outside := t.TempDir() diff --git a/internal/modes/review.go b/internal/modes/review.go index 4d76d81..1814c3c 100644 --- a/internal/modes/review.go +++ b/internal/modes/review.go @@ -5,8 +5,8 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" - "time" "github.com/jadercorrea/gptcode/internal/agents" "github.com/jadercorrea/gptcode/internal/config" @@ -27,7 +27,7 @@ func RunReview(opts ReviewOptions) error { backendName := setup.Defaults.Backend backendCfg := setup.Backend[backendName] - model := backendCfg.GetModelForAgent("query") + model := configuredAgentModel(backendCfg, setup.Defaults.Profile, "query") if model == "" { model = backendCfg.DefaultModel } @@ -60,14 +60,20 @@ func RunReview(opts ReviewOptions) error { } var fileEvidence string + var relatedEvidence string if !info.IsDir() { content, err := os.ReadFile(targetPath) if err != nil { return fmt.Errorf("read review target: %w", err) } fileEvidence = string(content) + evidence, evidenceErr := collectRepositoryEvidence(cwd, target+" "+opts.Focus) + if evidenceErr != nil { + return fmt.Errorf("collect review evidence: %w", evidenceErr) + } + relatedEvidence = formatRelatedReviewEvidence(cwd, targetPath, evidence) } - reviewPrompt := buildReviewPrompt(targetPath, info.IsDir(), opts.Focus, fileEvidence) + reviewPrompt := buildReviewPrompt(targetPath, info.IsDir(), opts.Focus, fileEvidence, relatedEvidence) fmt.Printf("Reviewing: %s\n", target) if opts.Focus != "" { @@ -86,7 +92,7 @@ func RunReview(opts ReviewOptions) error { }, } - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), localModelTimeout) defer cancel() var result string if fileEvidence != "" { @@ -121,7 +127,7 @@ func RunReview(opts ReviewOptions) error { return nil } -func buildReviewPrompt(targetPath string, isDir bool, focus string, fileEvidence string) string { +func buildReviewPrompt(targetPath string, isDir bool, focus, fileEvidence, relatedEvidence string) string { var prompt strings.Builder if isDir { @@ -133,6 +139,11 @@ func buildReviewPrompt(targetPath string, isDir bool, focus string, fileEvidence prompt.WriteString("```text\n") prompt.WriteString(numberLines(fileEvidence)) prompt.WriteString("\n```\n") + if relatedEvidence != "" { + prompt.WriteString("\nRelated repository contracts and tests:\n") + prompt.WriteString(relatedEvidence) + prompt.WriteString("\n") + } } if focus != "" { @@ -150,10 +161,40 @@ func buildReviewPrompt(targetPath string, isDir bool, focus string, fileEvidence prompt.WriteString("Do not invent lifecycle requirements, background work, or new public API methods. When the requested focus is satisfied and no defect is evidenced, say so explicitly.\n") prompt.WriteString("Treat an evidenced violation of the requested focus as a defect even if no broader product requirements were supplied.\n") prompt.WriteString("Private implementation changes do not alter the public API; do not claim that adding private synchronization changes exported constructors or methods.\n") + prompt.WriteString("Reserve Critical Issues for a demonstrated failing input, unsafe interleaving, security boundary violation, or broken contract; place simplifications and micro-optimizations under Suggestions.\n") + prompt.WriteString("Go visibility is determined by capitalization; do not recommend underscore prefixes for private identifiers.\n") + prompt.WriteString("When reasoning about concurrency, an exclusive lock prevents concurrent map mutation for its duration. Do not invent a race or deadlock without showing a concrete interleaving from the supplied code.\n") return prompt.String() } +func formatRelatedReviewEvidence(cwd, targetPath string, evidence repositoryEvidence) string { + targetRelative, err := filepath.Rel(cwd, targetPath) + if err != nil { + return "" + } + targetRelative = filepath.ToSlash(targetRelative) + + paths := make([]string, 0, len(evidence.Contents)) + for path := range evidence.Contents { + if path != targetRelative { + paths = append(paths, path) + } + } + sort.Strings(paths) + + var related strings.Builder + for _, path := range paths { + fmt.Fprintf( + &related, + "\n\n%s\n\n", + path, + numberLines(evidence.Contents[path]), + ) + } + return related.String() +} + func numberLines(content string) string { lines := strings.Split(strings.TrimSuffix(content, "\n"), "\n") var numbered strings.Builder diff --git a/internal/modes/review_test.go b/internal/modes/review_test.go index 007a70a..bcc43e4 100644 --- a/internal/modes/review_test.go +++ b/internal/modes/review_test.go @@ -1,6 +1,7 @@ package modes import ( + "path/filepath" "strings" "testing" ) @@ -11,6 +12,7 @@ func TestBuildReviewPromptIncludesFileEvidenceAndConstraints(t *testing.T) { false, "concurrency correctness and public API stability", "package session\n\ntype Store struct { sessions map[string]Session }\n", + "\n1 | Preserve the public API.\n", ) for _, required := range []string{ @@ -24,9 +26,40 @@ func TestBuildReviewPromptIncludesFileEvidenceAndConstraints(t *testing.T) { "Treat an evidenced violation of the requested focus as a defect", "Private implementation changes do not alter the public API", "1 | package session", + `path="task.md"`, + "Preserve the public API", + "Reserve Critical Issues", + "Go visibility is determined by capitalization", + "exclusive lock prevents concurrent map mutation", } { if !strings.Contains(prompt, required) { t.Fatalf("review prompt missing %q:\n%s", required, prompt) } } } + +func TestFormatRelatedReviewEvidenceExcludesTargetAndNumbersContracts(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "cache.go") + formatted := formatRelatedReviewEvidence(root, target, repositoryEvidence{ + Contents: map[string]string{ + "cache.go": "package cache\n", + "cache_test.go": "package cache\nfunc TestExpiry() {}\n", + "task.md": "Run go test -race ./...\n", + }, + }) + + if strings.Contains(formatted, `path="cache.go"`) { + t.Fatalf("target was duplicated in related evidence: %s", formatted) + } + for _, required := range []string{ + `path="cache_test.go"`, + "2 | func TestExpiry", + `path="task.md"`, + "1 | Run go test -race ./...", + } { + if !strings.Contains(formatted, required) { + t.Fatalf("related evidence missing %q: %s", required, formatted) + } + } +} diff --git a/internal/observability/observer.go b/internal/observability/observer.go index 619bdb1..531411c 100644 --- a/internal/observability/observer.go +++ b/internal/observability/observer.go @@ -129,6 +129,10 @@ type Observer interface { // SetVerbose enables real-time console output SetVerbose(verbose bool) + + // SetOutcome records the authoritative final task outcome. Recoverable + // intermediate errors remain available as diagnostic evidence. + SetOutcome(success bool) } // AgentObserver is the concrete implementation of Observer @@ -151,6 +155,7 @@ type AgentObserver struct { totalCost float64 errors []string success bool + outcomeSet bool tokensSaved int costSaved float64 activeModel string @@ -325,12 +330,20 @@ func (o *AgentObserver) Summary() *ExecutionSummary { TokensOut: o.tokensOut, TotalCost: o.totalCost, Errors: o.errors, - Success: o.success && len(o.errors) == 0, + Success: o.success && (o.outcomeSet || len(o.errors) == 0), TokensSaved: o.tokensSaved, CostSaved: o.costSaved, } } +// SetOutcome records the authoritative result after retries and verification. +func (o *AgentObserver) SetOutcome(success bool) { + o.mu.Lock() + defer o.mu.Unlock() + o.success = success + o.outcomeSet = true +} + // Subscribe adds a channel to receive events func (o *AgentObserver) Subscribe(ch chan<- Event) { o.mu.Lock() diff --git a/internal/observability/observer_test.go b/internal/observability/observer_test.go index a58eac7..997d8a0 100644 --- a/internal/observability/observer_test.go +++ b/internal/observability/observer_test.go @@ -14,3 +14,26 @@ func TestSummaryTracksToolDurationSeparatelyFromTaskDuration(t *testing.T) { t.Fatalf("expected 25ms of tool time, got %s", summary.ToolDuration) } } + +func TestFinalOutcomeOverridesRecoverableIntermediateErrors(t *testing.T) { + observer := NewObserver() + observer.Emit(&ToolCallEvent{Name: "run_command", Error: "tests failed"}) + observer.SetOutcome(true) + + summary := observer.Summary() + if !summary.Success { + t.Fatal("successful final outcome must not be reported as failed") + } + if len(summary.Errors) != 1 { + t.Fatalf("errors = %v, want recoverable error retained", summary.Errors) + } +} + +func TestSummaryWithErrorFailsBeforeFinalOutcome(t *testing.T) { + observer := NewObserver() + observer.Emit(&ToolCallEvent{Name: "run_command", Error: "tests failed"}) + + if observer.Summary().Success { + t.Fatal("intermediate errors must fail a summary without a final outcome") + } +} diff --git a/scripts/evidence-run/main.go b/scripts/evidence-run/main.go new file mode 100644 index 0000000..39d4eb5 --- /dev/null +++ b/scripts/evidence-run/main.go @@ -0,0 +1,48 @@ +// Command evidence-run executes a controlled agent experiment and writes an +// inspectable, replayable evidence bundle. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/jadercorrea/gptcode/internal/evidence" +) + +func main() { + configPath := flag.String("config", "", "path to an experiment JSON file") + flag.Parse() + if *configPath == "" { + fmt.Fprintln(os.Stderr, "usage: evidence-run -config ") + os.Exit(2) + } + + file, err := os.Open(*configPath) + if err != nil { + fmt.Fprintln(os.Stderr, "opening experiment configuration failed") + os.Exit(1) + } + defer file.Close() + + var experiment evidence.Experiment + decoder := json.NewDecoder(file) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&experiment); err != nil { + fmt.Fprintln(os.Stderr, "decoding experiment configuration failed") + os.Exit(1) + } + + result, err := evidence.RunExperiment(context.Background(), experiment) + if err != nil { + fmt.Fprintln(os.Stderr, "experiment failed safely") + os.Exit(1) + } + if !result.Passed { + fmt.Fprintln(os.Stderr, "experiment completed with failed verification") + os.Exit(1) + } + fmt.Println("experiment completed with verified evidence") +} diff --git a/scripts/evidence-suite/main.go b/scripts/evidence-suite/main.go new file mode 100644 index 0000000..ba633d7 --- /dev/null +++ b/scripts/evidence-suite/main.go @@ -0,0 +1,65 @@ +// Command evidence-suite repeats controlled agent experiments across a +// versioned fixture corpus and writes an aggregate, failure-inclusive report. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/jadercorrea/gptcode/internal/evidence" +) + +func main() { + configPath := flag.String("config", "", "path to a suite JSON file") + output := flag.String("output", "", "override suite output directory") + repetitions := flag.Int("repetitions", 0, "override repetitions per fixture") + verbose := flag.Bool("verbose", false, "stream agent progress while preserving evidence") + flag.Parse() + if *configPath == "" { + fmt.Fprintln(os.Stderr, "usage: evidence-suite -config ") + os.Exit(2) + } + + file, err := os.Open(*configPath) + if err != nil { + fmt.Fprintln(os.Stderr, "opening suite configuration failed") + os.Exit(1) + } + defer file.Close() + + var suite evidence.Suite + decoder := json.NewDecoder(file) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&suite); err != nil { + fmt.Fprintln(os.Stderr, "decoding suite configuration failed") + os.Exit(1) + } + if *output != "" { + suite.Output = *output + } + if *repetitions > 0 { + suite.Repetitions = *repetitions + } + if *verbose { + suite.Progress = os.Stdout + } + + summary, err := evidence.RunSuite(context.Background(), suite) + if err != nil { + fmt.Fprintln(os.Stderr, "suite execution failed safely") + os.Exit(1) + } + fmt.Printf( + "suite completed: %d/%d passed (%.1f%%), median %s\n", + summary.PassedRuns, + summary.TotalRuns, + summary.PassRate*100, + summary.MedianDuration, + ) + if summary.PassedRuns != summary.TotalRuns { + os.Exit(1) + } +} From 3a73c322ce93b16ea90a95ffa635eccd140b375b Mon Sep 17 00:00:00 2001 From: Jader Correa Date: Wed, 29 Jul 2026 12:59:57 -0300 Subject: [PATCH 3/3] docs: launch evidence-based AI series The local-agent study had the depth of an engineering paper but made its central capability-versus-reliability result expensive to discover and difficult to distribute outside the site. Launch Evidence-Based AI Engineering with an executive summary, accessible experiment figure, stronger conclusion, 1,071-word technical brief, and measured copy for LinkedIn, X, Hacker News, and a five-minute video. Tie every format back to the immutable evidence release and protect the editorial contract with site tests. The paper remains the canonical technical account and the brief is an entry point, not a replacement. Browser-based page inspection was unavailable in this session, so the generated HTML, responsive markup, full-resolution graphic, Jekyll build, and repository quality gate were verified independently. --- README.md | 3 +- _roadmap.md | 1 + ...026-07-29-capability-is-not-reliability.md | 266 +++++++++++ docs/_layouts/post.html | 52 +++ ...026-07-29-capability-is-not-reliability.md | 183 ++++++++ ...cessful-agent-run-proves-almost-nothing.md | 431 ++++++++++++++++++ docs/assets/agent-reliability-results.png | Bin 0 -> 100642 bytes docs/assets/agent-reliability-results.svg | 43 ++ docs/test/site_identity_test.rb | 68 +++ 9 files changed, 1046 insertions(+), 1 deletion(-) create mode 100644 docs/_distribution/2026-07-29-capability-is-not-reliability.md create mode 100644 docs/_posts/2026-07-29-capability-is-not-reliability.md create mode 100644 docs/_posts/2026-07-29-one-successful-agent-run-proves-almost-nothing.md create mode 100644 docs/assets/agent-reliability-results.png create mode 100644 docs/assets/agent-reliability-results.svg diff --git a/README.md b/README.md index c9586fd..49d506a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ code. [Website](https://gptcode.dev) · [Architecture](https://gptcode.dev/#architecture) · [Documentation](https://gptcode.dev/guides/getting-started) · -[Engineering essay](https://gptcode.dev/blog/the-workflow-is-the-source-of-truth/) +[Engineering thesis](https://gptcode.dev/blog/the-workflow-is-the-source-of-truth/) · +[Evaluation essay](https://gptcode.dev/blog/2026-07-29-one-successful-agent-run-proves-almost-nothing)

diff --git a/_roadmap.md b/_roadmap.md index 258f7a1..e97fd32 100644 --- a/_roadmap.md +++ b/_roadmap.md @@ -48,6 +48,7 @@ engineering portfolio. - [x] Find a fully GPU-backed local configuration that passes the strengthened safe-store contract, while retaining Qwen's extended-budget failure as negative evidence. - [x] Stream verbose agent stages from the evidence suite without sacrificing the replayable output bundle. - [x] Measure Devstral safe-store repeatability and retain the 0/3 timeout result alongside the earlier reviewed capability pass. +- [x] Launch Evidence-Based AI Engineering with a full paper, technical brief, reusable result figure, and evidence-linked distribution package. - [ ] Improve local safe-store convergence; neither Qwen nor Devstral currently supports a reliability claim on the strengthened contract. - [ ] Extend reviewed repeatability beyond one fixture before publishing a general coding-agent success-rate claim. - [ ] Add an OpenCode importer to challenge the vendor-neutral evidence model. diff --git a/docs/_distribution/2026-07-29-capability-is-not-reliability.md b/docs/_distribution/2026-07-29-capability-is-not-reliability.md new file mode 100644 index 0000000..e9dfc92 --- /dev/null +++ b/docs/_distribution/2026-07-29-capability-is-not-reliability.md @@ -0,0 +1,266 @@ +# Capability Is Not Reliability + +Canonical paper: +https://gptcode.dev/blog/2026-07-29-one-successful-agent-run-proves-almost-nothing + +Technical brief: +https://gptcode.dev/blog/2026-07-29-capability-is-not-reliability + +Evidence release: +https://github.com/jadercorrea/ai-experiments/releases/tag/2026.07.29.1 + +## LinkedIn + +I watched a local coding agent solve a security-sensitive task. + +It changed one Go file, preserved the public API, passed the tests under the +race detector, and passed static analysis. No hosted model was used. + +Then I ran the same configuration three more times. + +It failed every run. + +That experiment reinforced a distinction that is easy to miss when evaluating +AI systems: + +Capability is not reliability. + +One successful run proves that a system can produce a result. It tells us very +little about how often it will produce that result under controlled repetition. + +The first campaign demonstrated bounded capability: 1/1 passed in 26 minutes. +The subsequent repeatability campaign was 0/3, with every run reaching the +30-minute timeout. + +The failed runs were not noise to discard. They exposed different failures at +the security boundary and defects in the surrounding agent harness: hidden +progress, mismatched timeouts, wasteful retries, and incorrect final status. + +I published the full protocol, patches, command logs, snapshots, verification +results, limitations, and checksum as an immutable release. + +Engineering decisions should depend on repeated, inspectable evidence, not the +most memorable demonstration. + +One successful run demonstrates possibility. Engineering depends on +repeatability. + +Full paper: +https://gptcode.dev/blog/2026-07-29-one-successful-agent-run-proves-almost-nothing + +Raw evidence: +https://github.com/jadercorrea/ai-experiments/releases/tag/2026.07.29.1 + +## X thread + +1/10 + +I watched a local coding agent solve a security-sensitive Go task. It preserved +the public API, passed `go test -race`, passed `go vet`, and used no paid API. + +Then I ran the same configuration three more times. + +It failed every run. + +2/10 + +The lesson was not that local models are useless. + +It was that capability is not reliability. + +One successful run proves a system can produce a result. It does not tell us +how often it will. + +3/10 + +The first campaign was a capability check: + +1/1 passed in 26m08s. + +The subsequent repeatability campaign: + +0/3 passed. Every run timed out at roughly 30 minutes. + +These campaigns answer different questions and should not be combined into an +artificial 1/4 score. + +4/10 + +The task enforced filesystem containment: + +- preserve legitimate nested paths; +- reject traversal and absolute paths; +- reject a pre-existing symlink escape; +- preserve the public API; +- pass the race detector and static analysis. + +5/10 + +An earlier model “passed” by rejecting every filename containing `..`. + +That also rejected the legitimate name `v1..v2.txt`. + +The agent had not solved traversal. It had prohibited a substring. + +Better evaluation made the model look worse and the evidence more truthful. + +6/10 + +The failed runs mattered. + +Two rejected legitimate nested paths. Another accepted an absolute path and a +symlink escape. Some retries regressed cases that had already passed. + +Those artifacts explain more than an aggregate score. + +7/10 + +The experiment also found defects in the agent system: + +- a provider timeout shorter than the suite budget; +- progress hidden until completion; +- retries repeating equivalent failures; +- final status contradicting successful verification. + +8/10 + +Fixing those defects did not make the model smarter. + +It made the system more observable, economical, and honest when the model was +not capable enough. + +9/10 + +A useful agent evaluation retains the initial state, task, configuration, +commands, patch, failed runs, verification output, final snapshot, duration, +and human-review boundary. + +The unit of evidence is the restorable run, not the selected transcript. + +10/10 + +I published the complete paper and immutable 2026.07.29.1 evidence release. + +One successful run demonstrates possibility. + +Engineering depends on repeatability. + +https://gptcode.dev/blog/2026-07-29-one-successful-agent-run-proves-almost-nothing + +## Hacker News + +Suggested title: + +One Successful Agent Run Proves Almost Nothing + +Submission URL: + +https://gptcode.dev/blog/2026-07-29-one-successful-agent-run-proves-almost-nothing + +First comment: + +Author here. I ran a local coding model against a small but security-sensitive +filesystem-containment fixture. It passed one capability check, then timed out +in all three runs of a subsequent repeatability campaign. + +The article is about the evaluation method rather than the model: clean +workspaces, required failing baselines, retained failed patches, deterministic +verification, final snapshots, human-review boundaries, and why capability and +reliability are different claims. + +The raw evidence is published in the immutable `2026.07.29.1` release: +https://github.com/jadercorrea/ai-experiments/releases/tag/2026.07.29.1 + +The main limitation is also documented: the historical runner came from a +precursor working tree whose exact transient build commit was not retained. +Future campaigns require a clean immutable harness revision before execution. + +I would be especially interested in how others predeclare agent evaluations, +retain negative runs, and decide when a local model should escalate to a hosted +one. + +## Five-minute video + +### 0:00–0:35 — Hook + +I watched a local coding agent solve a security-sensitive task. It changed one +Go file, preserved the public API, passed the Go race detector, and passed +static analysis. + +Then I ran the same configuration three more times. It failed every run. + +That is the difference between capability and reliability. + +### 0:35–1:15 — The task + +The fixture was a small Go file store. Legitimate nested files had to work, but +absolute paths, parent traversal, and escape through a pre-existing symlink had +to fail. + +It also contained a false-positive guard: `v1..v2.txt` is a legitimate +filename. An earlier model looked successful only because it rejected every +name containing two adjacent dots. + +The contract had to distinguish a real solution from a plausible shortcut. + +### 1:15–2:05 — The result + +With Devstral Small 2 running through Ollama at a 16,384-token context, the +capability check passed in 26 minutes and 8 seconds. + +The same configuration then entered a separate repeatability campaign. Three +fresh workspaces, the same task, the same model profile, the same verification +commands, and the same 30-minute budget. + +All three runs timed out. + +Show the result figure here. + +The first campaign proved possibility. The second did not support a reliability +claim. + +### 2:05–3:00 — What failures revealed + +The failed runs were valuable. Two rejected legitimate nested paths. Another +accepted an absolute path and a symlink escape. Some retries regressed +previously passing behavior. + +The experiment also exposed defects in the agent harness: an internal timeout +shorter than the suite budget, no visible progress during long runs, repeated +equivalent failures, and a final status that could contradict successful +verification. + +Fixing these problems did not improve the model. It improved the truthfulness +of the system around it. + +### 3:00–4:00 — What counts as evidence + +For agent evaluation, I now want the initial repository, exact task and +configuration, clean workspace, command stream, patch, deterministic +verification, final snapshot, duration, failed runs, and human-review boundary. + +Tests establish truth only within the encoded contract. The successful patch +still retained a possible check-to-write race against an actively hostile +concurrent process. That limitation belongs in the result. + +### 4:00–4:40 — The local inference boundary + +Local AI may follow the path of recording tools. Much of the work that once +required a professional studio can now be done well at home. Studios remain +important for specialized work, but they are no longer mandatory for every +recording. + +Likewise, frontier models and managed inference will remain essential, but not +every bounded engineering task should necessarily require them. + +The system should try locally, verify deterministically, and escalate structured +evidence when progress plateaus. + +### 4:40–5:00 — Close + +The complete experiment, including failed patches and snapshots, is published +under release `2026.07.29.1`. + +One successful run demonstrates possibility. + +Engineering depends on repeatability. diff --git a/docs/_layouts/post.html b/docs/_layouts/post.html index f400c22..ff8b453 100644 --- a/docs/_layouts/post.html +++ b/docs/_layouts/post.html @@ -4,6 +4,11 @@

+ {% if page.series %} +

+ {{ page.series }}{% if page.format %} · {{ page.format }}{% endif %} +

+ {% endif %}

{{ page.title }}