diff --git a/docs/roadmap/09-lint-test-feedback.md b/docs/roadmap/09-lint-test-feedback.md new file mode 100644 index 00000000000..3d159ee119e --- /dev/null +++ b/docs/roadmap/09-lint-test-feedback.md @@ -0,0 +1,143 @@ +# Lint / Test Feedback Tool + +## Branch: `feature/lint-test-feedback` + +## Problem + +The model can run `shell("npm test")` but gets back raw terminal output. It has +to parse error messages, stack traces, and lint output from unstructured text. +Weaker models frequently misparse this and fix the wrong thing. + +There's also no automatic feedback loop — the model writes code and moves on +unless the user tells it to test. + +## What it is + +A structured tool that: +1. Auto-detects the project's lint and test commands +2. Runs them and parses output into structured `{file, line, column, message, severity}` diagnostics +3. Returns a clean, actionable error list the model can iterate on + +## Auto-detection logic + +Scan the project root for known config files and extract commands: + +| File | Lint command | Test command | +|------|-------------|--------------| +| `package.json` | `scripts.lint` or `npx eslint .` | `scripts.test` | +| `pyproject.toml` | `ruff check .` or `flake8 .` | `pytest` | +| `Cargo.toml` | `cargo clippy` | `cargo test` | +| `Makefile` | `make lint` (if target exists) | `make test` | +| `go.mod` | `golangci-lint run` | `go test ./...` | +| `Gemfile` | `rubocop` | `bundle exec rspec` | +| `.eslintrc.*` | `npx eslint .` | — | +| `tsconfig.json` | `npx tsc --noEmit` | — | + +Fall back to user-configured commands in `.delta/settings.json`: + +```json +{ + "lintCommand": "npm run lint", + "testCommand": "npm run test", + "testFilePattern": "**/*.test.ts" +} +``` + +## Tool declarations + +### `lint` + +```typescript +{ + name: 'lint', + description: 'Run the project linter and return structured diagnostics. ' + + 'Auto-detects the lint command from project config. Returns errors ' + + 'as {file, line, column, message, severity} objects.', + parameters: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Optional: lint only this file or directory.', + }, + }, + }, +} +``` + +### `run_tests` + +```typescript +{ + name: 'run_tests', + description: 'Run project tests and return structured results. ' + + 'Auto-detects the test command from project config. Returns ' + + 'pass/fail per test with error details for failures.', + parameters: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Optional: run tests for this file/directory only.', + }, + filter: { + type: 'string', + description: 'Optional: filter test names matching this pattern.', + }, + }, + }, +} +``` + +## Output format + +```typescript +interface DiagnosticResult { + command: string; // what was actually run + exitCode: number; + diagnostics: Diagnostic[]; + summary: string; // "3 errors, 2 warnings" or "12/14 tests passed" +} + +interface Diagnostic { + file: string; + line: number; + column?: number; + message: string; + severity: 'error' | 'warning' | 'info'; + rule?: string; // e.g. "no-unused-vars", "E501" + source?: string; // e.g. "eslint", "typescript", "pytest" +} +``` + +## Output parsers + +Each linter/test runner has its own output format. Build parsers for: + +- **ESLint** — use `--format json` flag (structured natively) +- **TypeScript (tsc)** — parse `file(line,col): error TS####: message` +- **pytest** — use `--tb=short` + parse traceback format +- **Rust (cargo)** — JSON output with `--message-format=json` +- **Go** — parse `file:line:col: message` format +- **Generic fallback** — regex for common `file:line: message` patterns + +## Auto-run mode (optional) + +Config flag `autoLintAfterEdit: true` — after every `write_file` or `edit` +tool call, automatically run lint and inject diagnostics into the next turn. +This creates the self-correcting feedback loop without the model having to +remember to call lint. + +## Files to create +- `packages/core/src/tools/lint.ts` — lint tool +- `packages/core/src/tools/runTests.ts` — test runner tool +- `packages/core/src/tools/diagnostics/projectDetector.ts` — auto-detect commands +- `packages/core/src/tools/diagnostics/outputParsers.ts` — structured parsers +- `packages/core/src/tools/diagnostics/types.ts` — Diagnostic, DiagnosticResult +- Tests for each + +## Files to modify +- `packages/core/src/config/config.ts` — register tools, add settings +- `packages/cli/src/config/settingsSchema.ts` — lintCommand, testCommand, autoLintAfterEdit + +## Effort: 3-5 days diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 58df2bed6d0..4e76174ac6f 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -33,6 +33,8 @@ import { SubAgentTool } from '../tools/subAgentTool.js'; import { ThinkTool } from '../tools/think.js'; import { DiffPreviewTool } from '../tools/diffPreview.js'; import { MultiEditTool } from '../tools/multiEdit.js'; +import { LintTool } from '../tools/lint.js'; +import { RunTestsTool } from '../tools/runTests.js'; import { DeltaClient } from '../core/client.js'; import { FileDiscoveryService } from '../services/fileDiscoveryService.js'; import { GitService } from '../services/gitService.js'; @@ -922,6 +924,8 @@ export class Config { registerCoreTool(ThinkTool); registerCoreTool(DiffPreviewTool); registerCoreTool(MultiEditTool); + registerCoreTool(LintTool); + registerCoreTool(RunTestsTool); await registry.discoverAllTools(); return registry; diff --git a/packages/core/src/tools/diagnostics/outputParsers.ts b/packages/core/src/tools/diagnostics/outputParsers.ts new file mode 100644 index 00000000000..a1b00750d50 --- /dev/null +++ b/packages/core/src/tools/diagnostics/outputParsers.ts @@ -0,0 +1,84 @@ +/** + * Parse structured diagnostics from lint/test tool output. + * + * Each parser attempts to extract file:line:message tuples from the raw + * output of common lint and test tools. Falls back to returning raw output + * if no structured format is detected. + */ + +export interface Diagnostic { + file: string; + line: number; + column?: number; + message: string; + severity: 'error' | 'warning' | 'info'; + rule?: string; + source?: string; +} + +/** + * Parse diagnostics from generic file:line:col:message format. + * Covers: tsc, Go, Python tracebacks, and most linters' default output. + */ +export function parseGenericDiagnostics(output: string): Diagnostic[] { + const diagnostics: Diagnostic[] = []; + // Match patterns like: file.ts(10,5): error TS1234: message + // Or: file.py:10:5: E501 line too long + // Or: file.go:10:5: message + // Try TypeScript pattern first + const tsPattern = /^(.+?)\((\d+),(\d+)\):\s*(error|warning)\s*(TS\d+):\s*(.+)$/gm; + let match; + while ((match = tsPattern.exec(output)) !== null) { + diagnostics.push({ + file: match[1], + line: parseInt(match[2], 10), + column: parseInt(match[3], 10), + severity: match[4] as 'error' | 'warning', + rule: match[5], + message: match[6], + source: 'typescript', + }); + } + + if (diagnostics.length > 0) return diagnostics; + + // Try file:line:col: message pattern + const genericPattern = /^(.+?):(\d+):(\d+):\s*(?:(error|warning|note|info)\s+)?(.+)$/gm; + while ((match = genericPattern.exec(output)) !== null) { + // Skip lines that look like stack traces or URLs + if (match[1].includes('node_modules') || match[1].startsWith('http')) continue; + + diagnostics.push({ + file: match[1].trim(), + line: parseInt(match[2], 10), + column: parseInt(match[3], 10), + severity: (match[4] as 'error' | 'warning' | 'info') || 'error', + message: match[5].trim(), + }); + } + + return diagnostics; +} + +/** + * Format diagnostics into a concise summary for the model. + */ +export function formatDiagnostics(diagnostics: Diagnostic[], command: string, exitCode: number): string { + const errors = diagnostics.filter(d => d.severity === 'error'); + const warnings = diagnostics.filter(d => d.severity === 'warning'); + + let summary = `Command: ${command}\nExit code: ${exitCode}\n`; + summary += `${errors.length} error(s), ${warnings.length} warning(s)\n\n`; + + for (const d of diagnostics.slice(0, 50)) { + const loc = d.column ? `${d.file}:${d.line}:${d.column}` : `${d.file}:${d.line}`; + const rule = d.rule ? ` [${d.rule}]` : ''; + summary += `${d.severity.toUpperCase()} ${loc}: ${d.message}${rule}\n`; + } + + if (diagnostics.length > 50) { + summary += `\n... and ${diagnostics.length - 50} more diagnostic(s)`; + } + + return summary; +} diff --git a/packages/core/src/tools/diagnostics/projectDetector.ts b/packages/core/src/tools/diagnostics/projectDetector.ts new file mode 100644 index 00000000000..e39ed95c253 --- /dev/null +++ b/packages/core/src/tools/diagnostics/projectDetector.ts @@ -0,0 +1,86 @@ +/** + * Auto-detect the project's lint and test commands by scanning for known + * config files at the project root. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export interface DetectedCommands { + lint: string | null; + test: string | null; +} + +/** + * Scan the project root for known config files and infer lint/test commands. + * Returns null for commands that can't be detected. + */ +export function detectProjectCommands(projectRoot: string): DetectedCommands { + const result: DetectedCommands = { lint: null, test: null }; + + // Node.js / JavaScript / TypeScript + const pkgJsonPath = path.join(projectRoot, 'package.json'); + if (fs.existsSync(pkgJsonPath)) { + try { + const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')); + const scripts = pkg.scripts || {}; + if (scripts.lint) result.lint = 'npm run lint'; + if (scripts.test) result.test = 'npm run test'; + // If no lint script but eslint config exists + if (!result.lint && ( + fs.existsSync(path.join(projectRoot, '.eslintrc.js')) || + fs.existsSync(path.join(projectRoot, '.eslintrc.json')) || + fs.existsSync(path.join(projectRoot, '.eslintrc.yml')) || + fs.existsSync(path.join(projectRoot, 'eslint.config.js')) || + fs.existsSync(path.join(projectRoot, 'eslint.config.mjs')) + )) { + result.lint = 'npx eslint .'; + } + // TypeScript type-check as lint fallback + if (!result.lint && fs.existsSync(path.join(projectRoot, 'tsconfig.json'))) { + result.lint = 'npx tsc --noEmit'; + } + } catch { /* malformed package.json */ } + } + + // Python + const pyprojectPath = path.join(projectRoot, 'pyproject.toml'); + if (fs.existsSync(pyprojectPath)) { + if (!result.lint) result.lint = 'ruff check .'; + if (!result.test) result.test = 'pytest'; + } + + // Rust + if (fs.existsSync(path.join(projectRoot, 'Cargo.toml'))) { + if (!result.lint) result.lint = 'cargo clippy'; + if (!result.test) result.test = 'cargo test'; + } + + // Go + if (fs.existsSync(path.join(projectRoot, 'go.mod'))) { + if (!result.lint) result.lint = 'golangci-lint run'; + if (!result.test) result.test = 'go test ./...'; + } + + // Ruby + if (fs.existsSync(path.join(projectRoot, 'Gemfile'))) { + if (!result.lint) result.lint = 'rubocop'; + if (!result.test) result.test = 'bundle exec rspec'; + } + + // Makefile targets + const makefilePath = path.join(projectRoot, 'Makefile'); + if (fs.existsSync(makefilePath)) { + try { + const content = fs.readFileSync(makefilePath, 'utf-8'); + if (!result.lint && /^lint\s*:/m.test(content)) { + result.lint = 'make lint'; + } + if (!result.test && /^test\s*:/m.test(content)) { + result.test = 'make test'; + } + } catch { /* unreadable */ } + } + + return result; +} diff --git a/packages/core/src/tools/lint.ts b/packages/core/src/tools/lint.ts new file mode 100644 index 00000000000..9589ee40853 --- /dev/null +++ b/packages/core/src/tools/lint.ts @@ -0,0 +1,116 @@ +/** + * Lint tool — auto-detect and run the project linter with structured output. + */ + +import { + BaseDeclarativeTool, + BaseToolInvocation, + Kind, + ToolResult, +} from './tools.js'; +import { FunctionDeclaration } from '@google/genai'; +import { execSync } from 'node:child_process'; +import { detectProjectCommands } from './diagnostics/projectDetector.js'; +import { parseGenericDiagnostics, formatDiagnostics } from './diagnostics/outputParsers.js'; +interface LintParams { + path?: string; +} + +const lintSchemaData: FunctionDeclaration = { + name: 'lint', + description: + 'Run the project linter and return structured diagnostics. Auto-detects the lint command from project config (package.json, pyproject.toml, Cargo.toml, etc). Returns errors as file:line:message entries.', + parametersJsonSchema: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Optional: lint only this file or directory.', + }, + }, + $schema: 'http://json-schema.org/draft-07/schema#', + }, +}; + +class LintInvocation extends BaseToolInvocation { + constructor(params: LintParams, private projectRoot: string) { + super(params); + } + + getDescription(): string { + return this.params.path ? `Lint ${this.params.path}` : 'Lint project'; + } + + async execute(): Promise { + const detected = detectProjectCommands(this.projectRoot); + const command = detected.lint; + + if (!command) { + return { + llmContent: [{ text: 'No lint command detected. Configure one in package.json scripts or add a .eslintrc file.' }], + returnDisplay: 'No lint command detected.', + error: { message: 'No lint command detected' }, + }; + } + + const fullCommand = this.params.path ? `${command} ${this.params.path}` : command; + + let stdout = ''; + let stderr = ''; + let exitCode = 0; + + try { + const output = execSync(fullCommand, { + cwd: this.projectRoot, + timeout: 60_000, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + stdout = output || ''; + } catch (err: unknown) { + const execError = err as { stdout?: string; stderr?: string; status?: number }; + stdout = execError.stdout || ''; + stderr = execError.stderr || ''; + exitCode = execError.status || 1; + } + + const rawOutput = (stdout + '\n' + stderr).trim(); + const diagnostics = parseGenericDiagnostics(rawOutput); + + if (diagnostics.length > 0) { + const formatted = formatDiagnostics(diagnostics, fullCommand, exitCode); + return { + llmContent: [{ text: formatted }], + returnDisplay: formatted, + }; + } + + // No structured diagnostics parsed — return raw output + const summary = exitCode === 0 + ? `Lint passed (${fullCommand})` + : `Lint failed (exit ${exitCode}):\n${rawOutput.substring(0, 2000)}`; + + return { + llmContent: [{ text: summary }], + returnDisplay: summary, + }; + } +} + +export class LintTool extends BaseDeclarativeTool { + static readonly Name: string = lintSchemaData.name!; + + constructor() { + super( + LintTool.Name, + 'Lint', + lintSchemaData.description!, + Kind.Execute, + lintSchemaData.parametersJsonSchema, + ); + } + + protected createInvocation(params: LintParams) { + return new LintInvocation(params, process.cwd()); + } +} diff --git a/packages/core/src/tools/runTests.ts b/packages/core/src/tools/runTests.ts new file mode 100644 index 00000000000..686e5453f73 --- /dev/null +++ b/packages/core/src/tools/runTests.ts @@ -0,0 +1,130 @@ +/** + * Test runner tool — auto-detect and run project tests with structured output. + */ + +import { + BaseDeclarativeTool, + BaseToolInvocation, + Kind, + ToolResult, +} from './tools.js'; +import { FunctionDeclaration } from '@google/genai'; +import { execSync } from 'node:child_process'; +import { detectProjectCommands } from './diagnostics/projectDetector.js'; +import { parseGenericDiagnostics, formatDiagnostics } from './diagnostics/outputParsers.js'; +interface RunTestsParams { + path?: string; + filter?: string; +} + +const runTestsSchemaData: FunctionDeclaration = { + name: 'run_tests', + description: + 'Run project tests and return structured results. Auto-detects the test command from project config (package.json, pyproject.toml, Cargo.toml, etc). Returns pass/fail with error details for failures.', + parametersJsonSchema: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Optional: run tests for this file/directory only.', + }, + filter: { + type: 'string', + description: 'Optional: filter test names matching this pattern.', + }, + }, + $schema: 'http://json-schema.org/draft-07/schema#', + }, +}; + +class RunTestsInvocation extends BaseToolInvocation { + constructor(params: RunTestsParams, private projectRoot: string) { + super(params); + } + + getDescription(): string { + if (this.params.path) return `Test ${this.params.path}`; + if (this.params.filter) return `Test matching "${this.params.filter}"`; + return 'Run tests'; + } + + async execute(): Promise { + const detected = detectProjectCommands(this.projectRoot); + const command = detected.test; + + if (!command) { + return { + llmContent: [{ text: 'No test command detected. Configure one in package.json scripts.' }], + returnDisplay: 'No test command detected.', + error: { message: 'No test command detected' }, + }; + } + + // Build the full command with optional path/filter + let fullCommand = command; + if (this.params.path) fullCommand += ` ${this.params.path}`; + if (this.params.filter) fullCommand += ` -t "${this.params.filter}"`; + + let stdout = ''; + let stderr = ''; + let exitCode = 0; + + try { + const output = execSync(fullCommand, { + cwd: this.projectRoot, + timeout: 120_000, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + stdout = output || ''; + } catch (err: unknown) { + const execError = err as { stdout?: string; stderr?: string; status?: number }; + stdout = execError.stdout || ''; + stderr = execError.stderr || ''; + exitCode = execError.status || 1; + } + + const rawOutput = (stdout + '\n' + stderr).trim(); + const diagnostics = parseGenericDiagnostics(rawOutput); + + if (diagnostics.length > 0) { + const formatted = formatDiagnostics(diagnostics, fullCommand, exitCode); + return { + llmContent: [{ text: formatted }], + returnDisplay: formatted, + }; + } + + // Return raw output — test runners often have their own formatting + const truncated = rawOutput.length > 3000 + ? rawOutput.substring(0, 3000) + '\n... (truncated)' + : rawOutput; + + const summary = exitCode === 0 + ? `Tests passed (${fullCommand}):\n${truncated}` + : `Tests failed (exit ${exitCode}, ${fullCommand}):\n${truncated}`; + + return { + llmContent: [{ text: summary }], + returnDisplay: summary, + }; + } +} + +export class RunTestsTool extends BaseDeclarativeTool { + static readonly Name: string = runTestsSchemaData.name!; + + constructor() { + super( + RunTestsTool.Name, + 'Run Tests', + runTestsSchemaData.description!, + Kind.Execute, + runTestsSchemaData.parametersJsonSchema, + ); + } + + protected createInvocation(params: RunTestsParams) { + return new RunTestsInvocation(params, process.cwd()); + } +}