Lint and test runner tools with auto-detection

docs/roadmap/09-lint-test-feedback.md

@@ -0,0 +1,143 @@
+# Lint / Test Feedback Tool
+
+## Branch: `feature/lint-test-feedback`
+
+## Problem
+
+The model can run `shell("npm test")` but gets back raw terminal output. It has
+to parse error messages, stack traces, and lint output from unstructured text.
+Weaker models frequently misparse this and fix the wrong thing.
+
+There's also no automatic feedback loop — the model writes code and moves on
+unless the user tells it to test.
+
+## What it is
+
+A structured tool that:
+1. Auto-detects the project's lint and test commands
+2. Runs them and parses output into structured `{file, line, column, message, severity}` diagnostics
+3. Returns a clean, actionable error list the model can iterate on
+
+## Auto-detection logic
+
+Scan the project root for known config files and extract commands:
+
+| File | Lint command | Test command |
+|------|-------------|--------------|
+| `package.json` | `scripts.lint` or `npx eslint .` | `scripts.test` |
+| `pyproject.toml` | `ruff check .` or `flake8 .` | `pytest` |
+| `Cargo.toml` | `cargo clippy` | `cargo test` |
+| `Makefile` | `make lint` (if target exists) | `make test` |
+| `go.mod` | `golangci-lint run` | `go test ./...` |
+| `Gemfile` | `rubocop` | `bundle exec rspec` |
+| `.eslintrc.*` | `npx eslint .` | — |
+| `tsconfig.json` | `npx tsc --noEmit` | — |
+
+Fall back to user-configured commands in `.delta/settings.json`:
+
+```json
+{
+  "lintCommand": "npm run lint",
+  "testCommand": "npm run test",
+  "testFilePattern": "**/*.test.ts"
+}
+```
+
+## Tool declarations
+
+### `lint`
+
+```typescript
+{
+  name: 'lint',
+  description: 'Run the project linter and return structured diagnostics. '
+    + 'Auto-detects the lint command from project config. Returns errors '
+    + 'as {file, line, column, message, severity} objects.',
+  parameters: {
+    type: 'object',
+    properties: {
+      path: {
+        type: 'string',
+        description: 'Optional: lint only this file or directory.',
+      },
+    },
+  },
+}
+```
+
+### `run_tests`
+
+```typescript
+{
+  name: 'run_tests',
+  description: 'Run project tests and return structured results. '
+    + 'Auto-detects the test command from project config. Returns '
+    + 'pass/fail per test with error details for failures.',
+  parameters: {
+    type: 'object',
+    properties: {
+      path: {
+        type: 'string',
+        description: 'Optional: run tests for this file/directory only.',
+      },
+      filter: {
+        type: 'string',
+        description: 'Optional: filter test names matching this pattern.',
+      },
+    },
+  },
+}
+```
+
+## Output format
+
+```typescript
+interface DiagnosticResult {
+  command: string;        // what was actually run
+  exitCode: number;
+  diagnostics: Diagnostic[];
+  summary: string;        // "3 errors, 2 warnings" or "12/14 tests passed"
+}
+
+interface Diagnostic {
+  file: string;
+  line: number;
+  column?: number;
+  message: string;
+  severity: 'error' | 'warning' | 'info';
+  rule?: string;          // e.g. "no-unused-vars", "E501"
+  source?: string;        // e.g. "eslint", "typescript", "pytest"
+}
+```
+
+## Output parsers
+
+Each linter/test runner has its own output format. Build parsers for:
+
+- **ESLint** — use `--format json` flag (structured natively)
+- **TypeScript (tsc)** — parse `file(line,col): error TS####: message`
+- **pytest** — use `--tb=short` + parse traceback format
+- **Rust (cargo)** — JSON output with `--message-format=json`
+- **Go** — parse `file:line:col: message` format
+- **Generic fallback** — regex for common `file:line: message` patterns
+
+## Auto-run mode (optional)
+
+Config flag `autoLintAfterEdit: true` — after every `write_file` or `edit`
+tool call, automatically run lint and inject diagnostics into the next turn.
+This creates the self-correcting feedback loop without the model having to
+remember to call lint.
+
+## Files to create
+- `packages/core/src/tools/lint.ts` — lint tool
+- `packages/core/src/tools/runTests.ts` — test runner tool
+- `packages/core/src/tools/diagnostics/projectDetector.ts` — auto-detect commands
+- `packages/core/src/tools/diagnostics/outputParsers.ts` — structured parsers
+- `packages/core/src/tools/diagnostics/types.ts` — Diagnostic, DiagnosticResult
+- Tests for each
+
+## Files to modify
+- `packages/core/src/config/config.ts` — register tools, add settings
+- `packages/cli/src/config/settingsSchema.ts` — lintCommand, testCommand, autoLintAfterEdit
+
+## Effort: 3-5 days

packages/core/src/config/config.ts

@@ -33,6 +33,8 @@ import { SubAgentTool } from '../tools/subAgentTool.js';
 import { ThinkTool } from '../tools/think.js';
 import { DiffPreviewTool } from '../tools/diffPreview.js';
 import { MultiEditTool } from '../tools/multiEdit.js';
+import { LintTool } from '../tools/lint.js';
+import { RunTestsTool } from '../tools/runTests.js';
 import { DeltaClient } from '../core/client.js';
 import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
 import { GitService } from '../services/gitService.js';
@@ -922,6 +924,8 @@ export class Config {
     registerCoreTool(ThinkTool);
     registerCoreTool(DiffPreviewTool);
     registerCoreTool(MultiEditTool);
+    registerCoreTool(LintTool);
+    registerCoreTool(RunTestsTool);
 
     await registry.discoverAllTools();
     return registry;

packages/core/src/tools/diagnostics/outputParsers.ts

@@ -0,0 +1,84 @@
+/**
+ * Parse structured diagnostics from lint/test tool output.
+ *
+ * Each parser attempts to extract file:line:message tuples from the raw
+ * output of common lint and test tools. Falls back to returning raw output
+ * if no structured format is detected.
+ */
+
+export interface Diagnostic {
+  file: string;
+  line: number;
+  column?: number;
+  message: string;
+  severity: 'error' | 'warning' | 'info';
+  rule?: string;
+  source?: string;
+}
+
+/**
+ * Parse diagnostics from generic file:line:col:message format.
+ * Covers: tsc, Go, Python tracebacks, and most linters' default output.
+ */
+export function parseGenericDiagnostics(output: string): Diagnostic[] {
+  const diagnostics: Diagnostic[] = [];
+  // Match patterns like: file.ts(10,5): error TS1234: message
+  // Or: file.py:10:5: E501 line too long
+  // Or: file.go:10:5: message
+  // Try TypeScript pattern first
+  const tsPattern = /^(.+?)\((\d+),(\d+)\):\s*(error|warning)\s*(TS\d+):\s*(.+)$/gm;
+  let match;
+  while ((match = tsPattern.exec(output)) !== null) {
+    diagnostics.push({
+      file: match[1],
+      line: parseInt(match[2], 10),
+      column: parseInt(match[3], 10),
+      severity: match[4] as 'error' | 'warning',
+      rule: match[5],
+      message: match[6],
+      source: 'typescript',
+    });
+  }
+
+  if (diagnostics.length > 0) return diagnostics;
+
+  // Try file:line:col: message pattern
+  const genericPattern = /^(.+?):(\d+):(\d+):\s*(?:(error|warning|note|info)\s+)?(.+)$/gm;
+  while ((match = genericPattern.exec(output)) !== null) {
+    // Skip lines that look like stack traces or URLs
+    if (match[1].includes('node_modules') || match[1].startsWith('http')) continue;
+
+    diagnostics.push({
+      file: match[1].trim(),
+      line: parseInt(match[2], 10),
+      column: parseInt(match[3], 10),
+      severity: (match[4] as 'error' | 'warning' | 'info') || 'error',
+      message: match[5].trim(),
+    });
+  }
+
+  return diagnostics;
+}
+
+/**
+ * Format diagnostics into a concise summary for the model.
+ */
+export function formatDiagnostics(diagnostics: Diagnostic[], command: string, exitCode: number): string {
+  const errors = diagnostics.filter(d => d.severity === 'error');
+  const warnings = diagnostics.filter(d => d.severity === 'warning');
+
+  let summary = `Command: ${command}\nExit code: ${exitCode}\n`;
+  summary += `${errors.length} error(s), ${warnings.length} warning(s)\n\n`;
+
+  for (const d of diagnostics.slice(0, 50)) {
+    const loc = d.column ? `${d.file}:${d.line}:${d.column}` : `${d.file}:${d.line}`;
+    const rule = d.rule ? ` [${d.rule}]` : '';
+    summary += `${d.severity.toUpperCase()} ${loc}: ${d.message}${rule}\n`;
+  }
+
+  if (diagnostics.length > 50) {
+    summary += `\n... and ${diagnostics.length - 50} more diagnostic(s)`;
+  }
+
+  return summary;
+}

packages/core/src/tools/diagnostics/projectDetector.ts

@@ -0,0 +1,86 @@
+/**
+ * Auto-detect the project's lint and test commands by scanning for known
+ * config files at the project root.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export interface DetectedCommands {
+  lint: string | null;
+  test: string | null;
+}
+
+/**
+ * Scan the project root for known config files and infer lint/test commands.
+ * Returns null for commands that can't be detected.
+ */
+export function detectProjectCommands(projectRoot: string): DetectedCommands {
+  const result: DetectedCommands = { lint: null, test: null };
+
+  // Node.js / JavaScript / TypeScript
+  const pkgJsonPath = path.join(projectRoot, 'package.json');
+  if (fs.existsSync(pkgJsonPath)) {
+    try {
+      const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8'));
+      const scripts = pkg.scripts || {};
+      if (scripts.lint) result.lint = 'npm run lint';
+      if (scripts.test) result.test = 'npm run test';
+      // If no lint script but eslint config exists
+      if (!result.lint && (
+        fs.existsSync(path.join(projectRoot, '.eslintrc.js')) ||
+        fs.existsSync(path.join(projectRoot, '.eslintrc.json')) ||
+        fs.existsSync(path.join(projectRoot, '.eslintrc.yml')) ||
+        fs.existsSync(path.join(projectRoot, 'eslint.config.js')) ||
+        fs.existsSync(path.join(projectRoot, 'eslint.config.mjs'))
+      )) {
+        result.lint = 'npx eslint .';
+      }
+      // TypeScript type-check as lint fallback
+      if (!result.lint && fs.existsSync(path.join(projectRoot, 'tsconfig.json'))) {
+        result.lint = 'npx tsc --noEmit';
+      }
+    } catch { /* malformed package.json */ }
+  }
+
+  // Python
+  const pyprojectPath = path.join(projectRoot, 'pyproject.toml');
+  if (fs.existsSync(pyprojectPath)) {
+    if (!result.lint) result.lint = 'ruff check .';
+    if (!result.test) result.test = 'pytest';
+  }
+
+  // Rust
+  if (fs.existsSync(path.join(projectRoot, 'Cargo.toml'))) {
+    if (!result.lint) result.lint = 'cargo clippy';
+    if (!result.test) result.test = 'cargo test';
+  }
+
+  // Go
+  if (fs.existsSync(path.join(projectRoot, 'go.mod'))) {
+    if (!result.lint) result.lint = 'golangci-lint run';
+    if (!result.test) result.test = 'go test ./...';
+  }
+
+  // Ruby
+  if (fs.existsSync(path.join(projectRoot, 'Gemfile'))) {
+    if (!result.lint) result.lint = 'rubocop';
+    if (!result.test) result.test = 'bundle exec rspec';
+  }
+
+  // Makefile targets
+  const makefilePath = path.join(projectRoot, 'Makefile');
+  if (fs.existsSync(makefilePath)) {
+    try {
+      const content = fs.readFileSync(makefilePath, 'utf-8');
+      if (!result.lint && /^lint\s*:/m.test(content)) {
+        result.lint = 'make lint';
+      }
+      if (!result.test && /^test\s*:/m.test(content)) {
+        result.test = 'make test';
+      }
+    } catch { /* unreadable */ }
+  }
+
+  return result;
+}