Repo map tool with dependency graph and PageRank

docs/roadmap/08-repo-map.md

@@ -0,0 +1,129 @@
+# Repo Map Tool
+
+## Branch: `feature/repo-map`
+
+## Problem
+
+Without a codebase overview, the model has to grep/glob/read speculatively —
+burning tokens and turns trying to find relevant code. A 32B local model with
+32k context can't afford to read 10 files to find the right one.
+
+## What it is
+
+A tree-sitter-powered codebase index that gives the model a compact "table of
+contents" showing the most relevant symbols for the current task.
+
+This is what makes Aider effective with smaller models. Their implementation
+uses tree-sitter for AST parsing + PageRank for relevance ranking.
+
+## Example output
+
+```
+repo_map({ query: "authentication middleware" })
+
+→ Repo Map (ranked by relevance to: "authentication middleware")
+
+  packages/core/src/core/auth.ts
+    ├─ class AuthManager
+    │   ├─ authenticate(token: string): Promise<User>
+    │   ├─ validateSession(sessionId: string): boolean
+    │   └─ refreshToken(user: User): string
+    ├─ interface AuthConfig { ... }
+    └─ function createAuthMiddleware(config: AuthConfig): Middleware
+
+  packages/cli/src/ui/components/AuthDialog.tsx
+    ├─ function AuthDialog(props: AuthDialogProps): JSX.Element
+    └─ const AUTH_PROVIDERS: AuthProvider[]
+
+  packages/core/src/core/contentGenerator.ts
+    ├─ enum AuthType { LOGIN_WITH_GOOGLE, USE_GEMINI, ... }
+    └─ function createContentGenerator(config, authType): ContentGenerator
+
+  (3 more files with lower relevance scores omitted)
+```
+
+## Architecture
+
+### Phase 1: Tree-sitter AST extraction
+
+- Use `web-tree-sitter` (WASM, works everywhere) or `tree-sitter` native bindings
+- Language grammars needed: TypeScript, JavaScript, Python, Go, Rust, Java, C/C++,
+  Ruby, PHP, Swift, Kotlin — ship the top 10, lazy-load others
+- Extract from each file:
+  - Module-level exports
+  - Class declarations + method signatures
+  - Function declarations + parameter types
+  - Interface/type declarations
+  - Import statements (for dependency graph)
+
+### Phase 2: Dependency graph + PageRank
+
+- Build a directed graph: file A imports from file B → edge A→B
+- Run PageRank to score each file's "importance" in the codebase
+- Weight scores toward files relevant to the current query using
+  TF-IDF on symbol names
+
+### Phase 3: Context-aware truncation
+
+- Given a token budget (configurable, default ~2000 tokens), select the
+  top-ranked files and include as many signatures as fit
+- Prioritize: files matching query > high PageRank > recently modified
+- For each file, show: exported symbols with signatures, omit function bodies
+
+### Phase 4: Caching
+
+- Cache the AST + graph per file, keyed on file content hash
+- Invalidate on file modification (watch via `fs.watch` or check at query time)
+- Full reindex only when >20% of files have changed
+
+## Tool declaration
+
+```typescript
+{
+  name: 'repo_map',
+  description: 'Get a ranked map of the most relevant files and symbols '
+    + 'in the codebase for a given query. Returns function/class signatures '
+    + 'without bodies. Use this before diving into unfamiliar code.',
+  parameters: {
+    type: 'object',
+    properties: {
+      query: {
+        type: 'string',
+        description: 'What you are looking for or trying to understand.',
+      },
+      max_tokens: {
+        type: 'number',
+        description: 'Max tokens for the output (default 2000).',
+      },
+    },
+    required: ['query'],
+  },
+}
+```
+
+## Dependencies
+
+```json
+{
+  "web-tree-sitter": "^0.24.0",
+  "tree-sitter-typescript": "^0.23.0",
+  "tree-sitter-javascript": "^0.23.0",
+  "tree-sitter-python": "^0.23.0"
+}
+```
+
+Additional grammars added as optional/lazy-loaded.
+
+## Files to create
+- `packages/core/src/tools/repoMap.ts` — tool entry point
+- `packages/core/src/tools/repoMap.test.ts`
+- `packages/core/src/indexing/astExtractor.ts` — tree-sitter parsing
+- `packages/core/src/indexing/dependencyGraph.ts` — import graph + PageRank
+- `packages/core/src/indexing/repoIndex.ts` — cache layer, incremental updates
+- `packages/core/src/indexing/symbolRanker.ts` — TF-IDF + PageRank scoring
+
+## Files to modify
+- `packages/core/package.json` — add tree-sitter dependencies
+- `packages/core/src/config/config.ts` — register repo_map tool
+
+## Effort: 1-2 weeks — the biggest feature but the biggest differentiator

packages/core/src/config/config.ts

@@ -37,6 +37,7 @@ import { LintTool } from '../tools/lint.js';
 import { RunTestsTool } from '../tools/runTests.js';
 import { CheckpointSaveTool, CheckpointRestoreTool, CheckpointListTool } from '../tools/checkpoint.js';
 import { FileSummaryTool } from '../tools/fileSummary.js';
+import { RepoMapTool } from '../tools/repoMap.js';
 import { DeltaClient } from '../core/client.js';
 import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
 import { GitService } from '../services/gitService.js';
@@ -932,6 +933,7 @@ export class Config {
     registerCoreTool(CheckpointRestoreTool, this);
     registerCoreTool(CheckpointListTool, this);
     registerCoreTool(FileSummaryTool);
+    registerCoreTool(RepoMapTool, this);
 
     await registry.discoverAllTools();
     return registry;

packages/core/src/indexing/dependencyGraph.ts

@@ -0,0 +1,133 @@
+/**
+ * Directed dependency graph with PageRank scoring.
+ *
+ * Nodes are files, edges are import relationships. PageRank identifies
+ * structurally central files (imported by many others) which are likely
+ * to be relevant for understanding the codebase.
+ */
+
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import type { ImportInfo } from './types.js';
+
+export class DependencyGraph {
+  // Adjacency lists: file → files it imports, file → files that import it
+  private forwardEdges = new Map<string, Set<string>>();
+  private reverseEdges = new Map<string, Set<string>>();
+  private allNodes = new Set<string>();
+
+  /** Add a node (file) to the graph. */
+  addNode(filePath: string): void {
+    this.allNodes.add(filePath);
+    if (!this.forwardEdges.has(filePath)) this.forwardEdges.set(filePath, new Set());
+    if (!this.reverseEdges.has(filePath)) this.reverseEdges.set(filePath, new Set());
+  }
+
+  /** Add an edge: sourceFile imports targetFile. */
+  addEdge(sourceFile: string, targetFile: string): void {
+    this.addNode(sourceFile);
+    this.addNode(targetFile);
+    this.forwardEdges.get(sourceFile)!.add(targetFile);
+    this.reverseEdges.get(targetFile)!.add(sourceFile);
+  }
+
+  /**
+   * Resolve imports from a file and add edges to the graph.
+   * Only resolves relative imports — bare specifiers (npm packages) are skipped
+   * since they point outside the repo.
+   */
+  resolveAndAddImports(filePath: string, imports: ImportInfo[]): void {
+    this.addNode(filePath);
+    const dir = path.dirname(filePath);
+
+    for (const imp of imports) {
+      if (!imp.isRelative) continue;
+
+      const resolved = this.resolveImport(dir, imp.source);
+      if (resolved) {
+        this.addEdge(filePath, resolved);
+      }
+    }
+  }
+
+  /**
+   * Resolve a relative import to an absolute file path.
+   * Tries common extensions and index files.
+   */
+  private resolveImport(fromDir: string, source: string): string | null {
+    const base = path.resolve(fromDir, source);
+    const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.py', ''];
+    const indexFiles = ['index.ts', 'index.tsx', 'index.js'];
+
+    // Try direct file with extensions
+    for (const ext of extensions) {
+      const candidate = base + ext;
+      if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
+        return candidate;
+      }
+    }
+
+    // Try as directory with index file
+    for (const idx of indexFiles) {
+      const candidate = path.join(base, idx);
+      if (fs.existsSync(candidate)) {
+        return candidate;
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Iterative PageRank with damping factor.
+   *
+   * Standard algorithm — converges in ~20 iterations for any reasonable
+   * codebase graph. For 5000 files this runs in <50ms.
+   */
+  computePageRank(iterations: number = 20, damping: number = 0.85): Map<string, number> {
+    const N = this.allNodes.size;
+    if (N === 0) return new Map();
+
+    const scores = new Map<string, number>();
+    const baseScore = 1 / N;
+
+    // Initialize: equal probability across all nodes
+    for (const node of this.allNodes) {
+      scores.set(node, baseScore);
+    }
+
+    for (let i = 0; i < iterations; i++) {
+      const newScores = new Map<string, number>();
+      const randomJump = (1 - damping) / N;
+
+      for (const node of this.allNodes) {
+        newScores.set(node, randomJump);
+      }
+
+      for (const node of this.allNodes) {
+        const outEdges = this.forwardEdges.get(node);
+        if (!outEdges || outEdges.size === 0) continue;
+
+        const share = damping * (scores.get(node) || 0) / outEdges.size;
+        for (const target of outEdges) {
+          newScores.set(target, (newScores.get(target) || 0) + share);
+        }
+      }
+
+      // Copy new scores
+      for (const [node, score] of newScores) {
+        scores.set(node, score);
+      }
+    }
+
+    return scores;
+  }
+
+  get nodeCount(): number {
+    return this.allNodes.size;
+  }
+
+  getImporters(filePath: string): Set<string> {
+    return this.reverseEdges.get(filePath) || new Set();
+  }
+}