diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 460d78e..226a640 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,7 +5,7 @@ { "name": "codebase-index", "source": "./", - "description": "Local-first hybrid codebase index skill with auto-provisioned CLI." + "description": "Find, trace, and predict across a local codebase with evidence-bearing retrieval." } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 1635e02..3f359dc 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,8 +2,8 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "codebase-index", "displayName": "Codebase Index", - "description": "Local-first hybrid codebase index. Auto-provisions its Python CLI on first session start; the skill searches the index so Claude reads only the most relevant files.", - "version": "1.6.0", + "description": "Give Claude a precise local map of your codebase: find implementations, trace behavior, and predict change impact with file-line evidence.", + "version": "1.7.0", "author": { "name": "codebase-index contributors" }, diff --git a/.claude/skills/codebase-index/.skill_version b/.claude/skills/codebase-index/.skill_version index dc1e644..bd8bf88 100644 --- a/.claude/skills/codebase-index/.skill_version +++ b/.claude/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/.claude/skills/codebase-index/SKILL.md b/.claude/skills/codebase-index/SKILL.md index ee31b7d..9404686 100644 --- a/.claude/skills/codebase-index/SKILL.md +++ b/.claude/skills/codebase-index/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/.claude/skills/codebase-index/references/commands.md b/.claude/skills/codebase-index/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/.claude/skills/codebase-index/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/.claude/skills/codebase-index/references/response-contract.md b/.claude/skills/codebase-index/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/.claude/skills/codebase-index/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/.claude/skills/codebase-index/scripts/cbx b/.claude/skills/codebase-index/scripts/cbx index 5666358..94a9114 100644 --- a/.claude/skills/codebase-index/scripts/cbx +++ b/.claude/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.claude/skills/codebase-index/scripts/cbx.ps1 b/.claude/skills/codebase-index/scripts/cbx.ps1 index bb8e05d..eee56ae 100644 --- a/.claude/skills/codebase-index/scripts/cbx.ps1 +++ b/.claude/skills/codebase-index/scripts/cbx.ps1 @@ -8,7 +8,10 @@ param( ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @( + "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", + "path", "describe", "graph", "stats", "doctor", "update", "index" +) if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" diff --git a/.codex/skills/codebase-index/.skill_version b/.codex/skills/codebase-index/.skill_version index dc1e644..bd8bf88 100644 --- a/.codex/skills/codebase-index/.skill_version +++ b/.codex/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/.codex/skills/codebase-index/SKILL.md b/.codex/skills/codebase-index/SKILL.md index ee31b7d..9404686 100644 --- a/.codex/skills/codebase-index/SKILL.md +++ b/.codex/skills/codebase-index/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/.codex/skills/codebase-index/references/commands.md b/.codex/skills/codebase-index/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/.codex/skills/codebase-index/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/.codex/skills/codebase-index/references/response-contract.md b/.codex/skills/codebase-index/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/.codex/skills/codebase-index/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/.codex/skills/codebase-index/scripts/cbx b/.codex/skills/codebase-index/scripts/cbx index 5666358..94a9114 100644 --- a/.codex/skills/codebase-index/scripts/cbx +++ b/.codex/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.codex/skills/codebase-index/scripts/cbx.ps1 b/.codex/skills/codebase-index/scripts/cbx.ps1 index bb8e05d..eee56ae 100644 --- a/.codex/skills/codebase-index/scripts/cbx.ps1 +++ b/.codex/skills/codebase-index/scripts/cbx.ps1 @@ -8,7 +8,10 @@ param( ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @( + "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", + "path", "describe", "graph", "stats", "doctor", "update", "index" +) if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e2657b..e0d031a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,12 +5,14 @@ on: branches: [main] pull_request: +permissions: read-all + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - run: python -m pip install --upgrade pip @@ -31,8 +33,8 @@ jobs: python-version: ["3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df2539c..5756979 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,12 +5,14 @@ on: tags: ["v*"] workflow_dispatch: # manual (re)publish — e.g. push an already-tagged version to PyPI +permissions: read-all + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - run: pip install -e ".[dev,build]" @@ -22,7 +24,7 @@ jobs: run: twine check dist/* - name: Clean-machine smoke run: python scripts/release_smoke.py - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: dist path: dist/ @@ -36,13 +38,13 @@ jobs: permissions: contents: write # create the GitHub release steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: dist path: dist - name: Create GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: files: dist/* generate_release_notes: true @@ -54,9 +56,9 @@ jobs: permissions: id-token: write # OIDC for PyPI Trusted Publishing — no token stored anywhere steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: dist path: dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 diff --git a/.opencode/skills/codebase-index/.skill_version b/.opencode/skills/codebase-index/.skill_version index dc1e644..bd8bf88 100644 --- a/.opencode/skills/codebase-index/.skill_version +++ b/.opencode/skills/codebase-index/.skill_version @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/.opencode/skills/codebase-index/SKILL.md b/.opencode/skills/codebase-index/SKILL.md index ee31b7d..9404686 100644 --- a/.opencode/skills/codebase-index/SKILL.md +++ b/.opencode/skills/codebase-index/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/.opencode/skills/codebase-index/references/commands.md b/.opencode/skills/codebase-index/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/.opencode/skills/codebase-index/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/.opencode/skills/codebase-index/references/response-contract.md b/.opencode/skills/codebase-index/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/.opencode/skills/codebase-index/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/.opencode/skills/codebase-index/scripts/cbx b/.opencode/skills/codebase-index/scripts/cbx index 5666358..94a9114 100644 --- a/.opencode/skills/codebase-index/scripts/cbx +++ b/.opencode/skills/codebase-index/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/.opencode/skills/codebase-index/scripts/cbx.ps1 b/.opencode/skills/codebase-index/scripts/cbx.ps1 index bb8e05d..eee56ae 100644 --- a/.opencode/skills/codebase-index/scripts/cbx.ps1 +++ b/.opencode/skills/codebase-index/scripts/cbx.ps1 @@ -8,7 +8,10 @@ param( ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @( + "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", + "path", "describe", "graph", "stats", "doctor", "update", "index" +) if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f1446e..a8e224b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,32 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +## [1.7.0] - 2026-07-29 + +### Added + +- **Diff-aware blast-radius analysis.** `codebase-index diff-impact` and the MCP + `impact_of_diff` tool aggregate graph impact for tracked working-tree changes + relative to a verified Git commit, with a changed-file safety cap, unresolved + file reporting, freshness, graph coverage, edge confidence, and exclusion of + the tool's own derived cache. +- **Progressive skill references.** The installed agent skill now keeps its + always-loaded evidence protocol compact and moves detailed commands and + response handling into on-demand reference files. +- **Reproducible product identity.** Added a graph-route mark and redesigned + README/social assets around Find / Trace / Predict. + +### Changed + +- Reworked the README, roadmap, product brief, package metadata, and plugin copy + around the evidence-first product promise. +- Expanded the safe `cbx` wrappers to allow the shipped read-only + `architecture`, `path`, `describe`, and `diff-impact` commands. +- Replaced broad skill shell permissions with an explicit command allowlist so + `clean`, `init`, and `watch` cannot bypass the safe wrapper. +- Hardened the release workflow with least-privilege defaults and immutable + commit pins for every GitHub Action that receives release or PyPI credentials. + ## [1.6.0] - 2026-06-24 ### Added @@ -375,7 +401,9 @@ All notable changes to this project are documented here. The format is based on - Hooks example + `watch` mode for keeping the index fresh without blocking the edit loop (M8). - `doctor`, `stats`, `clean` diagnostics/maintenance commands. -[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.5.0...HEAD +[Unreleased]: https://github.com/denfry/codebase-index/compare/v1.7.0...HEAD +[1.7.0]: https://github.com/denfry/codebase-index/compare/v1.6.0...v1.7.0 +[1.6.0]: https://github.com/denfry/codebase-index/compare/v1.5.0...v1.6.0 [1.5.0]: https://github.com/denfry/codebase-index/compare/v1.4.0...v1.5.0 [1.4.0]: https://github.com/denfry/codebase-index/compare/v1.3.0...v1.4.0 [1.3.0]: https://github.com/denfry/codebase-index/compare/v1.2.1...v1.3.0 diff --git a/README.md b/README.md index 630b3b7..9f31a88 100644 --- a/README.md +++ b/README.md @@ -1,707 +1,322 @@ -# codebase-index: Local Codebase Indexing for AI Coding Agents - -`codebase-index` is a local-first codebase indexing tool that helps Claude Code, -Codex CLI, OpenCode, and other AI coding agents find relevant files, symbols, and -references without scanning an entire repository. - -[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Python](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/) -[![CI](https://github.com/denfry/codebase-index/actions/workflows/ci.yml/badge.svg)](https://github.com/denfry/codebase-index/actions) -[![Claude Code Skill](https://img.shields.io/badge/Claude%20Code%20Skill-yes-green.svg)](skill/SKILL.md) -[![Codex CLI](https://img.shields.io/badge/Codex%20CLI-supported-green.svg)](#which-ai-clis-does-codebase-index-support) -[![OpenCode](https://img.shields.io/badge/OpenCode-supported-green.svg)](#which-ai-clis-does-codebase-index-support) -[![MCP](https://img.shields.io/badge/MCP-stdio%20server-green.svg)](docs/MCP.md) -[![Local First](https://img.shields.io/badge/local--first-yes-green.svg)](#safety-and-privacy) -[![No Telemetry](https://img.shields.io/badge/no%20telemetry-yes-green.svg)](#safety-and-privacy) -[![No Network By Default](https://img.shields.io/badge/no%20network%20by%20default-yes-green.svg)](#safety-and-privacy) -[![SQLite](https://img.shields.io/badge/database-SQLite-blue.svg)](docs/DATABASE_SCHEMA.md) -[![Tree-sitter](https://img.shields.io/badge/parsing-Tree--sitter-orange.svg)](docs/ARCHITECTURE.md) -

- codebase-index ranking a local search for 'where is user authentication implemented?' into scored files with recommended file:line ranges to read + codebase-index logo

-## What Is codebase-index? - -**codebase-index is a private, offline retrieval layer for AI code search.** It -builds a SQLite index of your repository, extracts symbols with Tree-sitter, -ranks matches with hybrid retrieval, and returns compact file:line ranges that -an AI coding agent can read instead of opening broad file sets. - -Use it when you want Cursor-like codebase awareness in terminal-based AI tools -while keeping source code, snippets, and search metadata on your machine. - -> **codebase-index is not an IDE and not a coding agent.** It is the local -> retrieval/index layer that gives terminal and MCP-based AI agents precise -> codebase context. The agent stays your interface; this gives it better aim. - -## Who Is It For? - -- **Claude Code / Codex CLI / OpenCode users** on medium-to-large repos who want - the agent to read 3 ranked files instead of grepping and scanning 60. -- **Privacy-constrained teams** (proprietary or regulated code) who cannot send - source to a cloud code-intelligence service. -- **MCP power users** who want a stable, queryable code index as a tool, not a - black box baked into one agent's prompt. -- **Tooling authors** who need scriptable retrieval (`--json`, SQLite, MCP) that - other tools can build on. - -Not for you if you want a full IDE, org-scale multi-repo search, or a hosted -platform — use Cursor or Sourcegraph for those. - -## Start Here - -If you are opening this repository for the first time, follow this order: - -1. [Quick Start (5 minutes)](docs/QUICKSTART.md) -2. [Installation Guide](docs/INSTALLATION.md) -3. [Benchmarks](docs/BENCHMARKS.md) -4. [How the skill works](skill/SKILL.md) -5. [MCP server](docs/MCP.md) -6. [FAQ](docs/FAQ.md) - -If you only need the shortest path, run: - -```bash -pip install codebase-index # from PyPI -cd your-project -codebase-index init # prompts for Claude Code / Codex CLI / OpenCode -codebase-index index -codebase-index search "where is authentication implemented?" -``` - -## Project Status - -**`1.6.0` is released.** The current release includes repository discovery, -SQLite FTS5 storage, Tree-sitter symbols and references, hybrid ranking, graph -impact analysis, token-budgeted retrieval packets, optional local embeddings, -hooks/watch support, multi-CLI installation, MCP server support, and PyPI + -`pipx` install paths (`pip install codebase-index`). - -The `1.6.0` release turns the dependency graph into a navigable map: every edge -carries a `confidence` audit trail (`extracted`/`inferred`/`ambiguous`, surfaced -in `refs`/`impact`); a new zero-dependency analytics pass computes modules -(communities), god nodes, and surprising cross-module links, exposed via the -`architecture` command/MCP tool; `path` traces the shortest dependency chain -between two symbols and `describe` prints a symbol's node card; and the HTML -graph is coloured by module and sized by connectivity, with `--format -graphml|dot|neo4j` exports for external tools. Requires a one-time reindex -(schema 2 → 3). - -The earlier `1.4.0` release hardened the MCP contract (a `schema_version` + -`tool` envelope on every payload, golden-locked per tool, plus a fix so the -server loads on current `mcp`/`pydantic`), dampened the god-class `in_degree` -rerank tiebreak (logarithmic, validated no-regression on the public benchmark), -and labelled config/IaC files (Dockerfile, Terraform, HCL, INI, Makefiles) so -infra surfaces in `stats` and search. - -The earlier `1.3.0` release added a content-addressed embedding cache (rebuilds reuse -vectors for unchanged content), a batched graph build (7–28× faster edge -resolution plus a new `edges(file_id)` index), a shared CLI/MCP service layer -(MCP hybrid search now uses the vector channel; `index_stats` reports the -per-language graph tier), graph-coverage signals in `stats`/`refs`/`impact`, -CLI pagination via `search --offset`, and single-source versioning with a CI -gate that keeps every committed skill copy in sync. -The `1.2.1` release added skill auto-update/rollback commands and version -stamps so installed skills stay in sync with the package automatically. -See [CHANGELOG.md](CHANGELOG.md) and -[docs/ROADMAP.md](docs/ROADMAP.md). - -MCP is now available as a stdio server via `codebase-index mcp --root `. -It exposes `healthcheck`, `search_code`, `find_symbol`, `find_refs`, -`impact_of`, `explain_code`, `architecture_overview`, `path_between`, -`describe_symbol`, and `index_stats`; see [docs/MCP.md](docs/MCP.md). - -``` -You: "Where is user authentication implemented?" -Agent: searches local index (symbols + FTS5 + graph) - reads only 3 ranked files instead of scanning 60 - answers with citations: src/auth/AuthService.ts:12-148 -``` - ---- - -## How Do I Install codebase-index? - -For most users, install the package from PyPI and run `init` inside the -repository you want to index: - -```bash -pip install codebase-index # or: pipx install codebase-index -cd your-project -codebase-index init # choose Claude Code, Codex CLI, OpenCode, or all -codebase-index index -``` - -In a non-interactive script, pass a target explicitly: - -```bash -codebase-index init --target auto # install into detected AI CLIs -codebase-index init --target codex # write AGENTS.md + Codex resources -codebase-index init --target claude # write .claude/skills/codebase-index -codebase-index init --target opencode # write OpenCode command + agent files -``` - -### Install as a Claude Code plugin - -One command in Claude Code: - -``` -/plugin marketplace add denfry/codebase-index -/plugin install codebase-index@codebase-index -``` - -Or just ask: "install the codebase-index plugin". - -**What happens on first run:** when a session starts, a `SessionStart` hook -(`scripts/bootstrap.sh` / `.ps1`) creates a private Python virtual environment under -`~/.claude/plugins/data/codebase-index-*/venv` and installs the pinned -`codebase-index` package (from `requirements.lock`) into it — using `uv` if present, -otherwise `python -m venv` + `pip`. It reinstalls only when the lock file changes. -Nothing is installed globally; uninstalling the plugin removes the data directory. - -**Prerequisite:** Python 3.11+ on your PATH. The first install needs network access to -fetch the package; later sessions are offline. The skill builds its index on -your first codebase question, so there is no manual `index` step. - -**Distribution note:** the plugin bootstrap installs the pinned requirement from -`requirements.lock`. In `1.6.0`, that lock points at the tagged GitHub release -instead of PyPI. You can override it with `CBX_INSTALL_SPEC` when testing a local -checkout or a different Git ref. - -## What Problem Does codebase-index Solve? - -AI coding agents struggle with large repositories when they rely on broad file -reads, grep output, or user-provided context. `codebase-index` gives those agents -a ranked local retrieval packet before they read source files. - -- **Token waste** — Scanning entire files or running broad grep/glob queries burns through the context window on irrelevant content. -- **No symbol awareness** — Standard search can't distinguish a function definition from a call, or a class from a variable. -- **No ranking** — Grep returns all matches with no relevance ordering. The agent must read everything. -- **No context** — Grep doesn't know which files are related or what to read next. -- **Cloud dependency** — External code indexing services send your proprietary code to remote servers. - -Developers get Cursor-like codebase awareness in Claude Code, Codex CLI, and -OpenCode without leaving the terminal or sending code to a remote indexing -service. - -## How Is This Different? - -Short answers to the questions people actually ask. The full, honest matrix — -including when you should pick the other tool — is in -[docs/COMPARISON.md](docs/COMPARISON.md). - -- **Why not just `grep`/`rg`?** Grep returns every match with no ranking, no - symbol awareness, and no idea which files relate. codebase-index ranks results, - knows a definition from a call, expands along the dependency graph, and returns - specific line ranges under a token budget — so the agent reads less and answers - with citations. -- **Why not Cursor?** Cursor is a great AI IDE with strong codebase awareness, but - it is proprietary and IDE-centric. codebase-index is a local, open retrieval - layer for **terminal and MCP** agents, offline by default, with no IDE lock-in. - If you live inside Cursor, keep using Cursor. -- **Why not Aider repo-map?** Aider's repo-map is a good graph-ranked, - token-budgeted context map — but it is optimized to feed Aider's own chat. - codebase-index is a **reusable, queryable index**: CLI/JSON/MCP commands return - ranked `file:line` ranges, symbols, references, and impact that *any* - shell-capable agent can consume, with freshness and security gates. -- **Why not Sourcegraph / Cody / Amp?** They are excellent enterprise-grade, - cross-repo code intelligence platforms. They are also heavier and - account/platform-oriented. codebase-index is single-repo, local, and - lightweight — no server, no account, no code leaving the machine by default. -- **Why not Codebase-Memory MCP?** It is the closest direct alternative — a - broader graph engine with a static binary and wide language/agent coverage. We - do **not** claim to beat it globally. We differentiate on simplicity, a strict - privacy model, token-budgeted retrieval packets, a transparent Python - implementation, the Claude/Codex/OpenCode workflow, and honest benchmarks. If - you need its broader graph and language reach today, choose it. - -**What makes it trustworthy?** No telemetry, no network by default, a multi-gate -exclusion pipeline (secrets/binaries/generated/dependencies never indexed), -output-time secret redaction, a `doctor --strict` safety self-check, and a -public benchmark suite wired as a CI regression gate. Claims that aren't proven -in this repo are marked as roadmap, not done. - -### Proven today vs. roadmap - -| Capability | Status | -|---|---| -| Hybrid retrieval (path + symbol + FTS5 + graph), token-budgeted packets | ✅ Shipped | -| Tree-sitter symbols for 12 Tier-A languages + Tier-B generic path | ✅ Shipped | -| Import/call/reference/inheritance graph, `refs`/`impact` | ✅ Shipped | -| Optional local embeddings; external embeddings gated 3 ways | ✅ Shipped | -| stdio MCP server; CLI/skill/MCP share one service layer | ✅ Shipped | -| Honest 55k LOC Java benchmark (recall@3 70% vs 40% `rg`, ~13× fewer tokens) | ✅ Shipped | -| 10k/100k/1M LOC public-repo benchmarks | 🚧 Roadmap | -| Framework-aware typed edges (route→handler→service→model) | 🚧 Roadmap | -| PyPI / `uvx` / Homebrew, signed checksums, SBOM | 🚧 Roadmap | -| Verified per-client MCP docs, paged/progressive results | 🚧 Roadmap | - -See [docs/PRODUCT_UPGRADE_PLAN.md](docs/PRODUCT_UPGRADE_PLAN.md) for the full -upgrade plan and ranked roadmap. +

codebase-index

-## How Does codebase-index Work? - -`codebase-index` builds a local hybrid index that combines: - -- **Symbol search** — Tree-sitter AST parsing extracts classes, functions, methods, and variables across the supported code-language set. -- **Full-text search** — SQLite FTS5 for fast lexical search across code chunks. -- **Path search** — File path matching for location-aware queries. -- **Optional semantic search** — Vector embeddings for similarity-based retrieval (opt-in, local by default). -- **Dependency graph** — Import, call, and reference edges for impact analysis and graph expansion. -- **Token-budgeted output** — Ranked retrieval packets with specific line ranges, not whole files. - -The AI agent reads only the recommended files and line ranges, not the entire -repository. - -## Quick Demo - -```bash -/codebase-index "where is user authentication implemented?" -``` - -Expected output: - -``` -Top matches: -┌──────┬──────────────────────────┬──────────────────────────┬───────┬──────────────────────────────┐ -│ Rank │ Path │ Symbols │ Score │ Reason │ -├──────┼──────────────────────────┼──────────────────────────┼───────┼──────────────────────────────┤ -│ 1 │ src/auth/AuthService.ts │ AuthService, login │ 0.92 │ exact symbol match │ -│ 2 │ src/routes/auth.ts │ loginHandler, logout │ 0.78 │ FTS match · 4 callers │ -│ 3 │ src/middleware/auth.ts │ requireAuth │ 0.65 │ path match · FTS match │ -└──────┴──────────────────────────┴──────────────────────────┴───────┴──────────────────────────────┘ - -Recommended reads: - 1. src/auth/AuthService.ts:12-148 - reason: matched AuthService, login(), validatePassword() - 2. src/routes/auth.ts:20-91 - reason: /login route calls AuthService.login() - 3. src/middleware/auth.ts:5-42 - reason: auth middleware validates sessions -``` - -## Installation Options - -If you are new to this repo, start with [docs/QUICKSTART.md](docs/QUICKSTART.md). -If you want all install options and troubleshooting, use [docs/INSTALLATION.md](docs/INSTALLATION.md). - -**Multi-CLI installer (Claude Code + Codex CLI + OpenCode):** one command via -`install.sh` / `install.ps1` — see [docs/installer.md](docs/installer.md). - -```bash -# macOS / Linux -curl -fsSL https://raw.githubusercontent.com/denfry/codebase-index/main/install.sh | sh -``` -```powershell -# Windows PowerShell -irm https://raw.githubusercontent.com/denfry/codebase-index/main/install.ps1 | iex -``` - -### Option 1: Install from PyPI (recommended) - -```bash -pip install codebase-index # or: pipx install codebase-index -cd your-project -codebase-index init -codebase-index index -``` +

+ Give AI coding agents a precise map of your codebase — locally, + privately, and with evidence. +

-### Option 2: Pin to a tagged GitHub release +

+ Find implementations. Trace behavior. Predict change impact. +

-Pin to an exact version for reproducible installs, or grab an unreleased commit: +
-```bash -cd your-project -pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.6.0" -codebase-index init -codebase-index index -``` +

+ PyPI version + CI + Python 3.11+ + MCP ready + No network by default + MIT license +

-### Python version compatibility +

+ codebase-index showing a Find, Trace, Predict workflow with precise file and line evidence +

-`codebase-index` requires Python 3.11 or newer. +## The short version -If `codebase-index init --target opencode` fails with: +`codebase-index` is a local retrieval and code-graph layer for Claude Code, +Codex CLI, OpenCode, and MCP clients. It indexes a repository into SQLite, +extracts symbols and relationships with Tree-sitter, and gives agents ranked +`file:line` evidence instead of making them scan broad sets of files. ```text -ModuleNotFoundError: No module named 'importlib.resources.abc'; 'importlib.resources' is not a package +Question → ranked retrieval → dependency evidence → precise answer ``` -the `pipx` environment was likely created with an older Python version. Reinstall `codebase-index` using Python 3.11+ explicitly: +It is not an IDE and not another coding agent. Your existing agent remains the +interface; `codebase-index` gives it better aim. -```powershell -pipx uninstall codebase-index -py -0p -pipx install --python "\python.exe" codebase-index -``` +## Find. Trace. Predict. -For example: - -```powershell -pipx install --python "C:\Users\you\AppData\Local\Programs\Python\Python312\python.exe" codebase-index -``` +| Job | Question | Command | +|---|---|---| +| **Find** | Where is authentication implemented? | `codebase-index search "authentication"` | +| **Trace** | How does checkout reach the database? | `codebase-index explain "checkout flow"` | +| **Trace** | How are two components connected? | `codebase-index path ApiController Database` | +| **Predict** | What breaks if `User` changes? | `codebase-index impact User` | +| **Predict** | What does my current diff affect? | `codebase-index diff-impact` | -Then run initialization again: +Every retrieval packet carries: -```powershell -codebase-index init --target opencode -codebase-index index -``` +- ranked matches and the reason each match scored; +- exact line ranges to read next; +- index freshness; +- answer confidence and targeted fallbacks; +- graph coverage and edge confidence where relevant. +That evidence contract lets an agent distinguish “nothing references this” from +“the graph is partial, so verify with a targeted search.” -### Option 3: Install with pipx from a pinned GitHub tag +## Install in five minutes ```bash -pipx install "git+https://github.com/denfry/codebase-index.git@v1.6.0" +pip install codebase-index cd your-project -codebase-index init --target auto -codebase-index index -``` - -### Option 4: Install from source - -```bash -git clone https://github.com/denfry/codebase-index.git -cd codebase-index -pip install -e ".[dev]" -``` - -### Distribution roadmap - -As of `1.6.0`, **PyPI is shipped** — `pip install codebase-index` and -`pipx install codebase-index` work today. `uvx`, Homebrew, signed release -checksums, and SBOMs remain on the roadmap: - -```bash -uvx codebase-index init # planned -brew install denfry/tap/codebase-index # planned -``` - -### Verify the install - -```bash -codebase-index doctor -``` - -See [docs/INSTALLATION.md](docs/INSTALLATION.md) for the full guide, including optional extras (embeddings, watch mode) and troubleshooting. - -## Usage - -```bash -# Initialize the index for your project codebase-index init - -# Build the index codebase-index index - -# Search for something codebase-index search "where is authentication implemented?" - -# Look up a specific symbol -codebase-index symbol "AuthService" - -# Find callers and references -codebase-index refs "AuthService.login" - -# Analyze impact of a change -codebase-index impact "src/auth/AuthService.ts" - -# Map the codebase: modules, god nodes, surprising links, suggested questions -codebase-index architecture - -# How are two symbols/files connected? Shortest dependency/call path -codebase-index path "renew" "refresh_access_token" - -# Node card: definition, callers, callees, centrality, module -codebase-index describe "Database" - -# Visualize the graph (modules coloured, size = connectivity, edge style = confidence) -codebase-index graph --open -# …or export for external tools: graphml (Gephi/yEd), dot (Graphviz), neo4j (Cypher) -codebase-index graph --format graphml -o graph.graphml - -# View index statistics -codebase-index stats - -# Run diagnostics -codebase-index doctor ``` -Add `--json` to any command for machine-readable output. - -## How Does Retrieval Flow Through codebase-index? - -``` -User question - ↓ -CLI instructions or skill - ↓ -Hybrid retrieval - ├─ Path search - ├─ Symbol search (Tree-sitter AST) - ├─ SQLite FTS5 full-text search - ├─ Optional embeddings (vector search) - └─ Graph expansion (callers, imports, references) - ↓ -Ranked retrieval packet - ↓ -Agent reads only the recommended line ranges - ↓ -Answer with precise file:line citations -``` - -## Features - -- [x] **Local-first indexing** — All data stays on your machine -- [x] **No network by default** — Zero external API calls out of the box -- [x] **Respects ignore files** — `.gitignore`, `.claudeignore`, `.codeindexignore` -- [x] **SQLite storage** — Fast, reliable, single-file database -- [x] **FTS5 lexical search** — Full-text search with code-aware tokenization -- [x] **Tree-sitter AST parsing** — Tier-A symbol extraction for Python, JavaScript, TypeScript, Java, Go, Rust, C, C++, C#, Ruby, PHP, and Kotlin; Tier-B generic extraction for code languages with a loadable grammar such as Lua -- [x] **Symbol extraction** — Classes, functions, methods, variables with line ranges -- [x] **Incremental indexing** — Only changed files are re-indexed -- [x] **Token-budgeted output** — Configurable max output size -- [x] **Secret redaction** — Masks keys, tokens, and credentials in snippets -- [x] **Optional embeddings** — Local or remote vector search (opt-in) -- [x] **Optional hooks/watch** — Auto-update index after file edits -- [x] **Multi-CLI setup** — Claude Code, Codex CLI, and OpenCode instructions -- [x] **MCP server** — stdio MCP tools for search, symbols, refs, impact, explain, health, and stats - -## Safety and Privacy - -> **Trust model in 60 seconds** -> 1. **Offline by default** — the base install has zero network dependencies; nothing leaves your machine. -> 2. **One opt-in exit, triple-gated** — external embeddings require `allow_external` **and** an env API key **and** a printed endpoint warning, or they are refused. -> 3. **Secrets never get in** — `.env`, keys, certs, and credential files are excluded before parsing (multi-gate ignore pipeline). -> 4. **Secrets never get out** — every snippet is redacted (AWS keys, private keys, JWTs, bearer tokens, connection strings) before it reaches the agent. -> 5. **No telemetry, ever** — no analytics, no phone-home, no usage data. -> 6. **Verify it yourself** — `codebase-index doctor --strict` audits all of the above and exits non-zero in CI on any high-severity finding. - -`codebase-index` is designed with privacy as a first principle: - -- **No telemetry** — No usage data, analytics, or crash reports are collected or transmitted. -- **No external API calls by default** — All indexing, storage, and search happen locally. -- **Does not index sensitive files** — `.env`, private keys, certificates, tokens, and credential files are excluded before parsing. -- **Respects ignore files** — `.gitignore`, `.claudeignore`, `.codeindexignore`, and `.cursorignore` are all honored. -- **Index stored locally** — SQLite database in `.claude/cache/codebase-index/` (gitignored by default). -- **Optional embeddings are local by default** — External embedding APIs require explicit opt-in with warnings. -- **Secret redaction** — Snippets are scrubbed for AWS keys, private keys, JWTs, bearer tokens, and connection strings before output. - -See [docs/SECURITY_MODEL.md](docs/SECURITY_MODEL.md) for the full security model and threat analysis. - - -## Benchmark Results - -There are three benchmark surfaces today: - -1. **Public benchmark suite** in `tests/benchmark_public.py`: reproducible - multi-language fixture with Recall@1/3/5, MRR, nDCG, answer-correctness proxy, - token economy, language breakdown, freshness latency, graph tasks, and scale counters. -2. **Smoke benchmark** on `sample_repo`: validates the CLI is fast and stable on - a tiny fixture, but it is not evidence of production retrieval quality. -3. **Honest benchmark** on a real Java repository: `tests/benchmark_honest.py` - compares codebase-index against a disciplined `rg` + read-window baseline on - 10 realistic questions. Results are documented in - [tests/benchmark_honest_RESULTS.md](tests/benchmark_honest_RESULTS.md). - -Run the public suite: +`init` can install resources for Claude Code, Codex CLI, OpenCode, and detected +MCP clients: ```bash -python tests/benchmark_public.py --workdir .tmp-public-benchmark +codebase-index init --target auto +codebase-index init --target codex +codebase-index init --target claude +codebase-index init --target opencode ``` -Current honest benchmark headline: - -| Metric | Result | -|---|---| -| Repo | 303 Java files, ~55k LOC | -| Retrieval quality | recall@3: 70% index vs 40% `rg` baseline | -| Token economy | ~13x fewer answer tokens than `rg` + 80-line windows | -| Verified language impact | Java symbols fixed from 0 to 3,543 symbols | - -The public suite now has the metric framework. It still needs larger public or -documented external repos for 10k/100k/1M LOC scale claims and deeper framework -graph tasks. See [docs/BENCHMARKS.md](docs/BENCHMARKS.md). +`pipx install codebase-index` is supported as an isolated alternative. See the +[installation guide](docs/INSTALLATION.md) for pinned releases, editable +installs, hooks, Windows details, and troubleshooting. -## Repository Layout +### Claude Code plugin -``` -├── skill/ # Source instruction package (SKILL.md, scripts, examples) -├── skills/ # Plugin skill copy -├── src/codebase_index/ # Python package (CLI, indexer, retrieval, storage) -├── docs/ # Documentation (architecture, schema, security, FAQ) -├── examples/ # Sample queries, retrieval output, demo project -├── tests/ # Test suite with fixture repositories -├── bin/ # Plugin CLI wrappers (cbx, codebase-index) -├── scripts/ # Bootstrap scripts (bootstrap.sh, bootstrap.ps1) -├── hooks/ # Plugin hooks (hooks.json) -├── .claude-plugin/ # Plugin manifest + marketplace catalog -├── .github/ # Issue templates, CI workflows, PR template -├── README.md # This file -├── LICENSE # MIT License -├── CHANGELOG.md # Release history -├── CONTRIBUTING.md # Contributor guide -├── SECURITY.md # Security policy -├── ROADMAP.md # Development milestones -├── requirements.lock # Pinned install spec for bootstrap -└── pyproject.toml # Package configuration +```text +/plugin marketplace add denfry/codebase-index +/plugin install codebase-index@codebase-index ``` -## Configuration +The plugin provisions a private environment on first use. Later sessions run +offline, and the first codebase question builds the index automatically. -Create `.codeindex.json` in your project root: +## What the agent receives ```json { + "query": "where is authentication implemented?", + "confidence": "high", + "results": [ + { + "path": "src/auth/AuthService.ts", + "line_start": 12, + "line_end": 148, + "score": 0.92, + "reason": "exact symbol match, 4 callers" + } + ], + "recommended_reads": [ + { + "path": "src/auth/AuthService.ts", + "line_start": 12, + "line_end": 148 + } + ], "index": { - "max_file_bytes": 1048576, - "chunk_size": 500, - "chunk_overlap": 50 - }, - "embeddings": { - "backend": "noop", - "allow_external": false + "exists": true, + "stale": false } } ``` -### Ignore Files - -- `.codeindexignore` — Tool-specific ignore patterns (highest priority) -- `.gitignore` — Standard git ignore patterns -- `.claudeignore` — Claude-specific ignore patterns - -### Cache Location - -``` -.claude/cache/codebase-index/ -├── index.sqlite # SQLite database with FTS5 -└── config.json # Resolved configuration -``` +Snippets are skeletonized when that preserves evidence while saving tokens. +Unrelated bodies collapse, but imports, signatures, matched lines, and exact +read ranges remain. -## Which AI CLIs Does codebase-index Support? +## Core commands -`codebase-index init` can install instructions for three AI coding CLIs: +```bash +# Find +codebase-index search "auth token refresh" +codebase-index symbol AuthService + +# Trace +codebase-index explain "authentication flow" +codebase-index refs send_email +codebase-index path ApiController Database +codebase-index describe Database +codebase-index architecture -| CLI | Files written by `init` | Best command | -|---|---|---| -| Claude Code | `.claude/skills/codebase-index/` | `codebase-index init --target claude` | -| Codex CLI | `AGENTS.md` + `.codex/skills/codebase-index/` | `codebase-index init --target codex` | -| OpenCode | `.opencode/commands/` + `.opencode/agents/` + resources | `codebase-index init --target opencode` | +# Predict +codebase-index impact User --direction up --depth 2 +codebase-index diff-impact --base HEAD --direction up --depth 2 -Use `codebase-index init --target auto` to install into detected CLIs, or -`codebase-index init --target all` to write every supported integration. +# Inspect and visualize +codebase-index graph User --direction both --depth 2 --output graph.html +codebase-index stats +codebase-index doctor +``` -### Claude Code Integration +Add `--json` for agents and automation. Search supports `hybrid`, `fts`, +`symbol`, and opt-in `vector` modes. -The Claude Code skill is defined in [`skill/SKILL.md`](skill/SKILL.md) with -YAML frontmatter for automatic selection. +## Why not just grep? -Example `.claude/CLAUDE.md`: +Grep is excellent when you know the exact text. Repository questions often +need more: -```markdown -## Codebase Questions +| Capability | `rg` / grep | codebase-index | +|---|---:|---:| +| Exact text matching | Yes | Yes | +| Ranked results | No | Yes | +| Symbol definitions vs calls | No | Yes | +| Dependency and impact graph | No | Yes | +| Token-budgeted read plan | No | Yes | +| Freshness and coverage signals | No | Yes | +| Local and scriptable | Yes | Yes | -Before answering any question about this project's code: -1. Use the codebase-index skill to search the local index first. -2. Read only the recommended line ranges — do not scan entire files. -3. Answer with file:line citations. -``` +Use grep for one known string. Use `codebase-index` when the agent must locate, +understand, or assess a change across a repository. -### Optional Hooks +The [comparison guide](docs/COMPARISON.md) also covers Cursor, Aider repo-map, +Sourcegraph, Continue, Amp, and Codebase-Memory MCP—including when those tools +are the better choice. -Configure automatic index updates in `.codeindex.json`: +## Measured results -```json -{ - "hooks": { - "post_tool_use": { - "enabled": true, - "events": ["Write", "Edit"], - "command": "codebase-index update --quiet" - } - } -} -``` +On the published 55k LOC Java benchmark: -See [skill/examples/](skill/examples/) for full examples. +- Recall@3: **70%** for `codebase-index` versus **40%** for the `rg` baseline; +- answer-context tokens: approximately **13× fewer**; +- raw results and methodology are checked into the repository. -## FAQ +These results are evidence for that benchmark, not a claim of universal +superiority. Large public-repository and framework-graph evaluations remain on +the [roadmap](docs/ROADMAP.md). Read the complete methodology and limitations in +[BENCHMARKS.md](docs/BENCHMARKS.md). -### Is this a Cursor replacement? +## Local by default -No. `codebase-index` is not a replacement for Cursor or any IDE. It is a -local retrieval layer for terminal AI coding agents. You still use Claude Code, -Codex CLI, OpenCode, or another agent as your primary interface. +The base install: -### Does it send my code anywhere? +- makes no network requests; +- sends no telemetry; +- stores the derived index inside the project cache; +- excludes dependency, build, binary, oversized, generated, and secret-like + files before indexing; +- redacts secret patterns again at output time; +- exposes `doctor --strict` for CI and security checks. -No. By default, `codebase-index` is completely local-first and offline. All indexing, storage, and search happen on your machine. External embeddings are opt-in only and require explicit configuration. +Embeddings are optional. Local embeddings stay on the machine; external +embeddings require explicit configuration, an API key, and an endpoint +acknowledgement. -### Does it work without embeddings? +See the [security model](docs/SECURITY_MODEL.md) for trust boundaries, gates, +failure modes, and residual risks. -Yes. The default configuration disables embeddings entirely (`backend = "noop"`). Search uses SQLite FTS5, Tree-sitter symbol extraction, path matching, and graph expansion. Embeddings are an optional enhancement. +## How it works -### Does it support large repositories? +```text +Repository + │ + ├─ discovery + ignore and secret gates + ├─ Tree-sitter symbols and relationships + ├─ line and symbol-aligned chunks + └─ optional embeddings + │ + ▼ + local SQLite + FTS5 + symbols + graph + │ + ▼ + intent routing → hybrid retrieval → rerank → token budget + │ + ▼ + ranked file:line evidence for CLI, Skill, and MCP +``` + +The three product surfaces share one service layer, so retrieval behavior does +not drift between the CLI, installed agent skills, and MCP tools. + +Detailed internals: + +- [Architecture](docs/ARCHITECTURE.md) +- [Retrieval pipeline](docs/RETRIEVAL_PIPELINE.md) +- [Database schema](docs/DATABASE_SCHEMA.md) +- [Language and graph coverage](docs/LANGUAGES.md) + +## Supported surfaces + +| Surface | Integration | +|---|---| +| Claude Code | Skill, plugin, optional hooks | +| Codex CLI | `AGENTS.md` plus project skill | +| OpenCode | Command, agent, and skill resources | +| MCP clients | stdio server with versioned JSON envelopes | +| Shell and automation | CLI, `--json`, and local SQLite | -Yes. The index is incremental — only changed files are re-indexed. SQLite with FTS5 handles large datasets efficiently. Generated files, dependencies, and binaries are excluded automatically. +Run the MCP server with: -### Why not just use Grep? +```bash +codebase-index mcp --root /path/to/repository +``` -Grep returns all matches with no ranking, no symbol awareness, and no context about related files. `codebase-index` combines lexical search with symbol extraction and graph expansion to return **ranked, contextual results** with specific line ranges to read. +Available MCP tools include search, explain, symbols, references, impact, +diff impact, architecture, shortest path, node description, health, and index statistics. +See [MCP.md](docs/MCP.md) for client configuration. -### Does it support MCP? +## Project status -Yes. Run `codebase-index mcp --root ` to expose the local index over stdio -MCP. See [docs/MCP.md](docs/MCP.md) for tools and client config templates. +The latest released line is **1.7.0**. It includes: -### Can I use it with other agents? +- hybrid and optional vector retrieval; +- Tree-sitter symbol extraction across the documented language tiers; +- import, call, reference, and inheritance graphs; +- architecture communities, central nodes, and surprising cross-module links; +- shortest dependency paths and node descriptions; +- token-budgeted and skeletonized retrieval packets; +- CLI, Skill, plugin, and MCP delivery; +- incremental updates, watch hooks, diagnostics, skill rollback, and diff-aware + impact analysis. -Yes. The CLI is agent-agnostic. Any agent that can run shell commands can use -`codebase-index`, and JSON output (`--json`) is parseable by other tools. +Planned work is deliberately separated from shipped capability. The next +product priorities are stronger real-repository evaluations, typed framework +edges, and an even more direct task-context workflow. See the +[roadmap](docs/ROADMAP.md). -### How do I reset the index? +## Documentation -```bash -codebase-index clean # reset the index DB (keeps the skill) -codebase-index clean --all # wipe the whole .claude/cache/codebase-index/ dir -# Or manually: rm -rf .claude/cache/codebase-index/ -codebase-index index -``` +| Start here | Deep dives | Project trust | +|---|---|---| +| [Quick start](docs/QUICKSTART.md) | [Retrieval](docs/RETRIEVAL.md) | [Benchmarks](docs/BENCHMARKS.md) | +| [Installation](docs/INSTALLATION.md) | [Architecture](docs/ARCHITECTURE.md) | [Security](docs/SECURITY.md) | +| [FAQ](docs/FAQ.md) | [MCP](docs/MCP.md) | [Release checklist](docs/RELEASE_CHECKLIST.md) | +| [Skill design](docs/SKILL_DESIGN.md) | [Schema](docs/SCHEMA.md) | [Changelog](CHANGELOG.md) | ## Contributing -We welcome contributions! See [CONTRIBUTING.md](CONTRIBUTING.md) for the full guide. +Contributions should preserve three invariants: + +1. retrieval quality is measured, not asserted; +2. the default path remains local and fails closed at security boundaries; +3. machine-readable contracts stay stable across CLI and MCP. -Quick start: +Before opening a pull request: ```bash -git clone https://github.com/denfry/codebase-index.git -cd codebase-index -pip install -e ".[dev]" pytest -ruff check src/ tests/ +ruff check . +mypy src +python scripts/sync_skill_copies.py --check ``` -## Roadmap - -See [ROADMAP.md](ROADMAP.md) for the full milestone plan. - -| Milestone | Status | Description | -|---|---|---| -| M0 | ✅ Done | Repository packaging | -| M1 | ✅ Done | SQLite + FTS5 index | -| M2 | ✅ Done | Tree-sitter symbol extraction | -| M3 | ✅ Done | Hybrid retrieval | -| M4 | ✅ Done | Graph expansion | -| M5 | ✅ Done | Token-budgeted retrieval packets | -| M6 | ✅ Done | Optional local embeddings | -| M7 | ✅ Done | Claude Code Skill packaging | -| M7.5 | ✅ Done | One-command plugin install | -| M8 | ✅ Done | Hooks + watch mode | -| M9 | ✅ Done | Public release | +Add user-visible changes under `[Unreleased]` in [CHANGELOG.md](CHANGELOG.md). +See [CONTRIBUTING.md](CONTRIBUTING.md) if present and the repository +instructions for branch and review policy. ## License diff --git a/assets/demo.png b/assets/demo.png index 3ff97e2..a8077fd 100644 Binary files a/assets/demo.png and b/assets/demo.png differ diff --git a/assets/mark.png b/assets/mark.png new file mode 100644 index 0000000..1632fa2 Binary files /dev/null and b/assets/mark.png differ diff --git a/assets/social-preview.png b/assets/social-preview.png index 0f34eff..a24d4e6 100644 Binary files a/assets/social-preview.png and b/assets/social-preview.png differ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 32b0000..0117f29 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## 1. Overview -`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.6.0` +`codebase-index` is a **local-first** code intelligence layer for AI coding agents. In `1.7.0` it has two shipped faces: 1. **A Claude Code Skill** (`.claude/skills/codebase-index/SKILL.md`) that Claude auto-invokes for @@ -150,13 +150,14 @@ upward to nearest `.git`/`.claude`), and `--quiet`. Search-family commands accep | `symbol` | `""`, `--kind`, `--exact` | 0; empty list allowed | symbol defs | | `refs` | `""`, `--kind callers\|all` | 0 | reference sites | | `impact` | `""`, `--depth N`, `--direction up\|down\|both` | 0 | affected files ranked | +| `diff-impact` | `--base `, `--depth N`, `--direction up\|down\|both` | 0 | aggregate affected files for tracked changes | | `explain` | `""`, `--token-budget` | 0 | intent-aware bundle | | `stats` | — | 0 | counts, coverage %, freshness | | `doctor` | `--strict` | non-zero if unsafe config found | findings list | | `clean` | `--yes`, `--all` | resets index DB (`--all` wipes cache dir) | removed-count | | `watch` | `--debounce ms` | long-running | event log | -The skill only ever calls the **read-only** family (`search`, `symbol`, `refs`, `impact`, +The skill only ever calls the **read-only** family (`search`, `symbol`, `refs`, `impact`, `diff-impact`, `explain`, `stats`) plus `update`. It never calls `clean` or `init`. See SECURITY.md. ### Freshness contract @@ -191,7 +192,7 @@ If `exists=false` → skill runs `index`. If `stale=true` and cheap → skill ru ## 8. MCP server The same `retrieval` + `storage` layers are wrapped in a stdio MCP server exposing tools like -`search_code`, `find_symbol`, `find_refs`, `impact_of`, `explain_code`, `index_stats`, and +`search_code`, `find_symbol`, `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `index_stats`, and `healthcheck`. Current implementation: diff --git a/docs/FAQ.md b/docs/FAQ.md index adc17ec..7fd8bde 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -14,7 +14,7 @@ pip install codebase-index # or: pipx install codebase-index ``` To pin an exact version or grab an unreleased commit, install from a GitHub tag -instead: `pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.6.0"`. +instead: `pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.7.0"`. Then run `codebase-index init` inside your project and `codebase-index index` to build the first index. In Claude Code you can instead install the plugin @@ -81,6 +81,7 @@ The stdio MCP server exposes: - `find_symbol` - `find_refs` - `impact_of` +- `impact_of_diff` - `explain_code` - `index_stats` @@ -161,11 +162,12 @@ Yes. Use any of these methods: ## Is it production-ready? -Yes — `codebase-index` is released as **v1.6.0**. The core indexing and search -functionality is implemented and tested. The current `1.6.0` package includes: +Yes — `codebase-index` is released as **v1.7.0**. The core indexing and search +functionality is implemented and tested. The current `1.7.0` package includes: - Hybrid FTS/path/symbol/vector retrieval - Import/call/reference graph expansion and `impact` +- Diff-aware blast-radius analysis for tracked working-tree changes - Optional local embeddings, with external embeddings gated behind explicit opt-in - Hooks and watch mode for freshness - Multi-CLI setup for Claude Code, Codex CLI, and OpenCode diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index dd2b781..8f3ba48 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -67,7 +67,7 @@ pipx install codebase-index # isolated environment uv tool install codebase-index # uv-managed tool # Pin to a GitHub tag for an exact or unreleased version -pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.6.0" +pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.7.0" # From source (editable mode) git clone https://github.com/denfry/codebase-index.git @@ -90,7 +90,7 @@ pip install -e ".[embeddings-local,watch,dev]" ### uvx / Homebrew status -As of `1.6.0`, **PyPI is shipped** — `pip install codebase-index` and +As of `1.7.0`, **PyPI is shipped** — `pip install codebase-index` and `pipx install codebase-index` are the verified paths. `uvx codebase-index init`, Homebrew tap installation, signed checksums, and SBOMs remain distribution targets for a more complete release story. @@ -130,7 +130,7 @@ Expected output: === codebase-index Doctor === [OK] Python 3.12 (requires 3.11+) -[OK] codebase-index package installed (v1.6.0) +[OK] codebase-index package installed (v1.7.0) [OK] tree-sitter is available [INFO] Cache directory not yet created: ... [INFO] Skill not installed in .claude/skills/ diff --git a/docs/MCP.md b/docs/MCP.md index d0242c3..b89dd1d 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -36,6 +36,7 @@ The MCP server exposes the same retrieval contract as the CLI. | `find_symbol` | Locate definitions by symbol name/kind | `symbol` | | `find_refs` | Return callers/references for a symbol | `refs` | | `impact_of` | Return affected files/symbols from graph expansion | `impact` | +| `impact_of_diff` | Aggregate graph impact for tracked working-tree changes | `diff-impact` | | `explain_code` | Intent-aware retrieval packet for a natural-language question | `explain` | | `architecture_overview` | Modules, god nodes, surprising connections, suggested questions | `architecture` | | `path_between` | Shortest dependency/call path between two symbols or files | `path` | @@ -67,7 +68,7 @@ branch on the contract without sniffing the shape: breaking change (field removal or type change); additive fields keep the same version. The current version is **1**. - `tool` (string) — the emitting tool name (`search_code`, `find_symbol`, - `find_refs`, `impact_of`, `explain_code`, `architecture_overview`, + `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `architecture_overview`, `path_between`, `describe_symbol`, `index_stats`, `healthcheck`). - The no-index / error path carries the same envelope plus an `"error"` field. @@ -159,8 +160,9 @@ same trust boundaries: - Done: `src/codebase_index/mcp/server.py` thin adapter over retrieval/storage code. - Done: `codebase-index mcp --root ` CLI entrypoint. -- Done: `healthcheck`, `search_code`, `find_symbol`, `find_refs`, `impact_of`, `explain_code`, - `architecture_overview`, `path_between`, `describe_symbol`, and `index_stats` tools. +- Done: `healthcheck`, `search_code`, `find_symbol`, `find_refs`, `impact_of`, + `impact_of_diff`, `explain_code`, `architecture_overview`, `path_between`, + `describe_symbol`, and `index_stats` tools. - Done: focused tests for tool registration, missing-index behavior, config resolution, and run entrypoint. - Done: explicit `schema_version` + `tool` envelope on every structured tool payload (including the error path), asserted by `tests/test_mcp_server.py` and `tests/test_mcp_golden.py`. diff --git a/docs/PRODUCT_UPGRADE_PLAN.md b/docs/PRODUCT_UPGRADE_PLAN.md index c90e6fc..e52d07e 100644 --- a/docs/PRODUCT_UPGRADE_PLAN.md +++ b/docs/PRODUCT_UPGRADE_PLAN.md @@ -1,171 +1,109 @@ # Product Upgrade Plan -> Status: living document. Created 2026-06-12 alongside the `1.3.0` line. -> This is a planning artifact, not a claims document. Anything not marked -> **Shipped** is a roadmap item and must not be advertised as done. +> Living product brief. Shipped capability is documented in README and +> CHANGELOG; forward work belongs in ROADMAP. -## 1. Positioning +## Product promise -**codebase-index is not an IDE and not a coding agent. It is the local -retrieval/index layer that gives terminal and MCP-based AI agents precise -codebase context.** +> Give AI coding agents a precise map of the codebase — locally, privately, and +> with evidence. -One-line description used everywhere: +The product vocabulary is: -> Local-first codebase retrieval for AI coding agents — Cursor-like codebase -> awareness for Claude Code, Codex CLI, OpenCode and MCP, without cloud indexing -> or IDE lock-in. +- **Find** — locate implementations and definitions. +- **Trace** — explain behavior through auditable code relationships. +- **Predict** — estimate the blast radius of a change. -What that commits us to: +`codebase-index` sits below the coding agent. It is not an IDE, a hosted search +platform, or an autonomous agent. -- We sit **below** the agent, not beside it. The agent (Claude Code, Codex CLI, - OpenCode, any MCP client) stays the user's interface; we return ranked - `file:line` packets it reads instead of scanning the repo. -- We are a **queryable index with a stable contract** (CLI `--json`, MCP schema), - not a one-shot context blob baked into a single agent's prompt. -- We are **local-first and offline by default**. The only path that can leave the - machine is opt-in external embeddings, gated three ways (see SECURITY_MODEL.md). +## Defensible differentiators -What we explicitly do **not** claim: +1. **Evidence-bearing retrieval** — ranked `file:line` ranges and recommended + reads under a token budget. +2. **Queryable local graph** — callers, references, paths, architecture, and + impact rather than a one-shot context blob. +3. **Honest uncertainty** — index freshness, result confidence, graph coverage, + and edge confidence are part of the response contract. +4. **One implementation, several surfaces** — CLI, Skill, and MCP share the + service layer. +5. **Auditable privacy** — network off by default, no telemetry, exclusion gates, + output redaction, and strict diagnostics. +6. **Measured claims** — public benchmark code and raw results live with the + product. -- Not a Cursor/IDE replacement. -- Not best-in-class framework-aware graph retrieval *yet* — today the graph is - import/call/reference/inheritance, not full route→handler→service→model - intelligence. -- Not proven at 100k/1M LOC scale — the public suite is synthetic; the only - real-repo evidence is a single 55k LOC Java run. +## Primary users -## 2. Target users - -| Persona | Pain today | What we give them | -|---|---|---| -| Claude Code / Codex CLI / OpenCode user on a medium-to-large repo | Agent burns context window grepping and reading whole files | Ranked `file:line` packets; the agent reads 3 files, not 60 | -| Privacy-constrained team (proprietary / regulated code) | Cloud code-intelligence is a non-starter | No network by default, no telemetry, secret redaction, ignore gates | -| MCP power user wiring multiple tools | Wants a stable, queryable code index as a tool, not a black box | stdio MCP server with a documented tool contract + `--json` CLI | -| Tooling/automation author | Needs scriptable retrieval other tools can build on | Agent-agnostic CLI with machine-readable JSON, SQLite the index lives in | - -Non-users (be honest): people who want a full IDE, multi-repo enterprise code -search, or a turnkey hosted platform. Point them at Cursor / Sourcegraph. - -## 3. Competitor matrix - -Full prose lives in [COMPARISON.md](COMPARISON.md); this is the planning view. - -| Tool | Category | Strongest at | Where we differ | Choose them when | -|---|---|---|---|---| -| Manual grep/read | Baseline | Exact ad-hoc string match | Ranking, symbols, graph, token budget | One known string, tiny scope | -| Cursor | AI IDE | Integrated editor + codebase awareness | Terminal/MCP-agnostic, offline by default, open | You live in Cursor's IDE | -| Aider repo-map | Agent context | Graph-ranked, token-budgeted map feeding Aider chat | Reusable queryable API across agents, freshness/security gates | You use Aider as your agent | -| Sourcegraph / Cody / Amp | Enterprise code intelligence | Cross-repo search/graph at org scale | Single-repo, local, lightweight, no platform/account | You need org-wide multi-repo search | -| Continue | Open-source coding agent | IDE+CLI agent with context features | Standalone retrieval index any agent can query, not an agent itself | You want the agent, not just the index | -| Codebase-Memory MCP | Local graph code-memory MCP | Broad graph engine, static binary, many languages | Simplicity, strict privacy model, token-budgeted packets, transparent Python, honest benchmarks | You need its broader graph/language reach today | - -We **do not** claim to beat Codebase-Memory MCP globally. We differentiate on -simplicity, the Claude/Codex/OpenCode workflow, token-budgeted packets, a -transparent Python implementation, a strict privacy model, and honest benchmarks. - -## 4. Differentiators (defensible today) - -1. **Token-budgeted retrieval packets** — output is line ranges + recommended - reads under an explicit token budget, not whole files or raw grep dumps. - Shipped: `--token-budget`, `recommended_reads`, honest ~13× fewer answer - tokens than an `rg`+window baseline on the 55k LOC Java run. -2. **One index, three surfaces, one service layer** — CLI, Claude/Codex/OpenCode - skills, and stdio MCP all run through `service.py`, so they cannot drift. -3. **Strict, auditable privacy model** — no network by default, no telemetry, - multi-gate exclusion pipeline, output-time secret redaction, `doctor` safety - self-check with `--strict` CI gating. -4. **Freshness contract** — every search response carries an `index` block - (`exists`/`stale`/`files_changed_since_build`) so the agent knows when to - `update` before trusting results. -5. **Graph-coverage honesty** — Tier-A vs Tier-B languages are labeled in - `stats`/`refs`/`impact`/`doctor`; partial-graph languages tell the agent to - fall back to Grep rather than reading "no references" as proof. -6. **Transparent, testable implementation** — pure-Python, 80% coverage gate, - golden CLI snapshots, public benchmark suite as a CI regression gate. - -## 5. Current weaknesses (own them) - -| Weakness | Impact | Plan | +| User | Need | Product outcome | |---|---|---| -| No large-scale real-repo benchmark | Can't claim 100k/1M LOC quality | Benchmark tasks §8; recruit public repos | -| Graph is import/call/ref only | `impact` misses framework wiring | ARCHITECTURE §9 + design doc `specs/2026-06-14-typed-framework-edges-design.md`; implementation behind §8 benchmark | -| ~~GitHub-only distribution~~ | **PyPI shipped in 1.6.0** — `pip install codebase-index` / `pipx` work; `uvx`/Homebrew still pending | Distribution tasks §9 (uvx/Homebrew) | -| MCP client docs unverified | Templates may be wrong per client version | Verify against each client, add per-client docs | -| Single-repo only | No monorepo/fleet context | Out of scope near-term; documented as non-goal | -| `clean` was a stub vs documented | Doc/reality gap | **Shipped in this pass** — real cache reset + test | - -## 6. High-impact roadmap (ranked) - -1. **Scale benchmarks on real public repos** (10k → 100k LOC), published with raw - logs. Highest credibility lever. -2. **Typed framework edges** (route→handler→service→model, test→impl, config→consumer) - with source spans + confidence. Biggest product-quality lever for `impact`. - *Design approved this pass* (`specs/2026-06-14-typed-framework-edges-design.md`); - implementation gated on the §8 graph benchmark. -3. **Distribution hardening**: PyPI publish, `uvx`/`pipx` story, signed checksums, - SBOM. Lowers adoption friction and raises supply-chain trust. -4. **MCP contract hardening**: ✅ `schema_version` on every payload + golden - snapshots per tool (this pass). Remaining: verified client docs, paging/progressive results. -5. **Retrieval tuning**: ✅ dampened the god-class `in_degree` tiebreak this pass - (log curve + lower cap, validated no-regression on the public suite). Remaining: - confirm the real-repo gain on the 3 honest Java misses (needs M12.5), per-intent weights review. -6. **Language reach**: config/IaC awareness (Dockerfile, Terraform, migrations, - CI), plus Swift/Dart/Scala/Vue/Svelte gaps called out in FAQ. - -## 7. Documentation tasks - -- [x] `docs/PRODUCT_UPGRADE_PLAN.md` (this file). -- [x] README "How is this different?" section answering why-not-grep/Cursor/Aider/ - Sourcegraph/Codebase-Memory on the first screen. -- [x] `docs/COMPARISON.md` explicit rows + prose for Continue, Amp, Codebase-Memory MCP. -- [x] `docs/BENCHMARKS.md` "claims not to make yet" + TODO benchmark checklist. -- [x] `docs/RELEASE_CHECKLIST.md`. -- [ ] Verified per-client MCP setup docs (after testing each client version). -- [x] A short "trust model in 60 seconds" callout reused across README/SECURITY. - -## 8. Benchmark tasks - -Track in [BENCHMARKS.md](BENCHMARKS.md); none may be reported until run with logs. - -- [ ] 10k LOC public repo: Recall@1/3/5, MRR, nDCG, token economy. -- [ ] 100k LOC public repo: same, plus index build time + incremental update latency. -- [ ] Multi-language public repo (≥3 Tier-A languages) with per-language breakdown. -- [ ] Head-to-head vs vanilla agent grep/read behavior (tokens + recall). -- [ ] Head-to-head vs repo-map-style context (tokens + recall). -- [ ] Graph task benchmark: `refs`, `impact`, and route→handler→service paths - against hand-labeled ground truth. -- [ ] Publish raw logs next to every headline number, like - `tests/benchmark_honest_RESULTS.md`. - -## 9. Distribution / release tasks - -- [ ] Publish to PyPI; switch docs to `pip install codebase-index` with GitHub - pin as the reproducible alternative. -- [ ] `uvx codebase-index` and `pipx install codebase-index` once on PyPI. -- [ ] Homebrew tap. -- [ ] Signed release checksums (e.g. `cosign`/`minisign`) + published SBOM. -- [ ] Reproducible-install smoke on a clean machine per OS (extend - `scripts/release_smoke.py`). -- [x] `docs/RELEASE_CHECKLIST.md` to make releases repeatable. - -## 10. Technical improvements (ranked by impact / risk) - -| # | Improvement | Impact | Risk | Status | -|---|---|---|---|---| -| 1 | Implement `clean` (documented but was a stub) | Fixes doc/reality gap | Low | **Shipped (1.3.0 line)** | -| 2 | Dampen god-class `in_degree` tiebreak in rerank | +recall on real repos | Medium (retune) | **Shipped this pass** — log dampening + lower cap; no-regression on the public suite + a targeted regression test. Real-repo gain still needs M12.5. | -| 3 | `schema_version` on every MCP payload | Stable contract | Low | **Shipped this pass** — `schema_version` + `tool` envelope on every payload (incl. errors), asserted + golden-locked. | -| 4 | Golden snapshots for each MCP tool output | Regression safety | Low | **Shipped this pass** — `tests/golden/mcp_*.json` via `tests/test_mcp_golden.py`. | -| 5 | Typed framework edges in the graph | Better `impact` | High | Design doc shipped this pass (`docs/superpowers/specs/2026-06-14-typed-framework-edges-design.md`); implementation behind the §8 benchmark. | -| 6 | Config/IaC parsers (Dockerfile, Terraform, migrations) | Coverage | Medium | **Partly shipped this pass** — Tier-C labeling for Dockerfile/Terraform/HCL/INI/Make (already FTS-indexed, now language-labeled); tree-sitter parsing of these still roadmap. | -| 7 | Paging/progressive MCP results | Big-repo UX | Medium | Roadmap (MCP.md) | - -Also fixed this pass (not previously tracked): the MCP server failed to import on -`mcp>=1.27` + `pydantic>=2.10` (FastMCP auto-built a structured-output schema from -the `-> str` return annotation and raised). Tools now register as unstructured -(`structured_output=False` where supported), so the server loads on current `mcp`. - -Rule for this repo: small, safe, tested changes land directly; anything that -risks destabilizing retrieval quality or the security model is documented here -first and lands behind a benchmark. +| Terminal coding-agent user | Stop broad repo scans | Agent reads a small ranked evidence set | +| Privacy-constrained team | Keep source off hosted indexes | Local derived index and explicit network gates | +| MCP power user | Stable code-intelligence tools | Versioned, scriptable contracts | +| Tooling author | Compose retrieval into automation | JSON CLI, MCP, and local SQLite | + +People who need a full AI IDE or organization-wide cross-repository search +should use products built for those jobs. + +## Product experience + +The ideal interaction does not require users to understand retrieval modes: + +```text +Ask a repository question + ↓ +Skill routes Find / Trace / Predict + ↓ +Index self-checks freshness + ↓ +Agent reads the minimum evidence + ↓ +Answer includes citations and honest uncertainty +``` + +Advanced users retain direct access to the lower-level commands. + +## Presentation system + +The product identity uses: + +- graphite backgrounds; +- blue for definitions and retrieval; +- purple for traces; +- green for verified impact and safe state; +- amber for inferred evidence; +- red only for failure and high-risk impact. + +The graph-route mark represents a highlighted path through local code. Product +copy should prefer outcomes over implementation terms on the first screen; +Tree-sitter, SQLite FTS5, and ranking details belong below the core story. + +Canonical short copy: + +> Find implementations. Trace behavior. Predict change impact. + +Canonical trust copy: + +> Local by default. No telemetry. Evidence on every answer. + +## Success metrics + +Measure product outcomes, not command count: + +- task success rate; +- Recall@1/3/5 and MRR; +- tokens and files read before the answer; +- time to an evidence-backed answer; +- citation correctness; +- graph path precision; +- false-positive impact warnings; +- clean install and first-query completion rate. + +## Execution order + +1. Clarify product language and reduce documentation duplication. +2. Keep the skill protocol compact through progressive references. +3. Publish stronger real-repository and task-level evaluations. +4. Add task-context and diff-impact workflows. +5. Add typed framework edges behind graph-quality gates. + +See [ROADMAP.md](ROADMAP.md) for the live delivery sequence. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index a290f23..2ae5744 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -122,6 +122,7 @@ After this quick start, you should have: - Look up a specific symbol: `codebase-index symbol "AuthService"` - Find callers: `codebase-index refs "AuthService.login"` - Check impact: `codebase-index impact "src/auth/AuthService.ts"` +- Check the current diff: `codebase-index diff-impact` - View stats: `codebase-index stats` - Run diagnostics: `codebase-index doctor` diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index d6c0856..a159688 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -86,7 +86,7 @@ them by hand and verify: - [ ] `codebase-index mcp --root .` starts and registers all tools (`tests/test_mcp_server.py`): `healthcheck`, `search_code`, `find_symbol`, - `find_refs`, `impact_of`, `explain_code`, `index_stats`. + `find_refs`, `impact_of`, `impact_of_diff`, `explain_code`, `index_stats`. - [ ] MCP and CLI agree (shared `service.py`) — vector channel + graph tier surfaced in both. - [ ] `docs/MCP.md` client templates still match the shipped tool list. @@ -110,7 +110,6 @@ them by hand and verify: ## Future hardening (not yet implemented — do not claim as done) -- [ ] PyPI publish (then `pip install codebase-index`, `uvx`, `pipx` without a Git URL). - [ ] Homebrew tap. - [ ] Signed release checksums (`cosign` / `minisign`). - [ ] Published SBOM (e.g. CycloneDX) attached to each release. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d139f5e..fa67701 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,112 +1,103 @@ -# Roadmap & First Implementation Tasks - -Milestones are vertical-ish slices: each ends with something runnable and testable. -This is the **technical-milestone** view (M0–M10). The product-level -[ROADMAP.md](../ROADMAP.md) tells the same story at a finer grain and carries it -further (it splits the MCP server into M11 and adds M11.5/M12/M12.5/M13 for MCP -hardening, benchmarks, and the typed-edge graph). Where the two disagree on a -number, the product roadmap is the current product view; this file tracks the -original implementation slices. The `(Mx)` tags in -[CHANGELOG.md](../CHANGELOG.md) follow this technical numbering. - -## M0 — Architecture & scaffold ✅ (this repo) -- Repo tree, docs (ARCHITECTURE/RETRIEVAL/SCHEMA/SECURITY/INSTALLATION), SKILL.md draft. -- `pyproject.toml`, module skeletons with responsibilities, CLI command stubs. -- **Exit:** `pip install -e .` works; `codebase-index --help` lists all commands (stubs ok). - -## M1 — Storage + discovery + ignore rules ✅ -- `storage/db.py`: connection, pragmas, apply `schema.sql`, `meta.schema_version`. -- `discovery/`: walker, layered ignore (`pathspec`), `classify.py` (lang/binary/size/secret gates). -- **Exit:** `codebase-index index` populates `files` correctly; secrets/binaries/build dirs excluded. -- Tests: `test_ignore.py`, `test_discovery.py`, `test_storage.py`. - -## M2 — FTS5 lexical indexing ✅ -- `parsers/line_chunker.py`: window chunks with overlap + token estimate. -- `fts_chunks` virtual table + sync triggers + code-aware tokenizer. -- `retrieval/searchers.py` (FTS only) + `output/` renderers + `search --mode fts`. -- **Exit:** `codebase-index search ""` returns ranked lexical results with line ranges/snippets. -- `snake_case` is split at index time (plain unicode61 tokenizer); camelCase is expanded at query - time. A true custom FTS5 tokenizer via APSW is deferred. - -## M3 — Tree-sitter symbol extraction ✅ -- `parsers/treesitter.py` + `languages.py` (grammar registry, node→symbol maps for Python, - JavaScript, and TypeScript). -- Populate `symbols`; symbol-aligned chunks; `symbol` + `refs` (intra-file first). -- **Exit:** `codebase-index symbol ""` and `refs` work for supported languages; line fallback - for the rest. -- Shipped languages: Python, JavaScript, TypeScript. Go/Java/Rust/C/C++/Ruby/PHP follow the recipe - in `docs/LANGUAGES.md`. `refs` is intra-file (call sites + defs); cross-file resolution is M5. - -## M4 — Hybrid search + ranking ✅ -- `retrieval/intent.py`, path + symbol searchers, `fusion.py` (RRF), `rerank.py`, `budget.py`. -- `confidence` + `fallback_suggestions`; `search --mode hybrid` (default); `explain`. -- **Exit:** hybrid results outrank single-retriever on the fixture queries; token budget enforced. - -## M5 — Graph edges + impact ✅ -- `parsers/languages.py`: `imports_query` slot with import/extends/implements patterns (Python end-to-end; JS/TS query slots wired). -- `parsers/treesitter.py`: extract import + inheritance edges via capture-prefixed queries. -- `graph/builder.py`: cross-file edge resolution by unambiguous symbol name / module→file suffix; degree denormalization. -- `graph/expand.py`: bounded BFS impact walk (up/down/both, depth). -- **Exit:** `codebase-index impact "" --direction up|down|both --depth N` returns a sensible blast radius on fixtures. Ambiguous symbol names are left unresolved by design. - -## M6 — Optional embeddings / vector backend ✅ -- `embeddings/` package (protocol + noop default + lazy local + gated external), `sqlite-vec` `vec_chunks` store loaded on demand, indexer embedding pass behind `embeddings.enabled`, and a vector retriever fused into hybrid with per-intent weights. External backend refused unless `allow_external` + `$CBX_EMBEDDINGS_API_KEY` + an endpoint warning (SECURITY.md §4). Disabled path imports no optional dep and is byte-for-byte unchanged. -- **Exit:** with extras installed + enabled, semantic queries improve recall; disabled path unchanged. - -## M7 — Claude Code Skill packaging ✅ -- Shipped: `init` materializes the wheel-bundled skill template (SKILL.md + cbx/cbx.ps1) to `.claude/skills/codebase-index/`, writes resolved `config.json`, and idempotently gitignores the cache (`--force` to overwrite). The freshness contract is honored end-to-end — `search` returns real `stale`/`files_changed_since_build` (git clean-tree fast-path + mtime diff), so the skill triggers `update`/`index` per SKILL.md. `--with-hooks` writes a reviewable hooks example; auto-merging hooks + `watch` are M8. - -## M7.5 — One-command plugin install ✅ -- Repo doubles as a Claude Code plugin (`.claude-plugin/plugin.json` + `marketplace.json`). -- `SessionStart` hook (`scripts/bootstrap.sh`/`.ps1`) provisions a venv in `${CLAUDE_PLUGIN_DATA}` - with the pinned CLI (uv-preferred, pip fallback), reinstalling only when `requirements.lock` changes. -- `bin/cbx` + `bin/codebase-index` wrappers resolve the venv via a `.venv-path` pointer and keep the - subcommand whitelist. -- **Exit:** `/plugin install codebase-index@` → ask a codebase question → compact reads, - no manual `pip`/`init`/`index`. The bootstrap installs the package from the GitHub release tarball - pinned in `requirements.lock` (not PyPI); `CBX_INSTALL_SPEC` overrides the source for local/dev installs. - -## M8 — Hooks + watch mode ✅ -- Shipped: incremental `update` (mtime fast-path + sha verify + prune; `--since `, `--all`) is the engine the freshness contract calls; `init --with-hooks` auto-merges the `PostToolUse` update hook into `.claude/settings.json` idempotently; `watch` mode (optional `[watch]` extra) coalesces edit bursts into one debounced `update` and degrades to a clear error when watchdog is absent; `doctor` reports enabled hooks, cache-gitignore coverage, and freshness, exiting non-zero under `--strict` on high-severity findings. The full SECURITY.md §6 doctor checklist (secret-leak scan, perms, allowed-tools diff) is M9. - -## M9 — Tests, docs, examples, release ✅ -- Coverage across modules; CLI golden-output tests; perf check on a medium repo. -- `examples/queries.md`, finalized docs, CHANGELOG, tagged GitHub release + PyPI publish (PyPI shipped in 1.6.0). -- **Exit:** `pipx install "git+https://github.com/denfry/codebase-index.git@v1.2.0"` + `init` + ask a question works on a clean machine. - -*Shipped: golden-file tests lock CLI `--json` output; a `--runslow` perf smoke test guards -index/search latency on a synthetic medium repo; coverage is gated (`--cov-fail-under`) in a -CI matrix (Ubuntu/macOS/Windows x py3.11-3.13). `CHANGELOG.md` tracks releases; a tag-triggered -release pipeline builds, runs `twine check` + a clean-venv install smoke, and publishes a GitHub -release with the built artifacts, and — as of 1.6.0 — publishes to PyPI via Trusted Publishing -(OIDC, no stored token; a `workflow_dispatch` run can re-publish an already-tagged version). -`pip install codebase-index` -> `init` -> `index` -> ask a question is verified end-to-end by -`scripts/release_smoke.py`.* - -## M10 — MCP bridge ✅ (product roadmap M11) -- Shipped: a stdio Model Context Protocol server (`codebase-index mcp --root `, or the - `codebase-index-mcp` entry point) exposing `healthcheck`, `search_code`, `find_symbol`, - `find_refs`, `impact_of`, `explain_code`, and `index_stats` over the same `service.py` layer the - CLI uses — an optional addition, not a replacement for the Skill/CLI interface. Every payload - carries a `schema_version` + `tool` envelope, locked by golden snapshots (`tests/golden/mcp_*.json`). -- **Exit:** `codebase-index` can be used as an MCP tool by any MCP-compatible client. See - [MCP.md](MCP.md). -- Follow-up (product roadmap M11.5): verified per-client setup docs and paging/progressive results. - ---- - -## First concrete tasks (start here) - -1. **Scaffold the package** so `codebase-index --help` runs: flesh out `cli.py` Typer app with all - commands as stubs that print "not implemented" and parse the documented flags. -2. **Implement `storage/db.py` + `schema.sql`** exactly per SCHEMA.md; add a migration guard on - `meta.schema_version`. Test: open/close/recreate, pragma assertions. -3. **Implement `discovery/ignore.py`** with `pathspec`, merging the four ignore files + built-in - denylist + secret-filename + size + binary gates. Test against `tests/fixtures/` with planted - secrets and a `node_modules` dir — assert they're excluded. -4. **Implement `discovery/walker.py` + minimal `index`** to populate `files` (no chunks yet), so - `stats` shows real coverage. This gives the first end-to-end runnable slice. -5. **Build a fixture repo** under `tests/fixtures/sample_repo/` (a few py/ts files, an `.env`, a - `node_modules`, a binary, a generated file) used across all later milestone tests. - -Each task should land behind tests (TDD) and keep the base install network-free. +# Roadmap + +`codebase-index` is becoming the evidence layer between a coding agent and a +repository: **Find implementations, Trace behavior, Predict change impact.** + +This document separates shipped capability from forward work. Roadmap entries +are not product claims until they appear under `[Unreleased]` or a tagged +release in `CHANGELOG.md`. + +## Shipped foundation + +### Find + +- Local SQLite FTS5, path, and symbol retrieval. +- Hybrid ranking and optional embeddings. +- Intent-aware `search` and `explain`. +- Token-budgeted, skeletonized snippets and exact recommended reads. +- Freshness, confidence, pagination, and targeted fallbacks. + +### Trace + +- Tree-sitter symbols across the documented language tiers. +- Import, call, reference, and inheritance edges. +- `refs`, `path`, `describe`, and architecture community analysis. +- Human HTML graphs plus GraphML, DOT, and Neo4j exports. +- Edge confidence: `extracted`, `inferred`, and `ambiguous`. + +### Predict + +- Directional, depth-bounded `impact` analysis. +- Diff-aware impact aggregation over tracked working-tree changes. +- Graph-coverage honesty for partially supported languages. +- Diagnostics that surface stale indexes and incomplete graph extraction. + +### Delivery and trust + +- CLI, Claude Code plugin/skill, Codex CLI, OpenCode, and stdio MCP. +- Shared CLI/MCP service layer and versioned MCP payloads. +- Local and network-free default path with no telemetry. +- Secret exclusion and output redaction. +- Incremental update, watch hooks, doctor, skill update, and rollback. +- PyPI and `pipx` installation. + +## Now — prove and simplify + +The current priority is to make existing power obvious, measurable, and easy to +invoke. + +- [ ] Publish 10k and 100k LOC public-repository benchmark runs with raw logs. +- [ ] Add task-level agent evaluation: success rate, tokens, files read, time, + and citation correctness. +- [ ] Verify MCP setup against current releases of each documented client. +- [ ] Add progressive/paged MCP retrieval for large repositories. +- [ ] Complete `uvx` and clean-machine install verification on every CI OS. +- [ ] Publish signed checksums and an SBOM with releases. + +## Next — task-native context + +- [ ] Add a high-level task-context packet that groups implementation, + configuration, tests, and dependencies under one token budget. +- [x] Add diff-aware file impact analysis over tracked working-tree changes. +- [ ] Extend diff impact from files to exact changed symbols and line spans. +- [ ] Rank likely affected tests separately from general dependents. +- [ ] Surface public API and configuration-contract changes. +- [ ] Produce a concise evidence summary suitable for code review agents. + +These features must compose existing retrieval and graph primitives instead of +creating a second ranking implementation. + +## Then — framework-aware traces + +- [ ] Typed route → handler → service → repository → model edges. +- [ ] Test → implementation edges. +- [ ] Configuration → consumer edges. +- [ ] Migration → model edges. +- [ ] Dependency-injection and event/queue registration edges. +- [ ] Source spans, extractor identity, and confidence on every typed edge. + +Framework edges ship behind a hand-labelled graph benchmark. Unsupported +framework wiring must remain clearly marked as partial rather than silently +treated as absent. + +## Later + +- Additional language and config/IaC extraction where real usage justifies it. +- Faster indexing and lower-memory builds for very large single repositories. +- Homebrew or equivalent package-manager distribution. +- Multi-root workspace context if it can preserve the local-first trust model. + +Org-wide hosted search, an IDE, autonomous code editing, and opaque cloud +indexing are not current product goals. + +## Quality gates + +A roadmap item is ready to ship only when: + +1. CLI and MCP behavior share the same service implementation. +2. JSON contracts have regression tests. +3. retrieval or graph changes pass a labelled evaluation with no unexplained + quality regression; +4. security gates and the network-free default remain intact; +5. documentation distinguishes measured capability from inference; +6. installed skill copies pass `scripts/sync_skill_copies.py --check`. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index be4c959..8e4909a 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -78,15 +78,24 @@ allowed-tools: - Bash(codebase-index symbol:*) - Bash(codebase-index refs:*) - Bash(codebase-index impact:*) + - Bash(codebase-index diff-impact:*) + - Bash(codebase-index architecture:*) + - Bash(codebase-index path:*) + - Bash(codebase-index describe:*) + - Bash(codebase-index graph:*) - Bash(codebase-index stats:*) + - Bash(codebase-index doctor:*) - Bash(codebase-index update:*) + - Bash(codebase-index index:*) - Grep - Glob ``` -Explicitly **not** allowed via the skill: `clean`, `init`, `watch`, or unscoped `Bash`. Destructive -or scaffolding actions remain a human/manual decision. The wrapper scripts (`scripts/cbx`) only -forward to the installed `codebase-index` binary and reject unknown subcommands. +Explicitly **not** allowed via the skill: `clean`, `init`, `watch`, an unscoped +`codebase-index *`, `python -m codebase_index *`, or unscoped `Bash`. +Destructive or scaffolding actions remain a human/manual decision. The wrapper +scripts (`scripts/cbx`) only forward to the installed `codebase-index` binary +and reject unknown subcommands. ## 6. `doctor` — safety self-check diff --git a/docs/SEO.md b/docs/SEO.md index 6e0919e..18b832b 100644 --- a/docs/SEO.md +++ b/docs/SEO.md @@ -177,7 +177,7 @@ Features: - Secret redaction - Respects .gitignore -Install: pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.6.0" +Install: pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.7.0" GitHub: https://github.com/denfry/codebase-index ``` @@ -192,5 +192,5 @@ codebase-index builds a local hybrid index so Claude finds the right files witho - No network by default - Token-efficient output -pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.6.0" +pip install "codebase-index @ git+https://github.com/denfry/codebase-index.git@v1.7.0" ``` diff --git a/docs/SKILL_DESIGN.md b/docs/SKILL_DESIGN.md index beb5aed..d5fbef8 100644 --- a/docs/SKILL_DESIGN.md +++ b/docs/SKILL_DESIGN.md @@ -4,15 +4,18 @@ How the `codebase-index` Claude Code Skill works and how to extend it. ## Overview -The skill is defined in `skill/SKILL.md` with YAML frontmatter that Claude Code uses for automatic skill selection. +The skill is defined in `skill/SKILL.md` with YAML frontmatter that agent +clients use for automatic selection. The primary file is intentionally a short +operating protocol; detailed command and payload material lives in +`skill/references/` and is loaded only when needed. ## Frontmatter ```yaml --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. -allowed-tools: Bash(python *), Bash(python3 *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- ``` @@ -39,7 +42,9 @@ Restricts which tools Claude can use while executing this skill: | `Grep` | Fallback search when index is weak | | `Glob` | Fallback path discovery | -**Explicitly not allowed:** `Write`, `Edit`, `Bash` (unscoped), or any destructive commands. +**Explicitly not allowed:** `Write`, `Edit`, `Bash` (unscoped), +`codebase-index *` (unscoped), `python -m codebase_index *`, or destructive +and scaffolding commands such as `clean`, `init`, and `watch`. ## Skill Workflow @@ -48,14 +53,14 @@ User asks codebase question ↓ Skill auto-selected by Claude Code ↓ -Claude runs: codebase-index search "query" --json +Route intent: Find / Trace / Predict ↓ Parse JSON response: - Check index.exists / index.stale - Read recommended_reads line ranges - Check confidence level ↓ -Answer with citations +Answer + file:line evidence ↓ If confidence low → fallback to Grep/Glob ``` @@ -79,6 +84,31 @@ The skill enforces token-efficient behavior: - Use `symbol`/`refs`/`impact` for refinement, not reworded searches - Fallback to Grep/Glob only when confidence is low +## Progressive References + +`SKILL.md` links two optional resources: + +- `references/commands.md` — command options, graph commands, health commands, + and query examples. +- `references/response-contract.md` — payload fields, freshness handling, + partial-coverage behavior, and answer examples. + +Do not move the core evidence or freshness protocol out of `SKILL.md`; agents +need those rules on every invocation. Keep detailed option lists and examples in +references so they do not consume context on routine searches. + +## Answer Contract + +The skill asks agents to return: + +1. the direct answer; +2. the minimum supporting `file:line` evidence; +3. confidence only when evidence is partial, inferred, stale, or missing; +4. a next check only when it materially reduces uncertainty. + +This prevents tool narration from displacing the actual engineering answer and +prevents partial graph coverage from being presented as proof of absence. + ## Extending the Skill ### Adding New Commands @@ -86,8 +116,11 @@ The skill enforces token-efficient behavior: If you add a new CLI command, update: 1. `skill/SKILL.md` — add the command to the intent table -2. `skill/SKILL.md` — add to `allowed-tools` if needed -3. `skill/examples/basic-usage.md` — add usage example +2. `skill/references/commands.md` — document detailed options and examples +3. `skill/SKILL.md` — add to `allowed-tools` if needed +4. Both safe wrappers — add the subcommand only if it is read-only or a + freshness operation +5. Run `python scripts/sync_skill_copies.py` ### Custom Wrapper Scripts diff --git a/docs/installer.md b/docs/installer.md index 0c54c1b..689fda3 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -108,7 +108,7 @@ pwsh ./install.ps1 -Target claude -InstallDir "D:\skills\codebase-index" **Pinning по ветке/тегу** (воспроизводимость и безопасность): ```sh -sh install.sh --branch v1.6.0 +sh install.sh --branch v1.7.0 ``` --- @@ -151,7 +151,7 @@ sh install.sh --branch v1.6.0 ```json { "skill_name": "codebase-index", - "version": "1.6.0", + "version": "1.7.0", "installed_at": "2026-05-29T12:00:00Z", "target": "claude", "os": "linux", diff --git a/pyproject.toml b/pyproject.toml index cf35174..35ec711 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "codebase-index" dynamic = ["version"] -description = "Local-first hybrid codebase index for AI coding agents, exposed as CLI, Skill, and MCP tools." +description = "A precise local codebase map for AI coding agents: find, trace, and predict with file-line evidence." readme = "README.md" requires-python = ">=3.11" license = { text = "MIT" } @@ -73,6 +73,11 @@ packages = ["src/codebase_index"] line-length = 100 target-version = "py311" +[tool.ruff.lint] +# Keep the pre-0.16 default profile explicit. Ruff 0.16 expanded its implicit +# defaults substantially; upgrades must not silently redefine this CI gate. +select = ["E4", "E7", "E9", "F"] + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q --cov=codebase_index --cov-report=term-missing --cov-fail-under=80" diff --git a/requirements.lock b/requirements.lock index 6b38098..679ed63 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,3 +1,3 @@ -codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.6.0.tar.gz +codebase-index @ https://github.com/denfry/codebase-index/archive/refs/tags/v1.7.0.tar.gz tree-sitter==0.25.2 tree-sitter-language-pack==1.8.1 diff --git a/scripts/gen_assets.py b/scripts/gen_assets.py index 2cb2a94..0158659 100644 --- a/scripts/gen_assets.py +++ b/scripts/gen_assets.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate brand assets for codebase-index (social preview + README demo still). +"""Generate the reproducible codebase-index brand assets. Pure-Pillow, no external binaries. Renders at 3x and downsamples with LANCZOS for crisp typography. Re-run after changing copy: @@ -7,8 +7,9 @@ python scripts/gen_assets.py Outputs: + assets/mark.png 256x256 -> logo / avatar source assets/social-preview.png 1280x640 -> upload in Settings -> Social preview - assets/demo.png 1200x760 -> embed near the top of README.md + assets/demo.png 1200x760 -> README product story """ from __future__ import annotations @@ -127,6 +128,44 @@ def downsave(img: Image.Image, w: int, h: int, path: Path) -> None: print(f" {path} {w}x{h} {kb:.0f} KB") +# --------------------------------------------------------------------------- # +# Mark: 256 x 256 +# --------------------------------------------------------------------------- # +def build_mark(out: Path) -> None: + W = H = 256 + w, h = W * SS, H * SS + img = gradient_bg(w, h) + add_glow(img, int(w * 0.78), int(h * 0.18), 150 * SS, BLUE, 60) + d = ImageDraw.Draw(img) + + pad = 28 * SS + d.rounded_rectangle( + [pad, pad, w - pad, h - pad], + radius=46 * SS, + fill=PANEL, + outline=BORDER, + width=2 * SS, + ) + + # A compact dependency route: three evidence nodes connected through a + # highlighted path. It stays legible at GitHub-avatar size. + points = [(72, 82), (128, 128), (184, 82), (128, 184)] + for a, b in ((0, 1), (1, 2), (1, 3)): + d.line( + [points[a][0] * SS, points[a][1] * SS, + points[b][0] * SS, points[b][1] * SS], + fill=BLUE if b != 3 else GREEN, + width=8 * SS, + ) + for i, (x, y) in enumerate(points): + r = (15 if i == 1 else 11) * SS + color = GREEN if i == 3 else (FG if i == 1 else CYAN) + d.ellipse([x * SS - r, y * SS - r, x * SS + r, y * SS + r], + fill=color, outline=BG_TOP, width=4 * SS) + + downsave(img, W, H, out) + + # --------------------------------------------------------------------------- # # Social preview: 1280 x 640 # --------------------------------------------------------------------------- # @@ -156,7 +195,7 @@ def build_social(out: Path) -> None: d.text((LM + seg, wy), "-index", font=wm, fill=BLUE) # tagline - d.text((LM, 214 * SS), "Local codebase indexing for AI coding agents", + d.text((LM, 214 * SS), "A precise map of your codebase for AI coding agents", font=f_ui(36), fill=FG2) # terminal @@ -195,7 +234,7 @@ def build_social(out: Path) -> None: ry += 33 * SS # chips - chips = ["Tree-sitter", "SQLite FTS5", "Graph impact", "MCP server"] + chips = ["FIND", "TRACE", "PREDICT", "LOCAL BY DEFAULT"] cf = f_ui_b(16) cx = LM cy = 574 * SS @@ -215,65 +254,105 @@ def build_demo(out: Path) -> None: add_glow(img, int(w * 0.5), int(h * -0.05), 460 * SS, BLUE, 26) d = ImageDraw.Draw(img) - # terminal - x0, y0, x1, y1 = 48 * SS, 56 * SS, (W - 48) * SS, 660 * SS - window_chrome(d, x0, y0, x1, y1, "bash — codebase-index") - - mono = f_mono(20) - mono_b = f_mono_b(20) - bx = x0 + 32 * SS - cw = d.textlength("0", font=mono) - lh = 30 * SS - y = y0 + 70 * SS - - def col(n): # x position at character column n - return bx + cw * n - - # command - d.text((bx, y), "$", font=mono_b, fill=GREEN) - d.text((col(2), y), "codebase-index search ", font=mono, fill=FG) - cmdw = d.textlength("codebase-index search ", font=mono) - d.text((col(2) + cmdw, y), '"where is user authentication implemented?"', font=mono, fill=CYAN) - y += lh * 2 - - d.text((bx, y), "Top matches:", font=mono_b, fill=FG); y += lh - d.text((bx, y), "Rank Path Symbols Score Reason", - font=mono, fill=MUTED); y += lh - - table = [ - ("1", "src/auth/AuthService.ts", "AuthService, login", "0.92", "exact symbol match", GREEN), - ("2", "src/routes/auth.ts", "loginHandler, logout", "0.78", "FTS · 4 callers", BLUE), - ("3", "src/middleware/auth.ts", "requireAuth", "0.65", "path · FTS match", MUTED), - ] - for rank, path, syms, score, reason, scol in table: - d.text((col(2), y), rank, font=mono_b, fill=BLUE) - d.text((col(7), y), path, font=mono, fill=CYAN) - d.text((col(32), y), syms, font=mono, fill=FG2) - d.text((col(53), y), score, font=mono_b, fill=scol) - d.text((col(60), y), reason, font=mono, fill=FG2) - y += lh - y += lh - - d.text((bx, y), "Recommended reads:", font=mono_b, fill=FG); y += lh - reads = [ - ("1.", "src/auth/AuthService.ts:12-148", "matched AuthService, login(), validatePassword()"), - ("2.", "src/routes/auth.ts:20-91", "/login route calls AuthService.login()"), - ("3.", "src/middleware/auth.ts:5-42", "auth middleware validates sessions"), + # Header + wm = f_mono_b(34) + d.text((48 * SS, 46 * SS), "codebase", font=wm, fill=FG) + seg = d.textlength("codebase", font=wm) + d.text((48 * SS + seg, 46 * SS), "-index", font=wm, fill=BLUE) + d.text((48 * SS, 94 * SS), "One local map. Three engineering jobs.", + font=f_ui(23), fill=FG2) + + cards = [ + { + "title": "FIND", + "color": BLUE, + "query": '"where is auth implemented?"', + "lines": [ + ("01", "src/auth/AuthService.ts:12", "0.92"), + ("02", "src/routes/auth.ts:20", "0.78"), + ("03", "src/middleware/auth.ts:5", "0.65"), + ], + }, + { + "title": "TRACE", + "color": PURPLE, + "query": '"checkout → database"', + "lines": [ + ("", "CheckoutRoute", "extracted"), + ("→", "CheckoutService", "extracted"), + ("→", "OrderRepository", "inferred"), + ], + }, + { + "title": "PREDICT", + "color": GREEN, + "query": '"what changes with User?"', + "lines": [ + ("1", "direct dependents", "7"), + ("2", "affected tests", "4"), + ("!", "inferred edges", "2"), + ], + }, ] - for n, loc, reason in reads: - d.text((col(2), y), n, font=mono, fill=MUTED) - d.text((col(5), y), loc, font=mono_b, fill=CYAN) - y += lh - d.text((col(5), y), "reason: " + reason, font=mono, fill=MUTED) - y += lh + 4 * SS - - # footer wordmark + tagline - fy = (H - 64) * SS - wm = f_mono_b(26) - d.text((48 * SS, fy), "codebase-index", font=wm, fill=FG) - seg = d.textlength("codebase-index", font=wm) - d.text((48 * SS + seg + 14 * SS, fy + 6 * SS), - "local hybrid index · no network by default", font=f_ui(17), fill=MUTED) + + card_w = 352 + card_h = 430 + gap = 24 + x_start = 48 + y0 = 162 + for idx, card in enumerate(cards): + x0 = (x_start + idx * (card_w + gap)) * SS + y = y0 * SS + x1 = x0 + card_w * SS + y1 = y + card_h * SS + d.rounded_rectangle( + [x0, y, x1, y1], radius=18 * SS, fill=PANEL, + outline=BORDER, width=SS + SS // 2, + ) + d.rounded_rectangle( + [x0 + 20 * SS, y + 20 * SS, x0 + 102 * SS, y + 51 * SS], + radius=15 * SS, fill=PANEL_BAR, outline=card["color"], width=SS, + ) + d.text((x0 + 34 * SS, y + 27 * SS), card["title"], + font=f_ui_b(15), fill=card["color"]) + d.text((x0 + 22 * SS, y + 78 * SS), card["query"], + font=f_mono(17), fill=CYAN) + d.line( + [x0 + 22 * SS, y + 122 * SS, x1 - 22 * SS, y + 122 * SS], + fill=BORDER, width=SS, + ) + + row_y = y + 150 * SS + for lead, label, tail in card["lines"]: + d.text((x0 + 24 * SS, row_y), lead, font=f_mono_b(17), + fill=card["color"]) + d.text((x0 + 58 * SS, row_y), label, font=f_mono(16), fill=FG2) + tw = d.textlength(tail, font=f_mono_b(15)) + d.text((x1 - 24 * SS - tw, row_y + 2 * SS), tail, + font=f_mono_b(15), + fill=YELLOW if tail == "inferred" else card["color"]) + row_y += 55 * SS + + d.text((x0 + 24 * SS, y1 - 58 * SS), + ("precise file:line evidence" if idx == 0 else + "auditable dependency chain" if idx == 1 else + "ranked blast radius"), + font=f_ui(15), fill=MUTED) + + # Evidence strip + sy = 626 * SS + d.rounded_rectangle( + [48 * SS, sy, (W - 48) * SS, 702 * SS], + radius=16 * SS, fill=PANEL_BAR, outline=BORDER, width=SS, + ) + d.text((72 * SS, sy + 25 * SS), "EVIDENCE CONTRACT", font=f_ui_b(15), fill=GREEN) + d.text((246 * SS, sy + 23 * SS), + "freshness · confidence · coverage · recommended reads", + font=f_mono(17), fill=FG2) + + d.text((48 * SS, 726 * SS), + "local by default · no telemetry · CLI + Skill + MCP", + font=f_ui(15), fill=MUTED) downsave(img, W, H, out) @@ -282,6 +361,7 @@ def main() -> None: root = Path(__file__).resolve().parent.parent assets = root / "assets" print("Generating assets:") + build_mark(assets / "mark.png") build_social(assets / "social-preview.png") build_demo(assets / "demo.png") print("Done.") diff --git a/scripts/sync_skill_copies.py b/scripts/sync_skill_copies.py index 1cd76aa..7ea12d5 100644 --- a/scripts/sync_skill_copies.py +++ b/scripts/sync_skill_copies.py @@ -7,9 +7,9 @@ .claude/skills/codebase-index/ installed copy, committed for this repo .codex/skills/codebase-index/ installed copy, committed for this repo .opencode/skills/codebase-index/ installed copy, committed for this repo - skills/codebase-index/SKILL.md plugin skill (Claude Code picks up skills/) - skill/SKILL.md, skill/scripts/cbx, skill/scripts/cbx.ps1 - installer source package (shared files only; + skills/codebase-index/ plugin skill (SKILL.md + references) + skill/SKILL.md, skill/references/, skill/scripts/cbx, skill/scripts/cbx.ps1 + installer source package (shared files; the rest of skill/ is owned by install.sh) Version stamps maintained: @@ -39,7 +39,8 @@ Path(".opencode/skills/codebase-index"), ) PLUGIN_SKILL_REL = Path("skills/codebase-index") -INSTALLER_SHARED = ("SKILL.md", "scripts/cbx", "scripts/cbx.ps1") +SHARED_PREFIXES = ("SKILL.md", "references/") +INSTALLER_SHARED = ("scripts/cbx", "scripts/cbx.ps1") VERSION_RE = re.compile(r'^__version__ = "([^"]+)"$', re.M) PLUGIN_VERSION_RE = re.compile(r'("version"\s*:\s*)"[^"]+"') @@ -70,10 +71,15 @@ def expected_files(repo: Path, version: str) -> dict[Path, bytes]: expected[copy / rel] = (template / rel).read_bytes() expected[copy / ".skill_version"] = f"{version}\n".encode() - expected[PLUGIN_SKILL_REL / "SKILL.md"] = (template / "SKILL.md").read_bytes() + for rel in rels: + if rel.as_posix() == "SKILL.md" or rel.as_posix().startswith("references/"): + expected[PLUGIN_SKILL_REL / rel] = (template / rel).read_bytes() for rel in INSTALLER_SHARED: expected[Path("skill") / rel] = (template / rel).read_bytes() + for rel in rels: + if rel.as_posix() == "SKILL.md" or rel.as_posix().startswith("references/"): + expected[Path("skill") / rel] = (template / rel).read_bytes() return expected diff --git a/skill/SKILL.md b/skill/SKILL.md index ee31b7d..9404686 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/skill/examples/basic-usage.md b/skill/examples/basic-usage.md index 9c3d16a..e2da073 100644 --- a/skill/examples/basic-usage.md +++ b/skill/examples/basic-usage.md @@ -48,6 +48,12 @@ Returns all callers and references to the symbol. codebase-index impact "src/auth/AuthService.ts" ``` +For all tracked changes in the current working tree: + +```bash +codebase-index diff-impact --base HEAD +``` + Returns files and symbols that depend on the target, ordered by blast radius. ### Getting project overview diff --git a/skill/references/commands.md b/skill/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/skill/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/skill/references/response-contract.md b/skill/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/skill/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/skill/scripts/cbx b/skill/scripts/cbx index 5666358..94a9114 100644 --- a/skill/scripts/cbx +++ b/skill/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/skill/scripts/cbx.ps1 b/skill/scripts/cbx.ps1 index bb8e05d..eee56ae 100644 --- a/skill/scripts/cbx.ps1 +++ b/skill/scripts/cbx.ps1 @@ -8,7 +8,10 @@ param( ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @( + "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", + "path", "describe", "graph", "stats", "doctor", "update", "index" +) if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" diff --git a/skills/codebase-index/SKILL.md b/skills/codebase-index/SKILL.md index ee31b7d..9404686 100644 --- a/skills/codebase-index/SKILL.md +++ b/skills/codebase-index/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/skills/codebase-index/references/commands.md b/skills/codebase-index/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/skills/codebase-index/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/skills/codebase-index/references/response-contract.md b/skills/codebase-index/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/skills/codebase-index/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/src/codebase_index/__init__.py b/src/codebase_index/__init__.py index 77addf7..62be96d 100644 --- a/src/codebase_index/__init__.py +++ b/src/codebase_index/__init__.py @@ -4,4 +4,4 @@ See docs/ARCHITECTURE.md for the module map. """ -__version__ = "1.6.0" +__version__ = "1.7.0" diff --git a/src/codebase_index/cli.py b/src/codebase_index/cli.py index 3ad5f3b..ffd216f 100644 --- a/src/codebase_index/cli.py +++ b/src/codebase_index/cli.py @@ -479,6 +479,41 @@ def impact( typer.echo(json_out.render(resp) if is_json else md_out.render_impact(resp)) +@app.command("diff-impact") +def diff_impact( + ctx: typer.Context, + base_ref: str = typer.Option("HEAD", "--base", help="Git commit/ref to compare against."), + depth: int = typer.Option(2, "--depth", min=1), + direction: str = typer.Option("up", "--direction", help="up|down|both"), + max_files: int = typer.Option( + 200, "--max-files", min=1, + help="Safety cap for changed files analysed in one run.", + ), + json_flag: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Predict the graph blast radius of the current Git diff.""" + from .output import json as json_out + from .output import markdown as md_out + from .service import diff_impact_payload + + is_json = json_flag or bool(ctx.obj and ctx.obj.get("json")) + db_path, cfg = _ensure_index(ctx) + try: + payload = diff_impact_payload( + db_path, cfg, base_ref=base_ref, depth=depth, + direction=direction, max_files=max_files, + ) + except (ValueError, RuntimeError, OSError) as exc: + if is_json: + typer.echo(json_out.render({"error": str(exc), "base_ref": base_ref})) + else: + typer.echo(f"[codebase-index] {exc}") + raise typer.Exit(code=2) + typer.echo( + json_out.render(payload) if is_json else md_out.render_diff_impact(payload) + ) + + @app.command() def explain( ctx: typer.Context, diff --git a/src/codebase_index/mcp/server.py b/src/codebase_index/mcp/server.py index 0f7be5f..c842311 100644 --- a/src/codebase_index/mcp/server.py +++ b/src/codebase_index/mcp/server.py @@ -236,6 +236,37 @@ def impact_of( return _emit("impact_of", resp.model_dump()) +@_tool() +def impact_of_diff( + base_ref: str = "HEAD", + depth: int = 2, + direction: str = "up", + max_files: int = 200, +) -> str: + """Predict the graph blast radius of tracked changes in the working tree. + + Args: + base_ref: Git commit/ref used as the comparison base. + depth: How many graph hops to follow. + direction: "up", "down", or "both". + max_files: Safety cap for changed files analysed in one call. + """ + db_path, cfg = _resolve_db() + if not db_path.exists(): + return _emit("impact_of_diff", _no_index_payload()) + + from ..service import diff_impact_payload + + try: + payload = diff_impact_payload( + db_path, cfg, base_ref=base_ref, depth=depth, + direction=direction, max_files=max_files, + ) + except (ValueError, RuntimeError, OSError) as exc: + payload = {"error": str(exc), "base_ref": base_ref} + return _emit("impact_of_diff", payload) + + @_tool() def explain_code( query: str, diff --git a/src/codebase_index/output/markdown.py b/src/codebase_index/output/markdown.py index f120c5b..2994e1e 100644 --- a/src/codebase_index/output/markdown.py +++ b/src/codebase_index/output/markdown.py @@ -314,3 +314,51 @@ def render_impact(resp: ImpactResponse) -> str: if note: lines.append(note) return "\n".join(lines).rstrip() + "\n" + + +def render_diff_impact(payload: dict) -> str: + """Render a concise change-impact report for a human.""" + lines = [ + f"**diff impact:** `{payload['base_ref']}` → working tree ", + f"**direction:** `{payload['direction']}` · **depth:** {payload['depth']}", + "", + ] + changed = payload["changed_files"] + if not changed: + lines.append("_No tracked changes relative to the selected base._") + return "\n".join(lines) + "\n" + + lines.append(f"**changed files ({payload['changed_files_total']}):**") + lines.extend(f"- `{path}`" for path in changed) + if payload.get("truncated"): + lines.append( + f"- _Analysis capped at {len(changed)} files; raise `--max-files` to inspect more._" + ) + + affected = payload["affected_files"] + lines.append("") + if affected: + lines.extend([ + f"**affected files ({len(affected)}):**", + "| distance | path | changed by | edge | confidence |", + "|---:|---|---|---|---|", + ]) + for item in affected: + sources = ", ".join(f"`{p}`" for p in item["changed_by"]) + lines.append( + f"| {item['distance']} | `{item['path']}` | {sources} | " + f"{item.get('via_edge') or ''} | {item.get('via_confidence') or ''} |" + ) + else: + lines.append("_No graph-backed affected files found._") + + if payload.get("unresolved_files"): + lines.append("\n**not represented in the current index:**") + lines.extend(f"- `{path}`" for path in payload["unresolved_files"]) + if payload.get("coverage", {}).get("partial"): + lines.append( + "\n> Graph coverage is partial for " + + ", ".join(payload["coverage"]["languages"]) + + "; a short result is inconclusive." + ) + return "\n".join(lines).rstrip() + "\n" diff --git a/src/codebase_index/service.py b/src/codebase_index/service.py index 60bec24..1857221 100644 --- a/src/codebase_index/service.py +++ b/src/codebase_index/service.py @@ -11,6 +11,7 @@ import os import sqlite3 +import subprocess from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Union @@ -105,6 +106,141 @@ def search_payload( ) +def diff_impact_payload( + db_path: Path, + cfg: "Config", + *, + base_ref: str = "HEAD", + depth: int = 2, + direction: str = "up", + max_files: int = 200, +) -> dict[str, Any]: + """Aggregate graph impact for files changed relative to a Git commit. + + Git is invoked with argument arrays and a verified commit SHA; no shell is + involved. The file cap bounds worst-case graph work on very large diffs. + """ + from .graph.expand import impact_lookup + from .indexer.freshness import compute_freshness + from .storage.db import Database + + if direction not in {"up", "down", "both"}: + raise ValueError("direction must be one of: up, down, both") + if depth < 1: + raise ValueError("depth must be >= 1") + if max_files < 1: + raise ValueError("max_files must be >= 1") + if not base_ref or base_ref.startswith("-") or "\x00" in base_ref: + raise ValueError("base_ref must be a non-option Git revision") + + root = Path(cfg.root).resolve() + resolved = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", "--end-of-options", + f"{base_ref}^{{commit}}"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if resolved.returncode != 0: + detail = (resolved.stderr or resolved.stdout).strip() + raise ValueError(f"cannot resolve Git base {base_ref!r}: {detail or 'unknown revision'}") + base_commit = resolved.stdout.strip() + + changed = subprocess.run( + ["git", "-C", str(root), "diff", "--name-only", "--diff-filter=ACDMR", + base_commit, "--"], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + if changed.returncode != 0: + detail = (changed.stderr or changed.stdout).strip() + raise RuntimeError(f"git diff failed: {detail or 'unknown error'}") + + normalized_changed = list(dict.fromkeys( + line.strip().replace("\\", "/") + for line in changed.stdout.splitlines() + if line.strip() + )) + try: + own_cache = cache_dir_for(cfg).resolve().relative_to(root).as_posix() + except ValueError: + own_cache = "" + all_changed = [ + path for path in normalized_changed + if not own_cache + or (path != own_cache and not path.startswith(f"{own_cache}/")) + ] + truncated = len(all_changed) > max_files + targets = all_changed[:max_files] + + affected: dict[str, dict[str, Any]] = {} + unresolved: list[str] = [] + coverage_languages: set[str] = set() + coverage_reasons: list[str] = [] + + with Database(db_path) as db: + freshness = compute_freshness(db.conn, root, cfg).model_dump() + for target in targets: + impact = impact_lookup( + db.conn, target, depth=depth, direction=direction + ) + if not impact.nodes and not impact.files: + # A tracked file can be new, deleted after the last update, or + # excluded by the security/discovery gates. + from .storage import repo + if repo.file_by_path(db.conn, target) is None: + unresolved.append(target) + if impact.coverage.partial: + coverage_languages.update(impact.coverage.languages) + if impact.coverage.reason: + coverage_reasons.append(impact.coverage.reason) + for node in impact.nodes: + current = affected.get(node.path) + candidate = { + "path": node.path, + "distance": node.distance, + "changed_by": [target], + "via_edge": node.via_edge, + "via_confidence": node.via_confidence, + } + if current is None: + affected[node.path] = candidate + else: + if target not in current["changed_by"]: + current["changed_by"].append(target) + if node.distance < current["distance"]: + current.update({ + "distance": node.distance, + "via_edge": node.via_edge, + "via_confidence": node.via_confidence, + }) + + ranked = sorted( + affected.values(), + key=lambda item: (item["distance"], item["path"]), + ) + return { + "base_ref": base_ref, + "base_commit": base_commit, + "direction": direction, + "depth": depth, + "index": freshness, + "changed_files": targets, + "changed_files_total": len(all_changed), + "truncated": truncated, + "unresolved_files": unresolved, + "affected_files": ranked, + "coverage": { + "partial": bool(coverage_languages), + "languages": sorted(coverage_languages), + "reason": " ".join(dict.fromkeys(coverage_reasons)) or None, + }, + } + + def architecture_payload(db_path: Path, cfg: "Config") -> dict[str, Any]: """The cached architecture analytics (communities / god nodes / surprising / questions) plus index freshness — the payload both CLI and MCP serialize. diff --git a/src/codebase_index/skill_template/SKILL.md b/src/codebase_index/skill_template/SKILL.md index ee31b7d..9404686 100644 --- a/src/codebase_index/skill_template/SKILL.md +++ b/src/codebase_index/skill_template/SKILL.md @@ -1,198 +1,89 @@ --- name: codebase-index -description: Use this skill before answering questions about a repository's architecture, implementation locations, symbols, references, dependencies, refactoring impact, data flow, bugs, or where something is implemented. It searches a local hybrid codebase index so Claude reads only the most relevant files instead of scanning the entire project. -allowed-tools: Bash(python -m codebase_index *), Bash(python3 -m codebase_index *), Bash(codebase-index *), Bash(cbx *), Read, Grep, Glob +description: Use before answering repository questions about architecture, implementation, symbols, references, dependencies, refactoring impact, data flow, or bugs. Query the local hybrid index first so the agent reads only evidence-bearing file:line ranges instead of scanning the repository. +allowed-tools: Bash(codebase-index search *), Bash(codebase-index explain *), Bash(codebase-index architecture *), Bash(codebase-index symbol *), Bash(codebase-index refs *), Bash(codebase-index impact *), Bash(codebase-index diff-impact *), Bash(codebase-index path *), Bash(codebase-index describe *), Bash(codebase-index graph *), Bash(codebase-index stats *), Bash(codebase-index doctor *), Bash(codebase-index update *), Bash(codebase-index index *), Bash(cbx *), Read, Grep, Glob --- # Codebase Index -Use this skill first for codebase questions. +Use the local index before reading repository files. -Never scan the entire repository before searching the index. +The operating principle is **Find → Trace → Predict**: -## When to use +- **Find** the implementation with ranked retrieval. +- **Trace** behavior through definitions, callers, dependencies, and paths. +- **Predict** change impact while preserving an explicit evidence trail. -Invoke this skill **before reading any files** when the user asks about this project's code: +## Route the question -- "where is X implemented" / "find X" / "locate the X function" -- "how does X work" / "explain the X flow" -- "what breaks if I change X" / "what depends on X" (impact analysis) -- "who calls X" / "references to X" -- "trace the data flow of X" -- "why is this error happening" (error/stack trace) -- "explain the architecture" / "give me an overview" -- Any question about symbols, files, dependencies, or refactoring scope - -Do **not** use it for: editing files, running the application, or non-code questions. - -## How to call the CLI - -Use the `codebase-index` CLI directly, or the bundled `cbx` wrapper: - -```bash -codebase-index search "$QUERY" --json -``` - -Pick the subcommand by intent: - -| User intent | Command | +| Intent | Command | |---|---| -| "how does X work" / "explain X" / "walk me through" | `codebase-index explain "$QUERY" --json` | -| overview / architecture / "map the codebase" | `codebase-index architecture --json` | -| general / unsure | `codebase-index search "$QUERY" --json` | -| keyword / "where is" | `codebase-index search "$QUERY" --json` | -| a specific symbol name | `codebase-index symbol "" --json` | -| "who calls / references" | `codebase-index refs "" --json` | -| "what breaks if I change" | `codebase-index impact "" --json` | -| "how is X connected to Y" / dependency path | `codebase-index path "" "" --json` | -| "what is X" / describe a symbol's role | `codebase-index describe "" --json` | -| visual graph / "open graph" (for the human, not for you to read) | `codebase-index graph "" --open` | - -`architecture` returns the codebase map computed at index time — detected modules -(communities), god nodes (most-connected symbols), surprising cross-module links, -and suggested questions. Reach for it on "give me an overview" / "where do I -start" questions instead of a broad `explain`. - -`path "A" "B"` returns the shortest dependency/call chain between two symbols or -files; `describe "X"` returns a node card (definition, callers, callees, -in/out degree, module, god-node rank). Both annotate edges with a `confidence` -(`extracted` exact, `inferred` heuristic, `ambiguous` unresolved) — treat a path -or callee list that leans on `inferred`/`ambiguous` edges as less certain. - -The `graph` command renders an HTML dependency graph for a person to look at — -it is not a retrieval packet. Use it only when the user explicitly wants a visual -graph; for "what depends on X" answer from `impact`/`refs` instead. In a headless -session prefer `--output ` over `--open`. `--format graphml|dot|neo4j` -exports the graph for external tools (Gephi/yEd, Graphviz, Neo4j) instead of HTML. - -`explain` has a higher default token budget (2200) and HOW_IT_WORKS intent weights — use it whenever the question is about understanding behavior or flow. - -For `search`, pick a `--mode` when the intent is clear: -- `--mode symbol` — pure symbol lookups (faster, no FTS noise) -- `--mode fts` — text/keyword queries where symbol names don't matter -- `--mode hybrid` — default; best for mixed queries -- `--mode vector` — semantic / near-synonym queries ("where do we rate-limit - requests" without the exact words). Requires opt-in embeddings; falls back with - a clear message when they are not enabled. `hybrid` already blends vectors in - when embeddings are on, so reach for `vector` only for pure-semantic recall. - -Natural-language kind words such as `method`, `function`, `class`, `interface`, -`enum`, and `type` constrain the symbol retriever inside `search`. - -Use `--json` for programmatic parsing; omit for human-readable output. -Search/read commands auto-build the index when it is missing; still check -freshness and run `update`/`index` when responses report stale data. - -## Step-by-step workflow - -1. **Query the index** using the appropriate subcommand for `$QUERY`. -2. **Check index freshness** in the response: - - `index.exists: false` → run `codebase-index index` first, then re-query. - - `index.stale: true`, `files_changed_since_build < 20` → run `codebase-index update`, then re-query. - - `index.stale: true`, `files_changed_since_build ≥ 20` → run `codebase-index index` (full rebuild). - - Otherwise proceed with results. -3. **Read ONLY the `recommended_reads`** — use the Read tool with `offset`/`limit` to read the exact line ranges returned. Do not open whole files. -4. **Answer** with file:line citations (e.g., `src/auth/token.py:88-134`). -5. **Fallback** only if confidence is low or results are empty (see below). - -## Token-budgeted output interpretation - -The index returns a **ranked retrieval packet** with: - -- `rank` — result position (start with 1-3) -- `path` — file path -- `line_start` / `line_end` — exact line range to read -- `symbols` — symbols found in this range -- `score` — relevance score -- `reason` — why this result ranked (e.g., "exact symbol match, 4 callers") -- `snippet` — compact code excerpt (may already answer the question); `null` means budget was spent — read via `recommended_reads` instead -- `skeletonized` — when `true`, the `snippet` is a **focus skeleton**: import/signature/class lines and the line(s) matching your query are kept, while function bodies collapse to a marker like `... 24 lines elided (read 88-134)`. Read that line range (or the result's `line_start`/`line_end`) when you need a full body. -- `elided_lines` — how many source lines the skeleton folded away (`0` when not skeletonized). - -Top-level fields: - -- `recommended_reads` — the precise `{path, line_start, line_end}` list to open next. This is your read plan. -- `confidence` — `high` (answer directly), `medium` (read + optionally confirm with one Grep), `low` (use fallback). -- `fallback_suggestions` — ripgrep patterns and paths to try if the index is weak. -- `intent` / `mode` — how the query was classified and which retrievers ran; - useful to sanity-check a weak result (e.g. a "how does X work" question that - resolved to a bare symbol lookup may need `explain` instead). -- `pagination` — present only when more results exist than fit the page. It - reports `has_more` and `next_offset`. To page, re-run `search` with - `--offset ` (e.g. `search "query" --limit 10 --offset 10`). Prefer - refining with a more specific subcommand or raising `--token-budget` first — - page only when the top results genuinely miss the answer. -- `coverage` (on `refs`/`impact` only) — graph-completeness signal. Dependency - edges (imports/inheritance) are extracted only for fully supported languages. - When `coverage.partial` is `true` (the symbol/file is in a Tier-B language such - as Lua), an **empty or short `refs`/`impact` result is inconclusive** — it may - just be unanalyzed, not absent. Confirm with a Grep before concluding "nothing - references this". `coverage.languages` lists the affected languages. - -## Token efficiency rules - -- Trust the index. Read the **fewest** files needed — start with rank 1-3 only. -- Read **line ranges**, not whole files. Use `line_start`/`line_end` with Read's `offset`/`limit`. -- The `snippet` may already answer the question — re-read only if you need more context. -- Prefer `search`/`symbol`/`refs`/`impact`/`explain` over manual Grep/Glob — those are expensive fallbacks, not step 1. -- Don't re-run the query with trivially reworded text; refine with a different subcommand instead. -- For broad questions (`confidence: low`, architecture, data-flow), raise the budget: `--token-budget 3000`. -- Test files are demoted in ranking by default. Include "test" in the query to surface them. -- Snippets are skeletonized by default to fit more results in the budget. The matched line is always preserved; pass `--raw` (CLI) or `raw: true` (MCP) on the rare occasion you need full bodies inline instead of reading the cited line range. - -## Fallback behavior - -Fall back to built-in search **only** when: results are empty, `confidence` is `low`, or the user asks for something the index clearly doesn't cover. - -0. If confidence is consistently low across queries, run diagnostics first: - ```bash - codebase-index stats --json # per-language file/symbol counts + graph tier - codebase-index doctor # surface config or security issues - ``` - Low symbol counts for a language may mean the index needs a full rebuild: `codebase-index index`. - In `stats`, each language carries `graph: full|partial` (and `doctor` reports a - `graph_coverage` finding): `partial` (Tier-B) means `refs`/`impact` lack - import/inheritance edges for that language — treat empty results there as - inconclusive. - -1. Use `fallback_suggestions.ripgrep` patterns from the response via Grep. -2. If still nothing, Glob for likely paths, then Grep within them. -3. As a last resort, broaden the search — but tell the user the index was weak here (it may need a rebuild: `codebase-index index`). - -Never start with a full-repo scan when the index exists and is fresh. - -## Examples - -```bash -# "how does the auth flow work?" -codebase-index explain "auth flow" --json - -# "explain the overall architecture" / "where do I start" — modules, god nodes -codebase-index architecture --json - -# "where is auth token refresh implemented?" -codebase-index search "auth token refresh" --json - -# "what breaks if I change the User model?" -codebase-index impact "User" --json - -# "who calls send_email?" -codebase-index refs "send_email" --json - -# "find the AuthService class" -codebase-index symbol "AuthService" --json - -# precise symbol search (faster, no FTS noise) -codebase-index search "AuthService" --mode symbol --json - -# "how is the API layer connected to the database?" -codebase-index path "ApiController" "Database" --json - -# "what is the Database class and how is it used?" -codebase-index describe "Database" --json - -# generate and open an HTML graph around a file or symbol -codebase-index graph "User" --direction both --depth 2 --open -``` - -Then Read only the returned line ranges and answer with citations. +| Where is X implemented? | `codebase-index search "X" --json` | +| How does X work? | `codebase-index explain "X" --json` | +| What is this codebase? | `codebase-index architecture --json` | +| Find a named symbol | `codebase-index symbol "X" --json` | +| Who calls or references X? | `codebase-index refs "X" --json` | +| What changes if X changes? | `codebase-index impact "X" --json` | +| What does my current diff affect? | `codebase-index diff-impact --json` | +| How are X and Y connected? | `codebase-index path "X" "Y" --json` | +| Describe X and its neighborhood | `codebase-index describe "X" --json` | +| Produce a human graph | `codebase-index graph "X" --output ` | + +Use `search --mode symbol` for exact symbol work, `--mode fts` for text and +error messages, and the default `hybrid` mode for mixed questions. Use pure +`vector` mode only when embeddings are enabled and exact vocabulary is unknown. + +Read [references/commands.md](references/commands.md) only when command options +or routing remain unclear. + +## Evidence protocol + +1. Run the best-matching command with `--json`. +2. Check `index` before trusting the payload: + - missing → run `codebase-index index`, then repeat; + - stale with fewer than 20 changed files → run `codebase-index update`; + - stale with 20 or more changed files → run `codebase-index index`; + - fresh → continue. +3. Start with ranks 1–3. Read only `recommended_reads` line ranges. +4. Trace one additional hop only when the question requires behavior, + ownership, or impact. +5. Answer with `file:line` evidence and state uncertainty explicitly. + +Do not open whole files when a line range is available. A snippet may already +be sufficient. `skeletonized: true` means the response intentionally folded +unrelated body lines; read the supplied range when the missing body matters. + +## Confidence contract + +- **high** — answer from the indexed evidence. +- **medium** — read the recommended ranges and confirm the key claim with one + targeted lookup if necessary. +- **low** or no results — follow `fallback_suggestions`, then use a narrow + Grep/Glob fallback. + +On `refs` and `impact`, inspect `coverage`. If `coverage.partial` is true, an +empty result is inconclusive; confirm with targeted Grep before saying that +nothing references the target. + +Edges carry `confidence`: + +- `extracted` — exact parser evidence; +- `inferred` — heuristic resolution; +- `ambiguous` — unresolved or non-unique. + +Never present an inferred or ambiguous chain as certain. + +## Answer contract + +Structure repository answers around: + +1. **Answer** — the direct conclusion. +2. **Evidence** — the minimum supporting `file:line` references. +3. **Confidence** — only when evidence is partial, inferred, stale, or missing. +4. **Next check** — only when another check would materially reduce uncertainty. + +Do not narrate every search step. Do not claim absence from a partial graph. +Do not replace evidence with a generated HTML graph. + +For payload fields and failure handling, read +[references/response-contract.md](references/response-contract.md). diff --git a/src/codebase_index/skill_template/references/commands.md b/src/codebase_index/skill_template/references/commands.md new file mode 100644 index 0000000..0efb464 --- /dev/null +++ b/src/codebase_index/skill_template/references/commands.md @@ -0,0 +1,73 @@ +# Command Reference + +Load this reference only when the intent table in `SKILL.md` is insufficient. + +## Retrieval + +```bash +codebase-index search "" --json +codebase-index explain "" --json +``` + +Useful search options: + +- `--mode hybrid|fts|symbol|vector` +- `--token-budget ` +- `--limit ` +- `--offset ` +- `--raw` to disable snippet skeletonization +- `--no-fallback` to suppress fallback suggestions + +`explain` uses the HOW_IT_WORKS intent and a larger default token budget. Prefer +it over repeatedly rewording a broad search. + +## Code graph + +```bash +codebase-index architecture --json +codebase-index refs "" --json +codebase-index impact "" --direction up --depth 2 --json +codebase-index diff-impact --base HEAD --direction up --depth 2 --json +codebase-index path "" "" --json +codebase-index describe "" --json +``` + +- `architecture` reads module analysis cached at index time. +- `refs` finds definitions, calls, and graph-backed references. +- `impact` walks dependents (`up`), dependencies (`down`), or both. +- `diff-impact` aggregates impact for tracked changes relative to a verified + Git commit; new or excluded files are reported as unresolved. +- `path` returns the shortest known dependency/call chain. +- `describe` returns a node card with callers, callees, module, and centrality. + +Use `graph` only for a visualization intended for a person: + +```bash +codebase-index graph "" --direction both --depth 2 --output graph.html +``` + +For headless work, use `--output`; do not use `--open`. Exports also support +`--format graphml|dot|neo4j`. + +## Index health + +```bash +codebase-index stats --json +codebase-index doctor +codebase-index update +codebase-index index +``` + +Run `stats` and `doctor` when several unrelated queries have low confidence. +Low symbol counts or partial graph coverage can explain weak results. + +## Query examples + +```bash +codebase-index search "auth token refresh" --json +codebase-index search "AuthService class" --mode symbol --json +codebase-index search "connection reset by peer" --mode fts --json +codebase-index explain "checkout flow" --json +codebase-index impact "User" --direction up --depth 2 --json +codebase-index path "ApiController" "Database" --json +``` diff --git a/src/codebase_index/skill_template/references/response-contract.md b/src/codebase_index/skill_template/references/response-contract.md new file mode 100644 index 0000000..059d87a --- /dev/null +++ b/src/codebase_index/skill_template/references/response-contract.md @@ -0,0 +1,82 @@ +# Response Contract + +Load this reference when interpreting a retrieval packet or handling a weak +result. + +## Ranked results + +Each result can contain: + +- `rank` +- `path` +- `line_start` / `line_end` +- `symbols` +- `score` +- `reason` +- `snippet` +- `skeletonized` +- `elided_lines` + +`recommended_reads` is the read plan. Start with its first one to three entries +and use exact line ranges. + +`pagination.has_more` and `pagination.next_offset` indicate additional results. +Prefer a more specific command or a larger token budget before paging. + +## Freshness + +The `index` object is part of the evidence contract: + +```text +exists=false → index +stale=true, files_changed_since_build<20 → update +stale=true, files_changed_since_build≥20 → full index +stale=false → proceed +``` + +Repeat the original query after rebuilding or updating. + +## Weak results + +Fallback is allowed only when: + +- results are empty; +- confidence is low; +- graph coverage is partial; +- the requested information is not represented by the index. + +Use `fallback_suggestions.ripgrep` first. Otherwise construct one narrow Grep +pattern from the most distinctive symbol, error, or path in the question. +Avoid a full-repository scan unless targeted fallback also fails. + +If several queries are weak: + +```bash +codebase-index stats --json +codebase-index doctor +``` + +Report the limitation instead of inventing certainty. + +## Answer examples + +High confidence: + +```text +Session validation is implemented in `src/auth/session.py:44`. +The request middleware calls it from `src/http/auth.py:18`. +``` + +Partial graph: + +```text +The index found no graph-backed callers, but coverage for Lua is partial. +A targeted text search still finds two call sites in … +``` + +Inferred chain: + +```text +The route-to-service link is parser-extracted; the service-to-model link is +heuristic, so the final hop should be verified before refactoring. +``` diff --git a/src/codebase_index/skill_template/scripts/cbx b/src/codebase_index/skill_template/scripts/cbx index 5666358..94a9114 100644 --- a/src/codebase_index/skill_template/scripts/cbx +++ b/src/codebase_index/skill_template/scripts/cbx @@ -4,7 +4,7 @@ # - Whitelists subcommands so the skill can never invoke destructive ones (clean/init/watch). set -euo pipefail -ALLOWED="search explain symbol refs impact graph stats doctor update index" +ALLOWED="search explain architecture symbol refs impact diff-impact path describe graph stats doctor update index" sub="${1:-}" case " $ALLOWED " in diff --git a/src/codebase_index/skill_template/scripts/cbx.ps1 b/src/codebase_index/skill_template/scripts/cbx.ps1 index bb8e05d..eee56ae 100644 --- a/src/codebase_index/skill_template/scripts/cbx.ps1 +++ b/src/codebase_index/skill_template/scripts/cbx.ps1 @@ -8,7 +8,10 @@ param( ) $ErrorActionPreference = "Stop" -$allowed = @("search", "explain", "symbol", "refs", "impact", "graph", "stats", "doctor", "update", "index") +$allowed = @( + "search", "explain", "architecture", "symbol", "refs", "impact", "diff-impact", + "path", "describe", "graph", "stats", "doctor", "update", "index" +) if ($allowed -notcontains $Subcommand) { Write-Error "cbx: refusing subcommand '$Subcommand'. Allowed: $($allowed -join ', ')" diff --git a/tests/golden/mcp_impact_of_diff.json b/tests/golden/mcp_impact_of_diff.json new file mode 100644 index 0000000..40b1ec7 --- /dev/null +++ b/tests/golden/mcp_impact_of_diff.json @@ -0,0 +1,25 @@ +{ + "affected_files": [], + "base_commit": "", + "base_ref": "HEAD", + "changed_files": [], + "changed_files_total": 0, + "coverage": { + "languages": [], + "partial": false, + "reason": null + }, + "depth": 2, + "direction": "up", + "index": { + "built_at": "", + "exists": true, + "files_changed_since_build": 0, + "head_commit": "", + "stale": false + }, + "schema_version": 1, + "tool": "impact_of_diff", + "truncated": false, + "unresolved_files": [] +} diff --git a/tests/golden_utils.py b/tests/golden_utils.py index ced4803..eb7c10b 100644 --- a/tests/golden_utils.py +++ b/tests/golden_utils.py @@ -19,7 +19,7 @@ # Keys whose values are inherently volatile and must be masked, not compared. _TS_KEYS = {"built_at", "indexed_at", "generated_at"} -_SHA_KEYS = {"head_commit"} +_SHA_KEYS = {"head_commit", "base_commit"} # Released package version churns on every bump; mask it so goldens don't. Note # `schema_version` is deliberately NOT masked — it IS the contract under test. _VERSION_KEYS = {"package_version"} diff --git a/tests/test_diff_impact.py b/tests/test_diff_impact.py new file mode 100644 index 0000000..caaeca2 --- /dev/null +++ b/tests/test_diff_impact.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json +import shutil +import subprocess + +from typer.testing import CliRunner + +from codebase_index.cli import app + +runner = CliRunner() + + +def _git_repo(sample_repo, tmp_path): + root = tmp_path / "repo" + shutil.copytree(sample_repo, root) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "."], check=True) + subprocess.run( + [ + "git", "-C", str(root), + "-c", "user.name=Tests", + "-c", "user.email=tests@example.invalid", + "commit", "-qm", "baseline", + ], + check=True, + ) + return root + + +def test_diff_impact_reports_changed_and_affected_files(sample_repo, tmp_path): + root = _git_repo(sample_repo, tmp_path) + indexed = runner.invoke(app, ["--root", str(root), "index"]) + assert indexed.exit_code == 0, indexed.output + + changed = root / "src" / "models" / "user.py" + changed.write_text(changed.read_text(encoding="utf-8") + "\n# changed\n", encoding="utf-8") + + result = runner.invoke( + app, + ["--root", str(root), "--json", "diff-impact", "--depth", "2"], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["changed_files"] == ["src/models/user.py"] + assert payload["changed_files_total"] == 1 + assert payload["truncated"] is False + assert "src/api/service.py" in { + item["path"] for item in payload["affected_files"] + } + assert payload["index"]["stale"] is True + + +def test_diff_impact_empty_worktree_is_clear(sample_repo, tmp_path): + root = _git_repo(sample_repo, tmp_path) + assert runner.invoke(app, ["--root", str(root), "index"]).exit_code == 0 + + result = runner.invoke(app, ["--root", str(root), "diff-impact"]) + assert result.exit_code == 0, result.output + assert "No tracked changes" in result.output + + +def test_diff_impact_rejects_option_like_base(sample_repo, tmp_path): + root = _git_repo(sample_repo, tmp_path) + assert runner.invoke(app, ["--root", str(root), "index"]).exit_code == 0 + + result = runner.invoke( + app, + ["--root", str(root), "--json", "diff-impact", "--base=--output=/tmp/x"], + ) + assert result.exit_code == 2 + payload = json.loads(result.output) + assert "non-option Git revision" in payload["error"] diff --git a/tests/test_mcp_golden.py b/tests/test_mcp_golden.py index 8a1a608..6c76a3c 100644 --- a/tests/test_mcp_golden.py +++ b/tests/test_mcp_golden.py @@ -67,6 +67,7 @@ def _call(indexed_repo, tool_fn, **kwargs): "mcp_find_symbol": (lambda: mcp_server.find_symbol, {"name": "User"}), "mcp_find_refs": (lambda: mcp_server.find_refs, {"symbol": "refresh_access_token"}), "mcp_impact_of": (lambda: mcp_server.impact_of, {"target": "src/models/user.py", "direction": "up"}), + "mcp_impact_of_diff": (lambda: mcp_server.impact_of_diff, {}), "mcp_explain_code": (lambda: mcp_server.explain_code, {"query": "how does authentication work"}), "mcp_architecture": (lambda: mcp_server.architecture_overview, {}), "mcp_path_between": (lambda: mcp_server.path_between, {"source": "renew", "target": "refresh_access_token"}), @@ -81,6 +82,7 @@ def _call(indexed_repo, tool_fn, **kwargs): "mcp_find_symbol": "find_symbol", "mcp_find_refs": "find_refs", "mcp_impact_of": "impact_of", + "mcp_impact_of_diff": "impact_of_diff", "mcp_explain_code": "explain_code", "mcp_architecture": "architecture_overview", "mcp_path_between": "path_between", diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index b53c536..fd47394 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -35,6 +35,7 @@ def test_mcp_server_has_expected_tools(): "find_symbol", "find_refs", "impact_of", + "impact_of_diff", "explain_code", "architecture_overview", "path_between", @@ -71,6 +72,7 @@ def test_search_code_no_index(): "find_symbol": lambda: _call(mcp_server.find_symbol, name="Foo"), "find_refs": lambda: _call(mcp_server.find_refs, symbol="foo"), "impact_of": lambda: _call(mcp_server.impact_of, target="foo.py"), + "impact_of_diff": lambda: _call(mcp_server.impact_of_diff), "explain_code": lambda: _call(mcp_server.explain_code, query="how does foo work"), "architecture_overview": lambda: _call(mcp_server.architecture_overview), "path_between": lambda: _call(mcp_server.path_between, source="a", target="b"), @@ -108,6 +110,11 @@ def test_impact_of_no_index(): assert "error" in result +def test_impact_of_diff_no_index(): + result = _with_missing_db(lambda: _call(mcp_server.impact_of_diff)) + assert "error" in result + + def test_explain_code_no_index(): result = _with_missing_db(lambda: _call(mcp_server.explain_code, query="how does foo work")) assert "error" in result diff --git a/tests/test_packaging.py b/tests/test_packaging.py index ec76406..5297c2e 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -11,6 +11,8 @@ def _template(): def test_packaged_template_has_skill_and_scripts(): root = _template() assert (root / "SKILL.md").is_file() + assert (root / "references" / "commands.md").is_file() + assert (root / "references" / "response-contract.md").is_file() assert (root / "scripts" / "cbx").is_file() assert (root / "scripts" / "cbx.ps1").is_file() assert (root / "examples" / "hooks" / "settings.json").is_file() @@ -25,6 +27,18 @@ def test_packaged_skill_matches_dev_copy(): def test_packaged_cbx_whitelists_safe_subcommands_only(): cbx = (_template() / "scripts" / "cbx").read_text(encoding="utf-8") - assert 'ALLOWED="search explain symbol refs impact graph stats doctor update index"' in cbx + assert ( + 'ALLOWED="search explain architecture symbol refs impact diff-impact path describe ' + 'graph stats doctor update index"' + ) in cbx for forbidden in ("clean", "init", "watch"): assert f" {forbidden} " not in f' {cbx.split("ALLOWED=")[1].splitlines()[0]} ' + + +def test_packaged_skill_has_no_unscoped_cli_or_python_permission(): + skill = (_template() / "SKILL.md").read_text(encoding="utf-8") + frontmatter = skill.split("---", 2)[1] + assert "Bash(codebase-index *)" not in frontmatter + assert "Bash(python -m codebase_index *)" not in frontmatter + for forbidden in ("clean", "init", "watch"): + assert f"Bash(codebase-index {forbidden} *)" not in frontmatter

+ Quick start · + Install · + Benchmarks · + Security · + MCP +