diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index 3b95cffd..19c46ea9 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -5,13 +5,14 @@ on: permissions: contents: read - pull-requests: write env: ZIG_VERSION: 0.17.0-dev.813+2153f8143 ZIG_SHA256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 LEGACY_ZIG_VERSION: 0.16.0 LEGACY_ZIG_SHA256: 70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 + PAIRED_5829_BASE_SHA: dd36e9431925014ee2bed80346669a4afee7e42e + PAIRED_5829_SHA: 24e89c70d4f9cdaf5542a78d83d1890a42b4a046 jobs: bench: @@ -21,6 +22,9 @@ jobs: with: fetch-depth: 0 + - name: Test paired benchmark comparator + run: python3 -m unittest scripts/test_compare_bench_paired.py + - name: Install pinned Zig toolchains run: | set -euo pipefail @@ -47,12 +51,14 @@ jobs: run: python3 scripts/run-bench-json.py bench-head.json - name: Benchmark base with its declared compiler + id: base_bench env: - BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -euo pipefail - git fetch origin "$BASE_REF" --depth=1 - git worktree add ../codedb-base FETCH_HEAD + git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 + git worktree add ../codedb-base "$BASE_SHA" + test "$(git -C ../codedb-base rev-parse HEAD)" = "$BASE_SHA" cd ../codedb-base base_zig_version=$(sed -n 's/.*\.minimum_zig_version = "\([^"]*\)".*/\1/p' build.zig.zon) @@ -69,9 +75,84 @@ jobs: ;; esac test "$("$base_zig_dir/zig" version)" = "$base_zig_version" + echo "zig_version=$base_zig_version" >> "$GITHUB_OUTPUT" + echo "zig_dir=$base_zig_dir" >> "$GITHUB_OUTPUT" PATH="$base_zig_dir:$PATH" \ python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json" + - name: Paired counterbalanced benchmark with parity gate + env: + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_ZIG_VERSION: ${{ steps.base_bench.outputs.zig_version }} + BASE_ZIG_DIR: ${{ steps.base_bench.outputs.zig_dir }} + PAIRS: 5 + run: | + set -euo pipefail + if [[ "$BASE_ZIG_VERSION" != "$ZIG_VERSION" ]]; then + echo "Paired parity requires a pinned same-compiler baseline harness; refusing to skip the gate." >&2 + exit 1 + fi + + paired_base="../codedb-base" + if ! grep -q corpus_hash "$paired_base/src/bench.zig"; then + if [[ "$EVENT_BASE_SHA" != "$PAIRED_5829_BASE_SHA" ]]; then + echo "No pinned parity harness exists for base $EVENT_BASE_SHA" >&2 + exit 1 + fi + git cat-file -e "$PAIRED_5829_SHA^{commit}" 2>/dev/null || { + git fetch origin bench/v0.2.5829-paired-baseline --depth=1 + test "$(git rev-parse FETCH_HEAD)" = "$PAIRED_5829_SHA" + } + git merge-base --is-ancestor "$EVENT_BASE_SHA" "$PAIRED_5829_SHA" + test "$(git diff --name-only "$EVENT_BASE_SHA..$PAIRED_5829_SHA")" = $'src/bench.zig\nsrc/mcp.zig' + git worktree add ../codedb-paired-base "$PAIRED_5829_SHA" + paired_base="../codedb-paired-base" + fi + + mkdir -p bench-paired + corpus_source=$(realpath ../codedb-base) + head_source_sha=$(git rev-parse HEAD) + head_tree_sha=$(git rev-parse 'HEAD^{tree}') + git worktree add ../codedb-paired-head "$head_source_sha" + head_cwd=$(realpath ../codedb-paired-head) + run_sample() { + side="$1" + pair="$2" + order="$3" + sequence="$4" + if [[ "$side" == base ]]; then + cwd="$paired_base" + zig_dir="$BASE_ZIG_DIR" + production_sha="$EVENT_BASE_SHA" + else + cwd="$head_cwd" + zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${ZIG_VERSION}" + production_sha="$head_source_sha" + fi + ( + cd "$cwd" + PATH="$zig_dir:$PATH" python3 "$head_cwd/scripts/run-bench-json.py" \ + --bench-side "$side" --bench-pair "$pair" --bench-order "$order" --bench-sequence "$sequence" \ + --production-source-sha "$production_sha" --corpus-source-sha "$EVENT_BASE_SHA" \ + "$GITHUB_WORKSPACE/bench-paired/$side-$(printf '%02d' "$pair").json" \ + --corpus-source "$corpus_source" + ) + } + + for ((pair = 1; pair <= PAIRS; pair++)); do + if (( pair % 2 == 1 )); then + run_sample base "$pair" AB 1 + run_sample head "$pair" AB 2 + else + run_sample head "$pair" BA 1 + run_sample base "$pair" BA 2 + fi + done + python3 "$head_cwd/scripts/compare-bench-paired.py" bench-paired --require-parity --require-provenance \ + --allow-parity-skip codedb_snapshot --allow-parity-skip codedb_status \ + --expected-head-sha "$head_source_sha" --expected-head-tree-sha "$head_tree_sha" \ + --threshold-pct 10 --min-abs-ns 50000 --markdown-out bench-paired-report.md + - name: Compare run: | python3 scripts/compare-bench.py bench-base.json bench-head.json --threshold-pct 10 --markdown-out bench-report.md @@ -84,25 +165,12 @@ jobs: bench-base.json bench-head.json bench-report.md + bench-paired/ + bench-paired-report.md - - name: Comment PR - if: github.event_name == 'pull_request' - uses: actions/github-script@v9 - with: - script: | - const fs = require('fs'); - const body = fs.readFileSync('bench-report.md', 'utf8'); - try { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } catch (err) { - if (err.status === 403) { - core.warning('Skipping benchmark PR comment because this run token cannot write comments.'); - } else { - throw err; - } - } + - name: Publish benchmark summary + if: always() + run: | + report=bench-paired-report.md + [[ -f "$report" ]] || report=bench-report.md + [[ -f "$report" ]] && cat "$report" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-base-guard.yml b/.github/workflows/pr-base-guard.yml index ba97a38e..37483dc4 100644 --- a/.github/workflows/pr-base-guard.yml +++ b/.github/workflows/pr-base-guard.yml @@ -15,7 +15,7 @@ jobs: const msg = [ "๐Ÿ‘‹ Thanks for the contribution! Quick heads-up: this repo lands changes on the **current `release/*` branch**, not `main`.", "", - "Please retarget this PR via **Edit โ†’ base branch** to the active release branch (currently `release/0.2.5825`).", + "Please retarget this PR via **Edit โ†’ base branch** to the active `release/*` branch shown in the repository branch list.", "", "_(Automated hint โ€” reply here if you need a hand.)_", ].join("\n"); diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 6e83a8ce..e4c223e1 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -42,6 +42,11 @@ jobs: zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 zig_target: aarch64-linux asset_name: codedb-linux-arm64 + - runner: ubuntu-24.04 + zig_host: x86_64-linux + zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 + zig_target: x86_64-windows + asset_name: codedb-windows-x86_64.exe env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.event.release.tag_name }} APPLE_CERTIFICATE_P12: ${{ secrets.APPLE_CERTIFICATE_P12 }} @@ -106,7 +111,10 @@ jobs: else zig build -Doptimize=ReleaseFast -Dtarget=${{ matrix.zig_target }} fi - cp zig-out/bin/codedb "${{ matrix.asset_name }}" + binary=zig-out/bin/codedb + [[ -f "$binary" ]] || binary=zig-out/bin/codedb.exe + test -f "$binary" + cp "$binary" "${{ matrix.asset_name }}" - name: Upload build artifact uses: actions/upload-artifact@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e7ae37..c4421426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,33 @@ # Changelog +## 0.2.5830 - 2026-07-12 + +The first performance batch for this release accelerates steady-state MCP and +core read-only tools while preserving response and retrieval parity. Measured +MCP round trips improve from **2.16x to 99.11x** across tree, outline, symbol, +read, find, word, search, and bundle workloads. Large synthetic handler cases +improve **12.8x-109.7x** for outlines, deep reads, trees, fuzzy file lookup, and +word output; exact symbols improve **22.4x** through direct hash-index lookup. + +The implementation adds bounded, mutation-generation-validated render/score +caches, cached content hashes and line offsets, offset-based context body +extraction, a compact JSON-RPC framing fast path, and rolling generic trigram +construction. Ranking, parser, path-security, telemetry, and MCP schema behavior +are unchanged. Baseline/candidate MCP responses were byte-identical after ID +normalization, and the full suite, WASM build, and MCP E2E (20/20) pass. + +Detailed methodology, per-tool tables, memory bounds, limitations, and follow-up +targets are in [`docs/performance-0.2.5830.md`](docs/performance-0.2.5830.md). +The release gate ran 20 counterbalanced AB/BA pairs against the immutable +0.2.5829 source plus a pinned harness-only parity backport. Every parity-enabled +tool matched across every measured iteration after normalizing only response +duration, and no benchmark crossed the 10% plus 50us regression threshold. The +runner enforces a shared corpus fingerprint, full JSON-RPC response hashes, +commit/tree/compiler/corpus/order provenance, paired medians, and bootstrap +intervals; single-run minima are diagnostic only. + + ## 0.2.5829 - 2026-07-11 The release toolchain moves to pinned Zig `0.17.0-dev.813+2153f8143` without diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b69f9321..56c53c27 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -143,16 +143,35 @@ If you changed benchmarks: Benchmark-related PRs must say: - what layer is being measured - - driver-only - - HTTP-only - - end-to-end HTTP+DB + - handler/driver-only + - in-process MCP/HTTP + - end-to-end transport and storage - whether caches are on or off - whether numbers are cold-start or warmed steady-state -- number of runs -- whether values are single-run or median +- number of paired runs and their counterbalanced execution order +- paired median and dispersion/confidence interval - exact machine or CI environment +- whether base and head used the same compiler (source attribution) or each + revision's declared compiler (release-outcome comparison) -Do not publish cached results as uncached DB performance. +Performance claims must use output/hit parity on an identical corpus. Commit the +candidate so the runner can attest a clean source tree, then run: + +```bash +CODEDB_BENCH_PAIRS=10 scripts/bench-ab.sh +``` + +The runner gives both revisions the same fixed corpus, alternates `base โ†’ head` +and `head โ†’ base`, validates source/compiler/corpus/order provenance, requires +full normalized JSON-RPC response parity for parity-enabled tools, and reports +paired medians with a deterministic bootstrap interval. Increase to 20 or more +pairs for release claims. A parity exemption must be explicit in the benchmark +case, justified by genuinely nondeterministic output, and included in the +comparator's explicit allowlist. + +Single-run minima may be shown as diagnostics, but they are not acceptable +performance evidence and must not drive an acceptance claim. Do not publish +cached results as uncached performance. ## Review Expectations diff --git a/build.zig.zon b/build.zig.zon index 63178473..2be98c43 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .codedb2, .fingerprint = 0x6e18d96ca2a31757, - .version = "0.2.5829", + .version = "0.2.5830", .minimum_zig_version = "0.17.0-dev.813+2153f8143", .dependencies = .{ .nanoregex = .{ diff --git a/docs/bench-0.2.5830-paired-report.md b/docs/bench-0.2.5830-paired-report.md new file mode 100644 index 00000000..65617f4c --- /dev/null +++ b/docs/bench-0.2.5830-paired-report.md @@ -0,0 +1,40 @@ +# 0.2.5830 paired release benchmark + +- Date: 2026-07-12 +- Machine: Apple M3 Ultra Mac Studio, 28 cores, 256 GB, macOS 26.5.1 +- Production baseline: `dd36e9431925014ee2bed80346669a4afee7e42e` (`v0.2.5829`) +- Pinned parity-harness baseline: `24e89c70d4f9cdaf5542a78d83d1890a42b4a046` +- Candidate: `bc72bfca4f4dcb9316f2f00f0ca78ab18c4e3c74` (tree `2832e31eb08f7695518ad0d6c23967beb268d16b`) +- Compiler: Zig `0.17.0-dev.813+2153f8143`, executable SHA-256 `08abf236d78c05b8520431fbca99903ff98653b2f1cd1c3665a7f8c91247421c` +- Corpus source tree: `e705e2623b28d2456eb9d4934817b79f4de35216` +- Corpus: fixed 21-file set copied from `dd36e94`, plus generated `src/bench_target.zig`; fingerprint `13647708728832762745` +- Order: 20 counterbalanced AB/BA pairs, verified from per-sample pair/order/sequence metadata +- Regression gate: paired median >10% and >50,000 ns +- Parity gate: duration-normalized full JSON-RPC response hash rolled across every measured iteration +- Raw evidence archive: `bench-0.2.5830-paired-samples.tar.gz`, SHA-256 `104edc6875a9d121d2e5a4c1e69c12735f2c60128e48ae384aa8c6cfb950bc24` (attached to the GitHub Release) + +Corpus parity: **PASS** + +Source/compiler/order provenance: **PASS** + +| Tool | Base median | Head median | Paired delta | Abs delta | 95% CI | Head wins | Output parity | Status | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | +| `codedb_bundle` | 16980 | 12230 | -27.73% | -4790 | [-29.25%, -25.72%] | 20/20 | PASS | OK | +| `codedb_changes` | 2825 | 2830 | +1.77% | +50 | [-1.76%, +3.37%] | 7/20 (1 tie) | PASS | OK | +| `codedb_context` | 110990 | 97730 | -11.89% | -13230 | [-12.55%, -11.13%] | 20/20 | PASS | OK | +| `codedb_deps` | 60 | 60 | +0.00% | +0 | [+0.00%, +20.00%] | 5/20 (6 ties) | PASS | OK | +| `codedb_edit` | 51700 | 50900 | -2.04% | -1100 | [-5.09%, +2.52%] | 12/20 | PASS | OK | +| `codedb_find` | 600 | 590 | -3.27% | -20 | [-7.43%, +3.39%] | 11/20 (1 tie) | PASS | OK | +| `codedb_hot` | 4560 | 3750 | -17.81% | -820 | [-19.48%, -16.70%] | 20/20 | PASS | OK | +| `codedb_outline` | 5280 | 2115 | -59.79% | -3170 | [-60.91%, -59.05%] | 20/20 | PASS | OK | +| `codedb_read` | 14400 | 13630 | -4.81% | -685 | [-6.70%, -3.40%] | 18/20 | PASS | OK | +| `codedb_search` | 9770 | 9885 | +1.60% | +155 | [-1.02%, +2.76%] | 8/20 (1 tie) | PASS | OK | +| `codedb_snapshot` | 31225 | 31200 | -0.24% | -75 | [-0.88%, +0.24%] | 12/20 (1 tie) | SKIP | OK | +| `codedb_status` | 1815 | 1815 | -0.22% | -5 | [-2.69%, +1.91%] | 10/20 (2 ties) | SKIP | OK | +| `codedb_symbol` | 11780 | 2155 | -81.72% | -9625 | [-82.12%, -81.31%] | 20/20 | PASS | OK | +| `codedb_tree` | 5445 | 1780 | -67.60% | -3700 | [-68.49%, -66.73%] | 20/20 | PASS | OK | +| `codedb_word` | 3035 | 1915 | -37.16% | -1135 | [-38.46%, -35.19%] | 20/20 | PASS | OK | + +`codedb_snapshot` is exempt because output embeds its temporary destination path. `codedb_status` is exempt because it intentionally exposes cache counters. Both exemptions are declared in the benchmark cases, must match between revisions, and are accepted only through the comparator's explicit tool allowlist. + +No single-run minima were used. The confidence intervals are deterministic bootstrap intervals for the paired percentage median. diff --git a/docs/performance-0.2.5830.md b/docs/performance-0.2.5830.md new file mode 100644 index 00000000..e8f24a15 --- /dev/null +++ b/docs/performance-0.2.5830.md @@ -0,0 +1,225 @@ +# 0.2.5830 performance work + +This document records the first performance batch queued for the 0.2.5830 +release branch. More experiments may be added before the release is finalized. + +## Scope + +The pass targets steady-state daemon and MCP usage while retaining correctness on +cold and incremental paths. It includes: + +- rolling trigram construction for generic and incremental indexing; +- direct hash-index lookup for exact symbol names; +- generation-validated rendered-output caches for trees, outlines, and exact + word postings; +- a generation-validated fuzzy-file score cache; +- cached file hashes and newline offsets for repeated/deep reads; +- zero-copy, offset-based symbol-body extraction in `codedb_context`; +- a compact JSON-RPC payload fast path that retains CR/LF sanitization. + +No ranking formula, result cap, path policy, parser behavior, telemetry behavior, +or MCP response schema changes in this batch. + +## Benchmark methodology + +### MCP round trips + +The client-visible MCP measurements used: + +- baseline source: `dd36e94` (`v0.2.5829` release branch head); +- candidate source: the performance changes documented here; +- compiler: pinned Zig `0.17.0-dev.813+2153f8143` for both binaries; +- build mode: native `ReleaseFast`; +- corpus: immutable git archive, 730 files on disk / 633 indexed files; +- process protocol: newline-delimited JSON-RPC over MCP stdio; +- sampling: three counterbalanced baseline/candidate process pairs; +- per process/tool: five warmups and 50 measured calls; +- statistic: median across the three per-process medians. + +The benchmark includes request serialization, stdin/stdout transport, JSON-RPC +parsing, tool dispatch, MCP content-envelope generation, JSON escaping, response +writing, and client receipt. It therefore complements `zig build bench`, whose +latency field measures in-process dispatch only. + +| MCP tool | Baseline | Candidate | Speedup | +|---|---:|---:|---:| +| `codedb_tree` | 343.8 us | 40.1 us | **8.56x** | +| `codedb_outline` | 995.0 us | 148.9 us | **6.68x** | +| `codedb_symbol` | 229.4 us | 20.9 us | **10.96x** | +| `codedb_read` | 189.3 us | 53.9 us | **3.51x** | +| `codedb_find` | 5.52 ms | 409.2 us | **13.50x** | +| `codedb_word` | 13.83 ms | 215.0 us | **64.32x** | +| `codedb_search` | 72.1 us | 33.4 us | **2.16x** | +| `codedb_bundle` | 13.36 ms | 134.7 us | **99.11x** | + +Normalized MCP responses were byte-identical between baseline and candidate for +`tree`, `outline`, `symbol`, `read`, `find`, `word`, `search`, `context`, and +`bundle`. + +### Paired release gate + +The final source-attribution gate used production baseline `dd36e94` +(`v0.2.5829`) with pinned harness-only descendant `24e89c7`, and performance +candidate `bc72bfc`. Both binaries used the same pinned Zig compiler and the same +fixed 21-file benchmark corpus copied from `dd36e94`. Twenty +AB/BA-counterbalanced pairs passed: + +- shared corpus fingerprint: **PASS** in all pairs; +- source commit/tree, clean-worktree, compiler executable, corpus source, pair, + order, and sequence provenance: **PASS** in all samples; +- duration-normalized, full JSON-RPC response hash across every measured + iteration: **PASS** for every parity-enabled tool in all pairs; +- paired-median regression gate (>10% and >50us): **PASS**; +- tree, outline, symbol, word, hot, and bundle improved by 17.81%-81.72%; +- context, read, edit, and find improved by 11.89%, 4.81%, 2.04%, and 3.27%; +- no parity-enabled tool had a material regression. + +The complete persisted report is +[`bench-0.2.5830-paired-report.md`](bench-0.2.5830-paired-report.md). + +### Core edge cases + +`zig build bench-edge -- --json` used a generated 3,005-file corpus to expose +algorithmic and output-size scaling behavior. + +| Handler case | Baseline | Candidate | Speedup | +|---|---:|---:|---:| +| exact symbol | 42.9 us | 1.92 us | **22.4x** | +| huge outline | 187.4 us | 14.6 us | **12.8x** | +| deep ranged read | 306.7 us | 14.9 us | **20.6x** | +| 3,005-file tree | 760.3 us | 6.93 us | **109.7x** | +| fuzzy file find | 2.19 ms | 46.9 us | **46.6x** | +| large exact-word output | 572.7 us | 29.5 us | **19.4x** | + +### Indexing versus Zig 0.16 + +The generic initial-index benchmark was also compared directly with commit +`428d8df` built by Zig 0.16.0. Both binaries used the same immutable corpus, +c allocator, two workers, and 20 counterbalanced pairs. + +- Zig 0.16 median: 136.0 ms +- current Zig 0.17 median: 131.0 ms +- release-outcome improvement: **3.7%** +- paired wins: **20/20** +- paired median saved: 5.0 ms +- paired-mean bootstrap 95% interval: 4.6-8.2 ms saved + +All 14 benchmark query rows retained identical hit counts. + +## Implementation details + +### Exact symbols + +The common `{ "name": "..." }` symbol request now reads the existing symbol hash +index directly. Prefix, pattern, fuzzy, kind-only, incomplete-index fallback, +max-result, and deterministic ordering behavior remain on their prior paths. + +### Render caches + +Tree, outline, exact-word, and fuzzy-file results are keyed by the Explorer +mutation generation. Mutations make old entries ineligible before newly indexed +or removed content can be served. + +Memory is bounded: + +- tree entries are capped at 16 MiB each, with separate plain/color slots; +- outline and word-render caches use 32-entry, 16 MiB LRUs with a 4 MiB + per-entry ceiling; +- fuzzy-file scoring keeps at most 32 query/limit entries. + +Fuzzy paths borrow stable outline keys only while their generation is current. +Callers receive a copied match array, so combo boosts cannot mutate cached scores. + +### Reads and context bodies + +Cached file hashes avoid hashing a whole file on every ranged read. Hash entries +validate the canonical content pointer and length and are explicitly invalidated +on update/removal, guarding against allocator-address reuse. + +Deep ranges reuse newline-offset tables, making extraction proportional to the +requested range after the first table build rather than proportional to every +byte before the range. The context composer uses the same path for symbol bodies +and retains its disk fallback when content is not resident. + +### MCP framing + +Compact generated JSON-RPC payloads normally contain no raw CR/LF bytes. The MCP +writer now proves that with the standard library's vectorized search and copies +the payload in one slice. A non-canonical payload still uses the original +sanitizing fallback, preserving one-response-per-line framing. + +## Correctness and security checks + +The new tests cover: + +- exact scalar-versus-rolling trigram masks, including short, whitespace-only, + mixed-case, final-position, null, and `0xff` bytes; +- tree, outline, word, fuzzy-find, content-hash, and line-offset cache hits; +- mutation invalidation after additions, replacements, and removals; +- deep range boundaries and content replacement; +- output parity across baseline and candidate MCP responses. + +Validation completed for this batch: + +```text +zig fmt --check +zig build test +zig build -Doptimize=ReleaseFast +python3 scripts/e2e_mcp_test.py --binary zig-out/bin/codedb --project +zig build wasm +git diff --check +``` + +MCP E2E passed 20/20 scenarios, including roots negotiation, explicit-root, +no-roots, and direct-inline-argument modes. The final local release gate also +passed 20 paired AB/BA samples with a shared corpus and full normalized JSON-RPC +response parity; GitHub's paired five-pair gate passed from the immutable PR +base and published its report and raw samples as workflow artifacts. + +## Required protocol for follow-up performance changes + +Further optimization commits on this branch use the automated paired gate from +a clean committed head: + +```bash +CODEDB_BENCH_PAIRS=10 CODEDB_BENCH_OUT=zig-out/bench-ab \ + scripts/bench-ab.sh +``` + +For release claims, use at least 20 pairs. The runner: + +1. builds base and committed head in clean throwaway worktrees; +2. gives both binaries the same fixed 21-file corpus copied from the base; +3. verifies the corpus fingerprint for every pair; +4. records and validates commit/tree, compiler executable, corpus source, side, + pair, order, and sequence provenance; +5. alternates `base -> head` and `head -> base` order; +6. requires duration-normalized full JSON-RPC response-hash parity across every + measured iteration for parity-enabled tools, with explicit comparator-owned + exemptions only; +7. reports paired medians, head win counts, and a deterministic bootstrap 95% + interval; +8. rejects regressions from the paired median rather than a single-run minimum. + +Raw samples and the Markdown report remain in `CODEDB_BENCH_OUT` when it is set. +Cross-compiler release-outcome comparisons remain separate from same-compiler +source-attribution runs. + +## Limits and follow-up work + +These numbers are workload-specific, and the largest cache-backed gains are +steady-state gains. First calls still construct the cached representation. +Exact-symbol lookup is an algorithmic improvement on both cold and warm paths. + +Not every operation is 2-3x faster yet. Remaining targets for later commits on +this branch include: + +- uncached ranked, regex, and high-candidate content search; +- edit/write/reparse/reindex latency; +- general context composition beyond deep body extraction; +- cold tree/outline construction; +- benchmark separation of handler-only, in-process MCP, and transport-level + latency in CI. + +Performance changes should continue to require output/hit parity and paired, +counterbalanced measurements rather than single-run minima. diff --git a/docs/release-notes-0.2.5830.md b/docs/release-notes-0.2.5830.md new file mode 100644 index 00000000..915bfa42 --- /dev/null +++ b/docs/release-notes-0.2.5830.md @@ -0,0 +1,50 @@ +# codedb 0.2.5830 + +codedb 0.2.5830 accelerates steady-state MCP and core read-only tools while preserving retrieval and client-visible response behavior. + +## Highlights + +- Generation-validated caches speed repeated tree, outline, exact-word, and fuzzy-file requests. +- Exact symbol lookup uses the existing symbol hash index directly. +- Cached content hashes and newline offsets accelerate repeated and deep reads. +- Context symbol bodies use offset-based extraction with the disk fallback retained. +- Rolling trigram construction reduces repeated byte loads and normalization. +- Compact JSON-RPC payloads use a fast copy path while CR/LF sanitization remains in place. +- Cache reads are synchronized with watcher mutations, and cache identity uses entry generations rather than allocator addresses. + +No ranking formula, result cap, parser behavior, path-security policy, telemetry behavior, or MCP response schema changes in this release. + +## Performance and parity + +The final release gate uses 20 counterbalanced AB/BA pairs against production baseline `v0.2.5829`, with a pinned harness-only backport, the same fixed 21-file benchmark corpus, and pinned Zig `0.17.0-dev.813+2153f8143` for both binaries. + +- shared corpus fingerprint: **PASS** +- auditable source/tree/compiler/corpus/pair/order/sequence provenance: **PASS** +- duration-normalized full JSON-RPC response parity across every measured iteration: **PASS** for every parity-enabled tool +- paired regression gate (>10% and >50us): **PASS** + +The largest in-process paired-median improvements are exact symbol **81.72%**, tree **67.60%**, outline **59.79%**, word **37.16%**, bundle **27.73%**, and hot-file lookup **17.81%**. Full methodology, transport-level measurements, limitations, and the persisted paired report are in [`docs/performance-0.2.5830.md`](https://github.com/justrach/codedb/blob/v0.2.5830/docs/performance-0.2.5830.md). + +## Verification + +- `zig build test` +- native `ReleaseFast` build +- Windows x86_64 cross-build +- WASM build +- MCP E2E: 20/20 scenarios +- paired comparator unit tests +- 20-pair local release gate +- GitHub paired benchmark workflow +- `git diff --check` + +## Release assets + +The release workflow publishes binaries for: + +- macOS arm64 +- macOS x86_64 +- Linux arm64 +- Linux x86_64 +- Windows x86_64 + +Verify downloads with the attached `checksums.sha256`. diff --git a/npm/package.json b/npm/package.json index 78a11d5d..69c7b460 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "codedeebee", - "version": "0.2.5829", + "version": "0.2.5830", "description": "Zig code intelligence MCP server โ€” npx launcher for the codedb native binary", "license": "MIT", "author": "justrach", diff --git a/scripts/bench-ab.sh b/scripts/bench-ab.sh index b6116cbe..7ab7dc6f 100755 --- a/scripts/bench-ab.sh +++ b/scripts/bench-ab.sh @@ -1,36 +1,112 @@ #!/usr/bin/env bash -# A/B the gated MCP tool bench: current working tree vs a base ref. +# Paired, counterbalanced A/B for the gated MCP tool benchmark. # -# scripts/bench-ab.sh # base = HEAD (measure uncommitted work) -# scripts/bench-ab.sh HEAD~3 # base = any ref -# scripts/bench-ab.sh origin/release/0.2.5828 +# scripts/bench-ab.sh # compare clean committed HEAD in isolated worktrees +# CODEDB_BENCH_PAIRS=20 scripts/bench-ab.sh HEAD~3 +# CODEDB_BENCH_OUT=bench-results scripts/bench-ab.sh origin/release/0.2.5830 # -# Builds the base ref in a throwaway worktree under $HOME (tests and bench -# misbehave under /tmp roots โ€” see root_policy), runs `zig build bench -- -# --json` for both sides on this machine back-to-back, and prints the same -# regression table CI posts on PRs (scripts/compare-bench.py, 10% + 50ยตs -# thresholds). Only same-machine, same-run deltas are meaningful โ€” never -# compare against a JSON from another day or box. +# Both revisions index the same fixed 21-file corpus copied from the base +# worktree, and every parity-enabled tool must emit the same response hash. Execution +# order alternates AB/BA. The report uses paired medians + bootstrap intervals; +# no single-run minimum is accepted as performance evidence. set -euo pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" BASE_REF="${1:-HEAD}" +PAIRS="${CODEDB_BENCH_PAIRS:-10}" +if ! [[ "$PAIRS" =~ ^[0-9]+$ ]] || (( PAIRS < 2 )); then + echo "CODEDB_BENCH_PAIRS must be an integer >= 2" >&2 + exit 2 +fi + BASE_SHA="$(git -C "$REPO_ROOT" rev-parse --short "$BASE_REF")" +BASE_FULL_SHA="$(git -C "$REPO_ROOT" rev-parse "$BASE_REF")" +HEAD_FULL_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)" +HEAD_TREE_SHA="$(git -C "$REPO_ROOT" rev-parse 'HEAD^{tree}')" +PAIRED_5829_BASE_SHA="dd36e9431925014ee2bed80346669a4afee7e42e" +PAIRED_5829_SHA="24e89c70d4f9cdaf5542a78d83d1890a42b4a046" WT="$HOME/.cache/codedb-bench-ab-$$" -OUT="$(mktemp -d "${TMPDIR:-/tmp}/codedb-bench-ab.XXXXXX")" +BENCH_WT="$WT" +HEAD_WT="$WT-head" +if [[ -n "${CODEDB_BENCH_OUT:-}" ]]; then + OUT="$CODEDB_BENCH_OUT" + mkdir -p "$OUT" + OUT="$(cd "$OUT" && pwd -P)" + KEEP_OUT=1 +else + OUT="$(mktemp -d "${TMPDIR:-/tmp}/codedb-bench-ab.XXXXXX")" + KEEP_OUT=0 +fi cleanup() { - git -C "$REPO_ROOT" worktree remove --force "$WT" >/dev/null 2>&1 || true - rm -rf "$OUT" + if [[ -d "$HEAD_WT" ]]; then + git -C "$REPO_ROOT" worktree remove "$HEAD_WT" >/dev/null 2>&1 || \ + echo "warning: benchmark worktree remains at $HEAD_WT" >&2 + fi + if [[ "$BENCH_WT" != "$WT" && -d "$BENCH_WT" ]]; then + git -C "$REPO_ROOT" worktree remove "$BENCH_WT" >/dev/null 2>&1 || \ + echo "warning: benchmark worktree remains at $BENCH_WT" >&2 + fi + if [[ -d "$WT" ]]; then + git -C "$REPO_ROOT" worktree remove "$WT" >/dev/null 2>&1 || \ + echo "warning: benchmark worktree remains at $WT" >&2 + fi + if (( KEEP_OUT == 0 )); then rm -rf "$OUT"; fi } trap cleanup EXIT -echo "base: $BASE_REF ($BASE_SHA) in throwaway worktree" -echo "head: working tree at $(git -C "$REPO_ROOT" rev-parse --short HEAD)$(git -C "$REPO_ROOT" diff --quiet || echo ' + uncommitted changes')" +echo "base: $BASE_REF ($BASE_SHA) in throwaway worktree" +echo "head: committed $HEAD_FULL_SHA in throwaway worktree" +echo "pairs: $PAIRS (AB/BA counterbalanced)" +echo "corpus: fixed 21-file set copied from the base worktree" git -C "$REPO_ROOT" worktree add --detach "$WT" "$BASE_REF" >/dev/null -(cd "$WT" && python3 scripts/run-bench-json.py "$OUT/base.json" >/dev/null) -(cd "$REPO_ROOT" && python3 scripts/run-bench-json.py "$OUT/head.json" >/dev/null) +if ! grep -q 'corpus_hash' "$WT/src/bench.zig"; then + if [[ "$BASE_FULL_SHA" != "$PAIRED_5829_BASE_SHA" ]]; then + echo "base ref lacks paired/parity benchmark schema and has no pinned harness" >&2 + exit 2 + fi + git -C "$REPO_ROOT" cat-file -e "$PAIRED_5829_SHA^{commit}" 2>/dev/null || { + git -C "$REPO_ROOT" fetch origin bench/v0.2.5829-paired-baseline --depth=1 + [[ "$(git -C "$REPO_ROOT" rev-parse FETCH_HEAD)" == "$PAIRED_5829_SHA" ]] + } + git -C "$REPO_ROOT" merge-base --is-ancestor "$BASE_FULL_SHA" "$PAIRED_5829_SHA" + [[ "$(git -C "$REPO_ROOT" diff --name-only "$BASE_FULL_SHA..$PAIRED_5829_SHA")" == $'src/bench.zig\nsrc/mcp.zig' ]] + BENCH_WT="$WT-paired" + git -C "$REPO_ROOT" worktree add --detach "$BENCH_WT" "$PAIRED_5829_SHA" >/dev/null +fi + +git -C "$REPO_ROOT" worktree add --detach "$HEAD_WT" "$HEAD_FULL_SHA" >/dev/null + +run_sample() { + local side="$1" pair="$2" order="$3" sequence="$4" cwd production_sha + if [[ "$side" == "base" ]]; then + cwd="$BENCH_WT" + production_sha="$BASE_FULL_SHA" + else + cwd="$HEAD_WT" + production_sha="$HEAD_FULL_SHA" + fi + echo "pair $pair/$PAIRS: $side ($order sequence $sequence)" >&2 + (cd "$cwd" && python3 "$HEAD_WT/scripts/run-bench-json.py" \ + --bench-side "$side" --bench-pair "$pair" --bench-order "$order" --bench-sequence "$sequence" \ + --production-source-sha "$production_sha" --corpus-source-sha "$BASE_FULL_SHA" \ + "$OUT/$side-$(printf '%02d' "$pair").json" --corpus-source "$WT" >/dev/null) +} + +for ((pair = 1; pair <= PAIRS; pair++)); do + if (( pair % 2 == 1 )); then + run_sample base "$pair" AB 1 + run_sample head "$pair" AB 2 + else + run_sample head "$pair" BA 1 + run_sample base "$pair" BA 2 + fi +done + +python3 "$HEAD_WT/scripts/compare-bench-paired.py" "$OUT" \ + --require-parity --require-provenance --allow-parity-skip codedb_snapshot --allow-parity-skip codedb_status \ + --expected-head-sha "$HEAD_FULL_SHA" --expected-head-tree-sha "$HEAD_TREE_SHA" \ + --threshold-pct 10 --min-abs-ns 50000 --markdown-out "$OUT/report.md" -python3 "$REPO_ROOT/scripts/compare-bench.py" "$OUT/base.json" "$OUT/head.json" \ - --threshold-pct 10 --min-abs-ns 50000 +if (( KEEP_OUT == 1 )); then echo "raw samples: $OUT"; fi diff --git a/scripts/compare-bench-paired.py b/scripts/compare-bench-paired.py new file mode 100755 index 00000000..0874f750 --- /dev/null +++ b/scripts/compare-bench-paired.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Compare counterbalanced codedb benchmark samples with output parity gates.""" +from __future__ import annotations + +import argparse +import json +import random +import re +import statistics +import sys +from pathlib import Path + + +PINNED_BASELINE_HARNESSES = { + "24e89c70d4f9cdaf5542a78d83d1890a42b4a046": "dd36e9431925014ee2bed80346669a4afee7e42e", +} +PINNED_COMMIT_TREES = { + "24e89c70d4f9cdaf5542a78d83d1890a42b4a046": "e0012e49b5819b8ac800831d7e0dce6a84bca1a1", + "dd36e9431925014ee2bed80346669a4afee7e42e": "e705e2623b28d2456eb9d4934817b79f4de35216", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("samples", help="directory containing base-NN.json and head-NN.json") + parser.add_argument("--threshold-pct", type=float, default=10.0) + parser.add_argument("--min-abs-ns", type=int, default=50_000) + parser.add_argument("--require-parity", action="store_true") + parser.add_argument("--require-provenance", action="store_true") + parser.add_argument("--allow-parity-skip", action="append", default=[], metavar="TOOL") + parser.add_argument("--expected-head-sha") + parser.add_argument("--expected-head-tree-sha") + parser.add_argument("--markdown-out") + parser.add_argument("--bootstrap-samples", type=int, default=20_000) + return parser.parse_args() + + +def load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def paired_files(root: Path) -> list[tuple[Path, Path]]: + bases = {p.stem.removeprefix("base-"): p for p in root.glob("base-*.json")} + heads = {p.stem.removeprefix("head-"): p for p in root.glob("head-*.json")} + if not bases or set(bases) != set(heads): + raise ValueError(f"unpaired samples: base={sorted(bases)} head={sorted(heads)}") + return [(bases[key], heads[key]) for key in sorted(bases)] + + +def percentile(values: list[float], q: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, int(q * len(ordered)))) + return ordered[index] + + +def bootstrap_median_ci(values: list[float], samples: int, seed: int = 0xC0DE) -> tuple[float, float]: + if not values: + return (0.0, 0.0) + rng = random.Random(seed) + medians = [] + for _ in range(samples): + draw = [values[rng.randrange(len(values))] for _ in values] + medians.append(float(statistics.median(draw))) + return percentile(medians, 0.025), percentile(medians, 0.975) + + +def tool_map(data: dict) -> dict[str, dict]: + tools = data["tools"] + mapped = {tool["tool"]: tool for tool in tools} + if len(mapped) != len(tools): + raise ValueError("duplicate tool records in benchmark sample") + return mapped + + +def validate_provenance( + samples: list[tuple[dict, dict]], + expected_head_sha: str | None = None, + expected_head_tree_sha: str | None = None, +) -> tuple[list[str], str | None]: + failures: list[str] = [] + base_sources: set[str] = set() + head_sources: set[str] = set() + base_production_sources: set[str] = set() + base_source_trees: set[str] = set() + head_source_trees: set[str] = set() + compilers: set[str] = set() + compiler_hashes: set[str] = set() + corpus_sources: set[str] = set() + corpus_trees: set[str] = set() + sha_pattern = re.compile(r"^[0-9a-f]{40}$") + + for pair_index, (base, head) in enumerate(samples, 1): + expected_order = "AB" if pair_index % 2 == 1 else "BA" + expected_sequences = {"base": 1 if expected_order == "AB" else 2, "head": 2 if expected_order == "AB" else 1} + for side, data in (("base", base), ("head", head)): + meta = data.get("benchmark_provenance") + if not isinstance(meta, dict): + failures.append(f"pair {pair_index} {side}: benchmark_provenance missing") + continue + expected = { + "side": side, + "pair": pair_index, + "order": expected_order, + "sequence": expected_sequences[side], + "build_mode": "ReleaseFast", + } + for field, value in expected.items(): + if meta.get(field) != value: + failures.append(f"pair {pair_index} {side}: {field}={meta.get(field)!r}, expected {value!r}") + if meta.get("source_dirty") is not False: + failures.append(f"pair {pair_index} {side}: source tree was dirty") + if meta.get("corpus_source_dirty") is not False: + failures.append(f"pair {pair_index} {side}: corpus source tree was dirty") + for field in ("source_sha", "source_tree_sha", "production_source_sha", "corpus_source_sha", "corpus_source_tree_sha"): + value = meta.get(field) + if not isinstance(value, str) or not sha_pattern.fullmatch(value): + failures.append(f"pair {pair_index} {side}: invalid {field}") + compiler = meta.get("compiler_version") + if not isinstance(compiler, str) or not compiler: + failures.append(f"pair {pair_index} {side}: compiler_version missing") + else: + compilers.add(compiler) + compiler_hash = meta.get("compiler_sha256") + if not isinstance(compiler_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", compiler_hash): + failures.append(f"pair {pair_index} {side}: invalid compiler_sha256") + else: + compiler_hashes.add(compiler_hash) + source_sha = meta.get("source_sha") + production_sha = meta.get("production_source_sha") + corpus_sha = meta.get("corpus_source_sha") + source_tree = meta.get("source_tree_sha") + corpus_tree = meta.get("corpus_source_tree_sha") + if isinstance(corpus_sha, str): + corpus_sources.add(corpus_sha) + if isinstance(corpus_tree, str): + corpus_trees.add(corpus_tree) + if source_sha in PINNED_COMMIT_TREES and source_tree != PINNED_COMMIT_TREES[source_sha]: + failures.append(f"pair {pair_index} {side}: source tree does not match pinned commit") + if corpus_sha in PINNED_COMMIT_TREES and corpus_tree != PINNED_COMMIT_TREES[corpus_sha]: + failures.append(f"pair {pair_index} {side}: corpus tree does not match pinned commit") + if side == "base": + if isinstance(source_sha, str): + base_sources.add(source_sha) + if isinstance(source_tree, str): + base_source_trees.add(source_tree) + if isinstance(production_sha, str): + base_production_sources.add(production_sha) + expected_production = PINNED_BASELINE_HARNESSES.get(source_sha, source_sha) + if production_sha != expected_production: + failures.append(f"pair {pair_index} base: unrecognized production/harness source mapping") + else: + if isinstance(source_sha, str): + head_sources.add(source_sha) + if isinstance(source_tree, str): + head_source_trees.add(source_tree) + if source_sha != production_sha: + failures.append(f"pair {pair_index} head: production_source_sha differs from built source_sha") + + for label, values in ( + ("base harness source", base_sources), + ("base production source", base_production_sources), + ("base source tree", base_source_trees), + ("head source", head_sources), + ("head source tree", head_source_trees), + ("compiler version", compilers), + ("compiler executable hash", compiler_hashes), + ("corpus source", corpus_sources), + ("corpus source tree", corpus_trees), + ): + if len(values) != 1: + failures.append(f"{label} changed across samples: {sorted(values)}") + if len(base_production_sources) == 1 and corpus_sources != base_production_sources: + failures.append("corpus source does not match the production baseline source") + if (expected_head_sha is None) != (expected_head_tree_sha is None): + failures.append("expected head SHA and tree SHA must be provided together") + if expected_head_sha is not None: + if not sha_pattern.fullmatch(expected_head_sha) or not sha_pattern.fullmatch(expected_head_tree_sha or ""): + failures.append("expected head SHA/tree is invalid") + if head_sources != {expected_head_sha}: + failures.append(f"head source does not match expected {expected_head_sha}") + if head_source_trees != {expected_head_tree_sha}: + failures.append(f"head source tree does not match expected {expected_head_tree_sha}") + + if failures: + return failures, None + summary = ( + f"base={next(iter(base_production_sources))[:12]} " + f"harness={next(iter(base_sources))[:12]} " + f"head={next(iter(head_sources))[:12]} " + f"compiler={next(iter(compilers))}/{next(iter(compiler_hashes))[:12]}" + ) + return [], summary + + +def compare( + samples: list[tuple[dict, dict]], + threshold_pct: float, + min_abs_ns: int, + require_parity: bool, + bootstrap_samples: int, + require_provenance: bool = False, + allowed_parity_skips: set[str] | None = None, + expected_head_sha: str | None = None, + expected_head_tree_sha: str | None = None, +) -> tuple[str, list[str]]: + failures: list[str] = [] + parity_failures: list[str] = [] + corpus_hashes: list[str] = [] + allowed_parity_skips = allowed_parity_skips or set() + provenance_failures, provenance_summary = ( + validate_provenance(samples, expected_head_sha, expected_head_tree_sha) + if require_provenance + else ([], None) + ) + + mapped: list[tuple[dict[str, dict], dict[str, dict]]] = [] + expected_tools: set[str] | None = None + for pair_index, (base, head) in enumerate(samples, 1): + base_hash = base.get("corpus_hash") + head_hash = head.get("corpus_hash") + if base_hash is None or head_hash is None: + if require_parity: + parity_failures.append(f"pair {pair_index}: benchmark schema lacks corpus_hash") + elif base_hash != head_hash: + parity_failures.append(f"pair {pair_index}: corpus hash differs ({base_hash} != {head_hash})") + else: + corpus_hashes.append(str(base_hash)) + + base_tools = tool_map(base) + head_tools = tool_map(head) + if set(base_tools) != set(head_tools): + parity_failures.append( + f"pair {pair_index}: tool set differs (base-only={sorted(set(base_tools)-set(head_tools))}, head-only={sorted(set(head_tools)-set(base_tools))})" + ) + common = set(base_tools) & set(head_tools) + if expected_tools is None: + expected_tools = common + elif common != expected_tools: + parity_failures.append(f"pair {pair_index}: common tool set changed across samples") + mapped.append((base_tools, head_tools)) + + if corpus_hashes and len(set(corpus_hashes)) != 1: + parity_failures.append(f"corpus hash changed across pairs: {sorted(set(corpus_hashes))}") + + rows = [] + for tool in sorted(expected_tools or ()): + base_ns = [int(base[tool]["avg_latency_ns"]) for base, _ in mapped] + head_ns = [int(head[tool]["avg_latency_ns"]) for _, head in mapped] + deltas = [h - b for b, h in zip(base_ns, head_ns)] + delta_pcts = [((h - b) / b * 100.0) if b else 0.0 for b, h in zip(base_ns, head_ns)] + base_median = int(statistics.median(base_ns)) + head_median = int(statistics.median(head_ns)) + delta_median = int(statistics.median(deltas)) + pct_median = float(statistics.median(delta_pcts)) + ci_low, ci_high = bootstrap_median_ci(delta_pcts, bootstrap_samples, seed=0xC0DE + len(rows)) + wins = sum(h < b for b, h in zip(base_ns, head_ns)) + ties = sum(h == b for b, h in zip(base_ns, head_ns)) + + parity_policies = { + (bool(base[tool].get("parity", False)), bool(head[tool].get("parity", False))) + for base, head in mapped + } + parity_enabled = parity_policies == {(True, True)} + parity_status = "SKIP" + if parity_enabled: + missing = any("response_hash" not in base[tool] or "response_hash" not in head[tool] for base, head in mapped) + mismatches = [ + index + for index, (base, head) in enumerate(mapped, 1) + if base[tool].get("response_hash") != head[tool].get("response_hash") + ] + if missing: + parity_status = "MISSING" + if require_parity: + parity_failures.append(f"{tool}: response_hash missing") + elif mismatches: + parity_status = "FAIL" + parity_failures.append(f"{tool}: output hash differs in pairs {mismatches}") + else: + parity_status = "PASS" + elif require_parity: + if parity_policies != {(False, False)}: + parity_failures.append(f"{tool}: parity policy differs between revisions or samples") + elif tool not in allowed_parity_skips: + parity_failures.append(f"{tool}: parity disabled without an explicit comparator allowlist entry") + + status = "OK" + if pct_median > threshold_pct and delta_median > min_abs_ns: + status = "FAIL" + failures.append(f"{tool} median paired regression {pct_median:+.2f}% ({delta_median:+d} ns)") + elif pct_median > threshold_pct: + status = "NOISE" + rows.append((tool, base_median, head_median, pct_median, delta_median, ci_low, ci_high, wins, ties, parity_status, status)) + + actual_parity_skips = {row[0] for row in rows if row[9] == "SKIP"} + if require_parity: + for tool in sorted(allowed_parity_skips - actual_parity_skips): + parity_failures.append(f"{tool}: parity skip allowlist entry was not used") + if parity_failures: + failures.extend(parity_failures) + if provenance_failures: + failures.extend(provenance_failures) + + counterbalance = "verified" if require_provenance and not provenance_failures else "not provenance-verified" + lines = [ + "## Paired Benchmark Report", + "", + f"Pairs: {len(samples)} (counterbalance {counterbalance})", + f"Regression gate: median paired delta > {threshold_pct:.2f}% and > {min_abs_ns:,} ns", + f"Corpus parity: {'PASS' if corpus_hashes and not any('corpus' in failure for failure in parity_failures) else 'FAIL' if parity_failures else 'UNAVAILABLE'}", + f"Provenance: {'PASS (' + provenance_summary + ')' if provenance_summary else 'FAIL' if require_provenance else 'NOT REQUIRED'}", + "", + "No single-run minima are used. CI is a deterministic bootstrap 95% interval for the paired percentage median.", + "", + "| Tool | Base median | Head median | Paired delta | Abs delta | 95% CI | Head wins | Output parity | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + ] + for tool, base_median, head_median, pct_median, delta_median, ci_low, ci_high, wins, ties, parity_status, status in rows: + lines.append( + f"| `{tool}` | {base_median} | {head_median} | {pct_median:+.2f}% | {delta_median:+d} | [{ci_low:+.2f}%, {ci_high:+.2f}%] | {wins}/{len(samples)} ({ties} ties) | {parity_status} | {status} |" + ) + if parity_failures: + lines.extend(["", "### Parity failures", ""] + [f"- {failure}" for failure in parity_failures]) + if provenance_failures: + lines.extend(["", "### Provenance failures", ""] + [f"- {failure}" for failure in provenance_failures]) + return "\n".join(lines) + "\n", failures + + +def main() -> int: + args = parse_args() + try: + files = paired_files(Path(args.samples)) + samples = [(load(base), load(head)) for base, head in files] + report, failures = compare( + samples, + args.threshold_pct, + args.min_abs_ns, + args.require_parity, + args.bootstrap_samples, + require_provenance=args.require_provenance, + allowed_parity_skips=set(args.allow_parity_skip), + expected_head_sha=args.expected_head_sha, + expected_head_tree_sha=args.expected_head_tree_sha, + ) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + sys.stdout.write(report) + if args.markdown_out: + Path(args.markdown_out).write_text(report, encoding="utf-8") + for failure in failures: + print(failure, file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-bench-json.py b/scripts/run-bench-json.py index 12a50b7e..9cb8e346 100644 --- a/scripts/run-bench-json.py +++ b/scripts/run-bench-json.py @@ -2,14 +2,24 @@ from __future__ import annotations import argparse +import hashlib +import json import pathlib +import shutil import subprocess import sys def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run `zig build bench -- --json` and persist the JSON payload.") + parser.add_argument("--bench-side", choices=("base", "head")) + parser.add_argument("--bench-pair", type=int) + parser.add_argument("--bench-order", choices=("AB", "BA")) + parser.add_argument("--bench-sequence", type=int, choices=(1, 2)) + parser.add_argument("--production-source-sha") + parser.add_argument("--corpus-source-sha") parser.add_argument("output", help="output JSON file") + parser.add_argument("bench_args", nargs=argparse.REMAINDER, help="arguments forwarded to the benchmark after --json") return parser.parse_args() @@ -26,10 +36,83 @@ def extract_json(stdout: str, stderr: str) -> str: raise RuntimeError("benchmark command did not emit JSON") +def command_output(*command: str) -> str: + return subprocess.run(command, check=True, capture_output=True, text=True).stdout.strip() + + +def file_sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def bench_arg_value(args: argparse.Namespace, name: str) -> str | None: + try: + index = args.bench_args.index(name) + except ValueError: + return None + if index + 1 >= len(args.bench_args): + raise ValueError(f"{name} requires a value") + return args.bench_args[index + 1] + + +def git_is_clean(path: pathlib.Path) -> bool: + status = command_output("git", "-C", str(path), "status", "--porcelain", "--untracked-files=all") + return status == "" + + +def benchmark_provenance(args: argparse.Namespace) -> dict | None: + fields = ( + args.bench_side, + args.bench_pair, + args.bench_order, + args.bench_sequence, + args.production_source_sha, + args.corpus_source_sha, + ) + if not any(value is not None for value in fields): + return None + if any(value is None for value in fields): + raise ValueError("paired benchmark provenance arguments must be provided together") + source_root = pathlib.Path.cwd().resolve() + corpus_arg = bench_arg_value(args, "--corpus-source") + if corpus_arg is None: + raise ValueError("paired benchmark provenance requires --corpus-source") + corpus_root = pathlib.Path(corpus_arg).resolve() + source_sha = command_output("git", "-C", str(source_root), "rev-parse", "HEAD") + corpus_source_sha = command_output("git", "-C", str(corpus_root), "rev-parse", "HEAD") + if corpus_source_sha != args.corpus_source_sha: + raise ValueError( + f"corpus source HEAD {corpus_source_sha} does not match claimed {args.corpus_source_sha}" + ) + zig_command = shutil.which("zig") + if zig_command is None: + raise ValueError("zig executable not found on PATH") + zig_path = pathlib.Path(zig_command).resolve() + return { + "side": args.bench_side, + "pair": args.bench_pair, + "order": args.bench_order, + "sequence": args.bench_sequence, + "source_sha": source_sha, + "source_tree_sha": command_output("git", "-C", str(source_root), "rev-parse", "HEAD^{tree}"), + "source_dirty": not git_is_clean(source_root), + "production_source_sha": args.production_source_sha, + "corpus_source_sha": corpus_source_sha, + "corpus_source_tree_sha": command_output("git", "-C", str(corpus_root), "rev-parse", "HEAD^{tree}"), + "corpus_source_dirty": not git_is_clean(corpus_root), + "compiler_version": command_output(str(zig_path), "version"), + "compiler_sha256": file_sha256(zig_path), + "build_mode": "ReleaseFast", + } + + def main() -> int: args = parse_args() proc = subprocess.run( - ["zig", "build", "bench", "--", "--json"], + ["zig", "build", "bench", "--", "--json", *args.bench_args], capture_output=True, text=True, check=False, @@ -43,8 +126,11 @@ def main() -> int: sys.stderr.write("---- stdout ----\n") sys.stderr.write(proc.stdout) return proc.returncode - payload = extract_json(proc.stdout, proc.stderr) - pathlib.Path(args.output).write_text(payload, encoding="utf-8") + payload = json.loads(extract_json(proc.stdout, proc.stderr)) + provenance = benchmark_provenance(args) + if provenance is not None: + payload["benchmark_provenance"] = provenance + pathlib.Path(args.output).write_text(json.dumps(payload, separators=(",", ":")) + "\n", encoding="utf-8") return 0 diff --git a/scripts/test_compare_bench_paired.py b/scripts/test_compare_bench_paired.py new file mode 100755 index 00000000..047b6e3f --- /dev/null +++ b/scripts/test_compare_bench_paired.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).with_name("compare-bench-paired.py") +SPEC = importlib.util.spec_from_file_location("compare_bench_paired", MODULE_PATH) +assert SPEC and SPEC.loader +paired = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(paired) + + +def payload(latency: int, response_hash: int = 7, corpus_hash: int = 11, parity: bool = True) -> dict: + return { + "corpus_hash": corpus_hash, + "tools": [ + { + "tool": "codedb_tree", + "avg_latency_ns": latency, + "response_hash": response_hash, + "parity": parity, + } + ], + } + + +def add_provenance(data: dict, side: str, pair: int, order: str, sequence: int) -> dict: + source_sha = ( + "24e89c70d4f9cdaf5542a78d83d1890a42b4a046" if side == "base" else "c" * 40 + ) + source_tree_sha = ( + "e0012e49b5819b8ac800831d7e0dce6a84bca1a1" if side == "base" else "e" * 40 + ) + production_sha = ( + "dd36e9431925014ee2bed80346669a4afee7e42e" if side == "base" else source_sha + ) + data["benchmark_provenance"] = { + "side": side, + "pair": pair, + "order": order, + "sequence": sequence, + "source_sha": source_sha, + "source_tree_sha": source_tree_sha, + "source_dirty": False, + "production_source_sha": production_sha, + "corpus_source_sha": "dd36e9431925014ee2bed80346669a4afee7e42e", + "corpus_source_tree_sha": "e705e2623b28d2456eb9d4934817b79f4de35216", + "corpus_source_dirty": False, + "compiler_version": "0.17.0-test", + "compiler_sha256": "d" * 64, + "build_mode": "ReleaseFast", + } + return data + + +class PairedComparisonTests(unittest.TestCase): + def test_uses_paired_median_not_single_minimum(self) -> None: + samples = [ + (payload(100), payload(10)), + (payload(100), payload(120)), + (payload(100), payload(130)), + ] + report, failures = paired.compare(samples, threshold_pct=10, min_abs_ns=0, require_parity=True, bootstrap_samples=500) + self.assertIn("+20.00%", report) + self.assertIn("1/3", report) + self.assertTrue(any("median paired regression" in failure for failure in failures)) + + def test_output_hash_mismatch_is_a_failure(self) -> None: + report, failures = paired.compare( + [(payload(100, response_hash=1), payload(90, response_hash=2))], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + ) + self.assertIn("Output parity", report) + self.assertIn("FAIL", report) + self.assertTrue(any("output hash differs" in failure for failure in failures)) + + def test_corpus_mismatch_is_a_failure(self) -> None: + _, failures = paired.compare( + [(payload(100, corpus_hash=1), payload(90, corpus_hash=2))], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + ) + self.assertTrue(any("corpus hash differs" in failure for failure in failures)) + + def test_parity_opt_out_is_explicit(self) -> None: + report, failures = paired.compare( + [(payload(100, response_hash=1, parity=False), payload(90, response_hash=2, parity=False))], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + allowed_parity_skips={"codedb_tree"}, + ) + self.assertIn("SKIP", report) + self.assertEqual([], failures) + + def test_unallowlisted_parity_opt_out_fails(self) -> None: + _, failures = paired.compare( + [(payload(100, parity=False), payload(90, parity=False))], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + ) + self.assertTrue(any("without an explicit comparator allowlist" in failure for failure in failures)) + + def test_legacy_schema_is_reported_but_allowed_without_requirement(self) -> None: + legacy = {"tools": [{"tool": "codedb_tree", "avg_latency_ns": 100}]} + report, failures = paired.compare( + [(legacy, legacy)], + threshold_pct=10, + min_abs_ns=0, + require_parity=False, + bootstrap_samples=100, + ) + self.assertIn("Corpus parity: UNAVAILABLE", report) + self.assertEqual([], failures) + + def test_corpus_change_across_pairs_is_a_failure(self) -> None: + _, failures = paired.compare( + [ + (payload(100, corpus_hash=1), payload(90, corpus_hash=1)), + (payload(100, corpus_hash=2), payload(90, corpus_hash=2)), + ], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + ) + self.assertTrue(any("corpus hash changed across pairs" in failure for failure in failures)) + + def test_counterbalance_provenance_is_verified(self) -> None: + samples = [ + ( + add_provenance(payload(100), "base", 1, "AB", 1), + add_provenance(payload(90), "head", 1, "AB", 2), + ), + ( + add_provenance(payload(100), "base", 2, "BA", 2), + add_provenance(payload(90), "head", 2, "BA", 1), + ), + ] + report, failures = paired.compare( + samples, + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + require_provenance=True, + expected_head_sha="c" * 40, + expected_head_tree_sha="e" * 40, + ) + self.assertIn("Provenance: PASS", report) + self.assertEqual([], failures) + + def test_incorrect_counterbalance_provenance_fails(self) -> None: + samples = [ + ( + add_provenance(payload(100), "base", 1, "BA", 1), + add_provenance(payload(90), "head", 1, "BA", 2), + ) + ] + _, failures = paired.compare( + samples, + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + require_provenance=True, + ) + self.assertTrue(any("order='BA', expected 'AB'" in failure for failure in failures)) + + def test_expected_head_identity_mismatch_fails(self) -> None: + samples = [ + ( + add_provenance(payload(100), "base", 1, "AB", 1), + add_provenance(payload(90), "head", 1, "AB", 2), + ) + ] + _, failures = paired.compare( + samples, + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + require_provenance=True, + expected_head_sha="f" * 40, + expected_head_tree_sha="e" * 40, + ) + self.assertTrue(any("head source does not match expected" in failure for failure in failures)) + + def test_duplicate_tool_records_are_rejected(self) -> None: + duplicate = payload(100) + duplicate["tools"].append(dict(duplicate["tools"][0])) + with self.assertRaisesRegex(ValueError, "duplicate tool records"): + paired.compare( + [(duplicate, payload(90))], + threshold_pct=10, + min_abs_ns=0, + require_parity=True, + bootstrap_samples=100, + ) + + def test_unpaired_sample_files_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "base-01.json").write_text("{}", encoding="utf-8") + with self.assertRaises(ValueError): + paired.paired_files(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench.zig b/src/bench.zig index f5b6b355..7e3da34e 100644 --- a/src/bench.zig +++ b/src/bench.zig @@ -11,6 +11,8 @@ const ToolBench = struct { tool: []const u8, avg_latency_ns: u64, response_bytes: usize, + response_hash: u64, + parity: bool, ops_per_sec: f64, telemetry_avg_ns: u64, telemetry_delta_pct: f64, @@ -21,6 +23,8 @@ const Case = struct { name: []const u8, args_json: []const u8, iterations: usize, + /// False only when output intentionally embeds per-run nondeterminism. + parity: bool = true, }; const cases = [_]Case{ @@ -34,13 +38,40 @@ const cases = [_]Case{ .{ .tool = .codedb_read, .name = "codedb_read", .args_json = "{\"path\":\"src/main.zig\",\"line_start\":1,\"line_end\":20}", .iterations = 100 }, .{ .tool = .codedb_edit, .name = "codedb_edit", .args_json = "{\"path\":\"src/bench_target.zig\",\"op\":\"replace\",\"range_start\":1,\"range_end\":1,\"content\":\"pub const bench_value = 2;\\n\"}", .iterations = 10 }, .{ .tool = .codedb_changes, .name = "codedb_changes", .args_json = "{\"since\":0}", .iterations = 100 }, - .{ .tool = .codedb_status, .name = "codedb_status", .args_json = "{}", .iterations = 100 }, - .{ .tool = .codedb_snapshot, .name = "codedb_snapshot", .args_json = "{}", .iterations = 20 }, + // Status intentionally exposes cache counters, which may differ when the + // candidate removes work while returning the same query results. + .{ .tool = .codedb_status, .name = "codedb_status", .args_json = "{}", .iterations = 100, .parity = false }, + // Snapshot output embeds its per-run temporary destination path. + .{ .tool = .codedb_snapshot, .name = "codedb_snapshot", .args_json = "{}", .iterations = 20, .parity = false }, .{ .tool = .codedb_bundle, .name = "codedb_bundle", .args_json = "{\"ops\":[{\"tool\":\"codedb_outline\",\"arguments\":{\"path\":\"src/main.zig\"}},{\"tool\":\"codedb_search\",\"arguments\":{\"query\":\"telemetry\",\"max_results\":5}},{\"tool\":\"codedb_word\",\"arguments\":{\"word\":\"Telemetry\"}}]}", .iterations = 50 }, .{ .tool = .codedb_find, .name = "codedb_find", .args_json = "{\"query\":\"main\"}", .iterations = 100 }, .{ .tool = .codedb_context, .name = "codedb_context", .args_json = "{\"task\":\"trace recordToolCall execution path through writePositionalAll and the SpinLock acquisition in Telemetry โ€” what is the hot path\"}", .iterations = 50 }, }; +const corpus_files = [_][]const u8{ + "README.md", + "build.zig", + "build.zig.zon", + "src/agent.zig", + "src/bench.zig", + "src/edit.zig", + "src/explore.zig", + "src/git.zig", + "src/index.zig", + "src/lib.zig", + "src/main.zig", + "src/mcp.zig", + "src/root_policy.zig", + "src/server.zig", + "src/snapshot.zig", + "src/snapshot_json.zig", + "src/store.zig", + "src/style.zig", + "src/telemetry.zig", + "src/version.zig", + "src/watcher.zig", +}; + pub fn main(init: std.process.Init.Minimal) !void { cio.setProcessArgs(cio.bootstrapArgs(init.args)); var gpa: std.heap.DebugAllocator(.{}) = .init; @@ -51,14 +82,21 @@ pub fn main(init: std.process.Init.Minimal) !void { defer threaded.deinit(); const io = threaded.io(); - const emit_json = blk: { - const args = try cio.argsAlloc(allocator); - defer cio.argsFree(allocator, args); - for (args[1..]) |arg| { - if (std.mem.eql(u8, arg, "--json")) break :blk true; + const process_args = try cio.argsAlloc(allocator); + defer cio.argsFree(allocator, process_args); + var emit_json = false; + var corpus_source_arg: ?[]const u8 = null; + var arg_index: usize = 1; + while (arg_index < process_args.len) : (arg_index += 1) { + const arg = process_args[arg_index]; + if (std.mem.eql(u8, arg, "--json")) { + emit_json = true; + } else if (std.mem.eql(u8, arg, "--corpus-source")) { + arg_index += 1; + if (arg_index >= process_args.len) return error.MissingCorpusSource; + corpus_source_arg = process_args[arg_index]; } - break :blk false; - }; + } var tmp_path_buf: [std.fs.max_path_bytes]u8 = undefined; const tmp_root = try makeTempCorpusDir(io, &tmp_path_buf); @@ -67,8 +105,10 @@ pub fn main(init: std.process.Init.Minimal) !void { var repo_path_buf: [std.fs.max_path_bytes]u8 = undefined; const repo_root_len = try std.Io.Dir.cwd().realPathFile(io, ".", &repo_path_buf); const repo_root = repo_path_buf[0..repo_root_len]; + const corpus_source = corpus_source_arg orelse repo_root; + const corpus_hash = try hashCorpus(io, allocator, corpus_source); - try copyCorpus(io, allocator, repo_root, tmp_root); + try copyCorpus(io, allocator, corpus_source, tmp_root); try writeBenchTarget(io, tmp_root); var store = Store.init(allocator); @@ -110,6 +150,8 @@ pub fn main(init: std.process.Init.Minimal) !void { .tool = case.name, .avg_latency_ns = base.avg_latency_ns, .response_bytes = base.response_bytes, + .response_hash = base.response_hash, + .parity = case.parity, .ops_per_sec = opsPerSec(base.avg_latency_ns), .telemetry_avg_ns = with_telem.avg_latency_ns, .telemetry_delta_pct = deltaPct(base.avg_latency_ns, with_telem.avg_latency_ns), @@ -119,7 +161,7 @@ pub fn main(init: std.process.Init.Minimal) !void { const corpus = summarizeCorpus(&explorer); try writeHumanSummary(allocator, cio.File.stderr(), corpus.files, corpus.bytes, &results); if (emit_json) { - try writeJsonSummary(allocator, cio.File.stdout(), repo_root, tmp_root, corpus.files, corpus.bytes, &results); + try writeJsonSummary(allocator, cio.File.stdout(), repo_root, tmp_root, corpus_hash, corpus.files, corpus.bytes, &results); } } @@ -133,9 +175,10 @@ fn runCase( case: Case, args: *const std.json.ObjectMap, telem: *telemetry.Telemetry, -) !struct { avg_latency_ns: u64, response_bytes: usize } { +) !struct { avg_latency_ns: u64, response_bytes: usize, response_hash: u64 } { var total_ns: u64 = 0; var response_bytes: usize = 0; + var response_hash: u64 = 0; for (0..case.iterations) |_| { if (case.tool == .codedb_edit) { @@ -145,40 +188,19 @@ fn runCase( const r = bench_ctx.runToolCall(io, allocator, case.name, case.tool, args, store, explorer, agents, telem); total_ns +|= r.dispatch_ns; response_bytes = r.response_bytes; + var iteration_hash = r.response_hash; + response_hash = std.hash.Wyhash.hash(response_hash, std.mem.asBytes(&iteration_hash)); } return .{ .avg_latency_ns = @intCast(@divTrunc(total_ns, case.iterations)), .response_bytes = response_bytes, + .response_hash = response_hash, }; } fn copyCorpus(io: std.Io, allocator: std.mem.Allocator, repo_root: []const u8, tmp_root: []const u8) !void { - const files = [_][]const u8{ - "README.md", - "build.zig", - "build.zig.zon", - "src/agent.zig", - "src/bench.zig", - "src/edit.zig", - "src/explore.zig", - "src/git.zig", - "src/index.zig", - "src/lib.zig", - "src/main.zig", - "src/mcp.zig", - "src/root_policy.zig", - "src/server.zig", - "src/snapshot.zig", - "src/snapshot_json.zig", - "src/store.zig", - "src/style.zig", - "src/telemetry.zig", - "src/version.zig", - "src/watcher.zig", - }; - - for (files) |rel| { + for (corpus_files) |rel| { const src = try std.fs.path.join(allocator, &.{ repo_root, rel }); defer allocator.free(src); const dst = try std.fs.path.join(allocator, &.{ tmp_root, rel }); @@ -192,6 +214,20 @@ fn copyCorpus(io: std.Io, allocator: std.mem.Allocator, repo_root: []const u8, t } } +fn hashCorpus(io: std.Io, allocator: std.mem.Allocator, corpus_root: []const u8) !u64 { + var hash = std.hash.Wyhash.init(0x434f_4445_4442); + for (corpus_files) |rel| { + const src = try std.fs.path.join(allocator, &.{ corpus_root, rel }); + defer allocator.free(src); + const content = try std.Io.Dir.cwd().readFileAlloc(io, src, allocator, .limited(64 * 1024 * 1024)); + defer allocator.free(content); + hash.update(rel); + hash.update(&[_]u8{0}); + hash.update(content); + } + return hash.final(); +} + fn makeTempCorpusDir(io: std.Io, buf: *[std.fs.max_path_bytes]u8) ![]const u8 { const base = cio.posixGetenv("TMPDIR") orelse "/tmp"; const ns = cio.nanoTimestamp(); @@ -234,7 +270,7 @@ fn writeHumanSummary(allocator: std.mem.Allocator, file: cio.File, file_count: u var out: std.ArrayList(u8) = .empty; defer out.deinit(allocator); const writer = cio.listWriter(&out, allocator); - try writer.print("โ”€โ”€ E2E MCP Tool Benchmarks ({d} files, {d}KB) โ”€โ”€\n", .{ file_count, total_bytes / 1024 }); + try writer.print("โ”€โ”€ In-process MCP Handler Benchmarks ({d} files, {d}KB) โ”€โ”€\n", .{ file_count, total_bytes / 1024 }); try writer.writeAll("Tool Latency Size Ops/sec Telemetryฮ”\n"); for (results) |result| { var latency_buf: [32]u8 = undefined; @@ -250,22 +286,25 @@ fn writeHumanSummary(allocator: std.mem.Allocator, file: cio.File, file_count: u try file.writeAll(out.items); } -fn writeJsonSummary(allocator: std.mem.Allocator, file: cio.File, repo_root: []const u8, corpus_root: []const u8, file_count: usize, total_bytes: u64, results: []const ToolBench) !void { +fn writeJsonSummary(allocator: std.mem.Allocator, file: cio.File, repo_root: []const u8, corpus_root: []const u8, corpus_hash: u64, file_count: usize, total_bytes: u64, results: []const ToolBench) !void { var out: std.ArrayList(u8) = .empty; defer out.deinit(allocator); const writer = cio.listWriter(&out, allocator); - try writer.print("{{\"repo_root\":\"{s}\",\"corpus_root\":\"{s}\",\"file_count\":{d},\"total_bytes\":{d},\"tools\":[", .{ + try writer.print("{{\"repo_root\":\"{s}\",\"corpus_root\":\"{s}\",\"corpus_hash\":{d},\"file_count\":{d},\"total_bytes\":{d},\"tools\":[", .{ repo_root, corpus_root, + corpus_hash, file_count, total_bytes, }); for (results, 0..) |result, idx| { if (idx > 0) try writer.writeByte(','); - try writer.print("{{\"tool\":\"{s}\",\"avg_latency_ns\":{d},\"response_bytes\":{d},\"ops_per_sec\":{d:.3},\"telemetry_avg_ns\":{d},\"telemetry_delta_pct\":{d:.3}}}", .{ + try writer.print("{{\"tool\":\"{s}\",\"avg_latency_ns\":{d},\"response_bytes\":{d},\"response_hash\":{d},\"parity\":{},\"ops_per_sec\":{d:.3},\"telemetry_avg_ns\":{d},\"telemetry_delta_pct\":{d:.3}}}", .{ result.tool, result.avg_latency_ns, result.response_bytes, + result.response_hash, + result.parity, result.ops_per_sec, result.telemetry_avg_ns, result.telemetry_delta_pct, diff --git a/src/explore.zig b/src/explore.zig index 8e017169..110e5a14 100644 --- a/src/explore.zig +++ b/src/explore.zig @@ -812,6 +812,62 @@ const LexFreqPenalty = struct { } }; +/// Per-file content hashes for cached reads. Entries are keyed by path and +/// validated against the content-cache entry generation; mutation paths also +/// invalidate explicitly so allocator reuse cannot serve a stale `if_hash` result. +const ContentHashCache = struct { + const Entry = struct { + content_generation: u64, + hash: u64, + }; + + map: std.StringHashMap(Entry), + mu: cio.Mutex = .{}, + + fn init(allocator: std.mem.Allocator) ContentHashCache { + return .{ .map = std.StringHashMap(Entry).init(allocator) }; + } + + fn deinit(self: *ContentHashCache) void { + var iter = self.map.keyIterator(); + while (iter.next()) |key| self.map.allocator.free(key.*); + self.map.deinit(); + } + + fn get(self: *ContentHashCache, path: []const u8, content: []const u8, content_generation: u64, cacheable: bool) u64 { + if (!cacheable) return std.hash.Wyhash.hash(0, content); + self.mu.lock(); + defer self.mu.unlock(); + if (self.map.getPtr(path)) |entry| { + if (entry.content_generation == content_generation) return entry.hash; + const hash = std.hash.Wyhash.hash(0, content); + entry.* = .{ .content_generation = content_generation, .hash = hash }; + return hash; + } + + const hash = std.hash.Wyhash.hash(0, content); + const key = self.map.allocator.dupe(u8, path) catch return hash; + self.map.put(key, .{ .content_generation = content_generation, .hash = hash }) catch { + self.map.allocator.free(key); + }; + return hash; + } + + fn clear(self: *ContentHashCache) void { + self.mu.lock(); + defer self.mu.unlock(); + var iter = self.map.keyIterator(); + while (iter.next()) |key| self.map.allocator.free(key.*); + self.map.clearRetainingCapacity(); + } + + fn invalidate(self: *ContentHashCache, path: []const u8) void { + self.mu.lock(); + defer self.mu.unlock(); + if (self.map.fetchRemove(path)) |removed| self.map.allocator.free(removed.key); + } +}; + /// Per-file newline-offset tables so Tier 0's line-number โ†’ line-text lookups /// skip rescanning file bytes on every query. Entries self-validate against /// the content slice (ptr+len) they were built from and are invalidated on @@ -892,6 +948,35 @@ const LineOffsetCache = struct { }; } + /// Return the current offset table for `path`, rebuilding stale entries. + /// Caller must hold `mu`. + fn offsetsForLocked(self: *LineOffsetCache, path: []const u8, content: []const u8) ?[]const u32 { + if (self.map.getPtr(path)) |entry| { + if (entry.content_ptr == @intFromPtr(content.ptr) and entry.content_len == content.len) { + return entry.offsets; + } + const fresh = buildOffsets(self.map.allocator, content) orelse return null; + self.total_bytes -= entry.offsets.len * @sizeOf(u32); + self.map.allocator.free(entry.offsets); + entry.* = .{ .content_ptr = @intFromPtr(content.ptr), .content_len = content.len, .offsets = fresh }; + self.total_bytes += fresh.len * @sizeOf(u32); + return fresh; + } + + const fresh = buildOffsets(self.map.allocator, content) orelse return null; + const key = self.map.allocator.dupe(u8, path) catch { + self.map.allocator.free(fresh); + return null; + }; + self.map.put(key, .{ .content_ptr = @intFromPtr(content.ptr), .content_len = content.len, .offsets = fresh }) catch { + self.map.allocator.free(fresh); + self.map.allocator.free(key); + return null; + }; + self.total_bytes += fresh.len * @sizeOf(u32); + return fresh; + } + /// Resolve ascending 1-based `target_lines` to byte spans in `content`, /// building (and caching) the offset table for `path` on first touch. /// Span semantics match std.mem.splitScalar(content, '\n'): a line ends @@ -901,32 +986,7 @@ const LineOffsetCache = struct { fn lineSpans(self: *LineOffsetCache, path: []const u8, content: []const u8, target_lines: []const u32, spans: []Span) ?usize { self.mu.lock(); defer self.mu.unlock(); - var offsets: []const u32 = undefined; - if (self.map.getPtr(path)) |e| { - if (e.content_ptr == @intFromPtr(content.ptr) and e.content_len == content.len) { - offsets = e.offsets; - } else { - const fresh = buildOffsets(self.map.allocator, content) orelse return null; - self.total_bytes -= e.offsets.len * @sizeOf(u32); - self.map.allocator.free(e.offsets); - e.* = .{ .content_ptr = @intFromPtr(content.ptr), .content_len = content.len, .offsets = fresh }; - self.total_bytes += fresh.len * @sizeOf(u32); - offsets = fresh; - } - } else { - const fresh = buildOffsets(self.map.allocator, content) orelse return null; - const key = self.map.allocator.dupe(u8, path) catch { - self.map.allocator.free(fresh); - return null; - }; - self.map.put(key, .{ .content_ptr = @intFromPtr(content.ptr), .content_len = content.len, .offsets = fresh }) catch { - self.map.allocator.free(fresh); - self.map.allocator.free(key); - return null; - }; - self.total_bytes += fresh.len * @sizeOf(u32); - offsets = fresh; - } + const offsets = self.offsetsForLocked(path, content) orelse return null; var n: usize = 0; for (target_lines) |ln| { @@ -941,6 +1001,39 @@ const LineOffsetCache = struct { if (self.total_bytes > MAX_BYTES) self.clearLocked(); return n; } + + /// Render a contiguous line range without scanning bytes before `start`. + /// Returns false only when the offset table could not be allocated. + fn appendRange( + self: *LineOffsetCache, + path: []const u8, + content: []const u8, + start: u32, + end: u32, + compact: bool, + language: Language, + line_prefix: []const u8, + allocator: std.mem.Allocator, + out: *std.ArrayList(u8), + ) !bool { + self.mu.lock(); + defer self.mu.unlock(); + const offsets = self.offsetsForLocked(path, content) orelse return false; + if (start == 0 or start > offsets.len or end < start) return true; + + const last: u32 = @intCast(@min(@as(usize, end), offsets.len)); + const writer = cio.listWriter(out, allocator); + var line_num = start; + while (line_num <= last) : (line_num += 1) { + const byte_start: usize = offsets[line_num - 1]; + const byte_end: usize = if (line_num < offsets.len) offsets[line_num] - 1 else content.len; + const line = content[byte_start..byte_end]; + if (compact and isCommentOrBlank(line, language)) continue; + try writer.print("{s}{d:>5} | {s}\n", .{ line_prefix, line_num, line }); + } + if (self.total_bytes > MAX_BYTES) self.clearLocked(); + return true; + } }; /// Whole-query result cache for searchContent: agents re-issue identical @@ -1266,6 +1359,238 @@ const PlainRenderCache = struct { } }; +pub const FuzzyFileMatch = struct { + path: []const u8, + score: f32, +}; + +/// Generation-keyed cache for fuzzy file scoring. Paths borrow stable outline +/// keys; callers receive a copied match array so combo boosts remain per-call. +const FuzzyFileCache = struct { + const Entry = struct { + query: []u8, + max_results: usize, + gen: SearchGeneration, + matches: []FuzzyFileMatch, + last_used: u64, + }; + + allocator: std.mem.Allocator, + entries: std.ArrayList(Entry) = .empty, + mu: cio.Mutex = .{}, + tick: u64 = 0, + + const MAX_ENTRIES: usize = 32; + + fn init(allocator: std.mem.Allocator) FuzzyFileCache { + return .{ .allocator = allocator }; + } + + fn freeEntry(self: *FuzzyFileCache, entry: *Entry) void { + self.allocator.free(entry.query); + self.allocator.free(entry.matches); + } + + fn deinit(self: *FuzzyFileCache) void { + for (self.entries.items) |*entry| self.freeEntry(entry); + self.entries.deinit(self.allocator); + } + + fn get(self: *FuzzyFileCache, query: []const u8, max_results: usize, gen: SearchGeneration, out_allocator: std.mem.Allocator) ?[]FuzzyFileMatch { + self.mu.lock(); + defer self.mu.unlock(); + for (self.entries.items) |*entry| { + if (entry.max_results != max_results or entry.gen != gen or !std.mem.eql(u8, entry.query, query)) continue; + const copy = out_allocator.dupe(FuzzyFileMatch, entry.matches) catch return null; + self.tick += 1; + entry.last_used = self.tick; + return copy; + } + return null; + } + + fn put(self: *FuzzyFileCache, query: []const u8, max_results: usize, gen: SearchGeneration, matches: []const FuzzyFileMatch) void { + const query_copy = self.allocator.dupe(u8, query) catch return; + const matches_copy = self.allocator.dupe(FuzzyFileMatch, matches) catch { + self.allocator.free(query_copy); + return; + }; + self.mu.lock(); + defer self.mu.unlock(); + var i: usize = 0; + while (i < self.entries.items.len) { + const entry = &self.entries.items[i]; + if (entry.max_results == max_results and std.mem.eql(u8, entry.query, query)) { + var dead = self.entries.swapRemove(i); + self.freeEntry(&dead); + continue; + } + i += 1; + } + if (self.entries.items.len >= MAX_ENTRIES) { + var lru: usize = 0; + for (self.entries.items, 0..) |entry, index| { + if (entry.last_used < self.entries.items[lru].last_used) lru = index; + } + var dead = self.entries.swapRemove(lru); + self.freeEntry(&dead); + } + self.tick += 1; + self.entries.append(self.allocator, .{ + .query = query_copy, + .max_results = max_results, + .gen = gen, + .matches = matches_copy, + .last_used = self.tick, + }) catch { + self.allocator.free(query_copy); + self.allocator.free(matches_copy); + }; + } +}; + +/// Generation-keyed render cache for `codedb_tree`. Tree construction sorts every +/// indexed path and rebuilds the directory set, while the result changes only on +/// an Explorer mutation. Keep separate plain/color entries and copy cached bytes +/// directly into the caller's response buffer. +const TreeRenderCache = struct { + const Entry = struct { + gen: SearchGeneration, + bytes: []u8, + }; + + allocator: std.mem.Allocator, + entries: [2]?Entry = .{ null, null }, + mu: cio.Mutex = .{}, + + fn init(allocator: std.mem.Allocator) TreeRenderCache { + return .{ .allocator = allocator }; + } + + fn deinit(self: *TreeRenderCache) void { + for (&self.entries) |*entry| { + if (entry.*) |e| self.allocator.free(e.bytes); + entry.* = null; + } + } + + fn render(self: *TreeRenderCache, gen: SearchGeneration, use_color: bool, out_allocator: std.mem.Allocator, out: *std.ArrayList(u8)) bool { + self.mu.lock(); + defer self.mu.unlock(); + const index: usize = @intFromBool(use_color); + const entry = self.entries[index] orelse return false; + if (entry.gen != gen) return false; + out.appendSlice(out_allocator, entry.bytes) catch return false; + return true; + } + + fn put(self: *TreeRenderCache, gen: SearchGeneration, use_color: bool, bytes: []const u8) void { + if (bytes.len > 16 * 1024 * 1024) return; + const copy = self.allocator.dupe(u8, bytes) catch return; + self.mu.lock(); + defer self.mu.unlock(); + const index: usize = @intFromBool(use_color); + if (self.entries[index]) |old| self.allocator.free(old.bytes); + self.entries[index] = .{ .gen = gen, .bytes = copy }; + } +}; + +/// Small generation-keyed cache for rendered outlines. Large outlines are +/// expensive to format repeatedly but cheap to copy into an MCP response. +const OutlineRenderCache = struct { + const Entry = struct { + path: []u8, + compact: bool, + gen: SearchGeneration, + bytes: []u8, + last_used: u64, + }; + + allocator: std.mem.Allocator, + entries: std.ArrayList(Entry) = .empty, + mu: cio.Mutex = .{}, + tick: u64 = 0, + total_bytes: usize = 0, + + const MAX_ENTRIES: usize = 32; + const MAX_BYTES: usize = 16 * 1024 * 1024; + const MAX_ENTRY_BYTES: usize = 4 * 1024 * 1024; + + fn init(allocator: std.mem.Allocator) OutlineRenderCache { + return .{ .allocator = allocator }; + } + + fn freeEntry(self: *OutlineRenderCache, entry: *Entry) void { + self.allocator.free(entry.path); + self.allocator.free(entry.bytes); + } + + fn deinit(self: *OutlineRenderCache) void { + for (self.entries.items) |*entry| self.freeEntry(entry); + self.entries.deinit(self.allocator); + } + + fn render(self: *OutlineRenderCache, path: []const u8, compact: bool, gen: SearchGeneration, out_allocator: std.mem.Allocator, out: *std.ArrayList(u8)) bool { + self.mu.lock(); + defer self.mu.unlock(); + for (self.entries.items) |*entry| { + if (entry.compact != compact or entry.gen != gen or !std.mem.eql(u8, entry.path, path)) continue; + out.appendSlice(out_allocator, entry.bytes) catch return false; + self.tick += 1; + entry.last_used = self.tick; + return true; + } + return false; + } + + fn put(self: *OutlineRenderCache, path: []const u8, compact: bool, gen: SearchGeneration, bytes: []const u8) void { + if (bytes.len > MAX_ENTRY_BYTES) return; + const path_copy = self.allocator.dupe(u8, path) catch return; + const bytes_copy = self.allocator.dupe(u8, bytes) catch { + self.allocator.free(path_copy); + return; + }; + + self.mu.lock(); + defer self.mu.unlock(); + var i: usize = 0; + while (i < self.entries.items.len) { + const entry = &self.entries.items[i]; + if (entry.compact == compact and std.mem.eql(u8, entry.path, path)) { + var dead = self.entries.swapRemove(i); + self.total_bytes -= dead.bytes.len; + self.freeEntry(&dead); + continue; + } + i += 1; + } + while (self.entries.items.len >= MAX_ENTRIES or + (self.entries.items.len > 0 and self.total_bytes + bytes_copy.len > MAX_BYTES)) + { + var lru: usize = 0; + for (self.entries.items, 0..) |entry, index| { + if (entry.last_used < self.entries.items[lru].last_used) lru = index; + } + var dead = self.entries.swapRemove(lru); + self.total_bytes -= dead.bytes.len; + self.freeEntry(&dead); + } + self.tick += 1; + self.entries.append(self.allocator, .{ + .path = path_copy, + .compact = compact, + .gen = gen, + .bytes = bytes_copy, + .last_used = self.tick, + }) catch { + self.allocator.free(path_copy); + self.allocator.free(bytes_copy); + return; + }; + self.total_bytes += bytes_copy.len; + } +}; + /// Fingerprint of the env kill-switches that change ranking/search output โ€” /// part of the SearchResultCache key so toggling one (tests do this /// mid-process) can never serve results computed under the other setting. @@ -1292,6 +1617,7 @@ pub const Explorer = struct { outlines: std.StringHashMap(FileOutline), dep_graph: DependencyGraph, contents: ContentCache, + content_hashes: ContentHashCache, line_offsets: LineOffsetCache, symbol_index: std.StringHashMap(std.ArrayList(SymbolLocation)), /// False after a snapshot fast-load until ensureSymbolIndex runs (#564). @@ -1352,6 +1678,14 @@ pub const Explorer = struct { ranked_cache: SearchResultCache, /// Rendered-output cache for renderPlainSearch (same validation). plain_render_cache: PlainRenderCache, + /// Rendered tree output, keyed by mutation generation and color mode. + tree_render_cache: TreeRenderCache, + /// Rendered outlines, keyed by path, compact mode, and mutation generation. + outline_render_cache: OutlineRenderCache, + /// Fuzzy file-score results, keyed by query, limit, and mutation generation. + fuzzy_file_cache: FuzzyFileCache, + /// Rendered exact-word postings, using an independent path-keyed LRU. + word_render_cache: OutlineRenderCache, /// Bumped (atomically โ€” searches run under the SHARED lock) by every /// mutation that can change search results; see bumpSearchGen callers. search_gen: std.atomic.Value(SearchGeneration) = std.atomic.Value(SearchGeneration).init(0), @@ -1390,10 +1724,15 @@ pub const Explorer = struct { .outlines = std.StringHashMap(FileOutline).init(allocator), .dep_graph = DependencyGraph.init(allocator), .contents = try ContentCache.initAlloc(allocator, content_cache_capacity), + .content_hashes = ContentHashCache.init(allocator), .line_offsets = LineOffsetCache.init(allocator), .search_cache = SearchResultCache.init(allocator), .ranked_cache = SearchResultCache.init(allocator), .plain_render_cache = PlainRenderCache.init(allocator), + .tree_render_cache = TreeRenderCache.init(allocator), + .outline_render_cache = OutlineRenderCache.init(allocator), + .fuzzy_file_cache = FuzzyFileCache.init(allocator), + .word_render_cache = OutlineRenderCache.init(allocator), .symbol_index = std.StringHashMap(std.ArrayList(SymbolLocation)).init(allocator), .symbol_index_complete = true, .word_index = WordIndex.init(allocator), @@ -1421,10 +1760,15 @@ pub const Explorer = struct { self.symbol_index.deinit(); self.contents.deinit(); + self.content_hashes.deinit(); self.line_offsets.deinit(); self.search_cache.deinit(); self.ranked_cache.deinit(); self.plain_render_cache.deinit(); + self.tree_render_cache.deinit(); + self.outline_render_cache.deinit(); + self.fuzzy_file_cache.deinit(); + self.word_render_cache.deinit(); if (self.call_centrality) |*c| c.deinit(); if (self.call_graph) |*cg| cg.deinit(self.allocator); if (self.co_change) |*cc| git.freeCoChange(cc, self.allocator); @@ -1477,6 +1821,7 @@ pub const Explorer = struct { self.mu.lock(); defer self.mu.unlock(); self.contents.clear(); + self.content_hashes.clear(); self.line_offsets.clear(); } @@ -1611,6 +1956,7 @@ pub const Explorer = struct { // Last fallible step: put frees the prior cache value in place, so it // must run only once nothing after it can still need prior_content. try self.contents.put(stable_path, content); + self.content_hashes.invalidate(stable_path); self.line_offsets.invalidate(stable_path); outline_gop.value_ptr.* = persistent_outline; @@ -2309,6 +2655,7 @@ pub const Explorer = struct { self.removeSymbolIndexFor(path); _ = self.skip_trigram_files.remove(path); self.contents.remove(path); + self.content_hashes.invalidate(path); self.line_offsets.invalidate(path); self.word_index.removeFile(path); self.trigram_index.removeFile(path); @@ -2344,6 +2691,10 @@ pub const Explorer = struct { self.mu.lockShared(); defer self.mu.unlockShared(); + const gen = self.search_gen.load(.acquire); + if (self.outline_render_cache.render(path, compact, gen, alloc, out)) return true; + const render_start = out.items.len; + const outline = self.outlines.getPtr(path) orelse return false; try out.ensureUnusedCapacity(alloc, 128 + outline.symbols.items.len * 128); const w = cio.listWriter(out, alloc); @@ -2359,6 +2710,7 @@ pub const Explorer = struct { w.writeAll("\n") catch {}; } } + self.outline_render_cache.put(path, compact, gen, out.items[render_start..]); return true; } @@ -2478,6 +2830,20 @@ pub const Explorer = struct { return self.line_offsets.lineSpans(path, content, target_lines, spans); } + /// Append a contiguous line range without copying cached content or scanning + /// from byte zero. A cache miss retains the prior disk-read fallback. + pub fn appendLineRange(self: *Explorer, path: []const u8, start: u32, end: u32, line_prefix: []const u8, allocator: std.mem.Allocator, out: *std.ArrayList(u8)) !bool { + self.mu.lockShared(); + defer self.mu.unlockShared(); + const content_ref = self.readContentForSearch(path, allocator) orelse return false; + defer content_ref.deinit(); + if (content_ref.owned) { + try appendExtractedLines(content_ref.data, start, end, true, false, .unknown, line_prefix, allocator, out); + return true; + } + return self.line_offsets.appendRange(path, content_ref.data, start, end, false, .unknown, line_prefix, allocator, out); + } + pub const ReadRenderOptions = struct { if_hash: ?[]const u8 = null, line_start: ?i64 = null, @@ -2498,8 +2864,9 @@ pub const Explorer = struct { self.mu.lockShared(); defer self.mu.unlockShared(); - const content = self.contents.get(path) orelse return false; - try renderReadBytes(path, content, allocator, out, opts); + const content_ref = self.contents.getWithGeneration(path) orelse return false; + const hash = self.content_hashes.get(path, content_ref.value, content_ref.generation, content_ref.value_owned); + try self.renderReadBytes(path, content_ref.value, hash, allocator, out, opts); return true; } @@ -2538,8 +2905,10 @@ pub const Explorer = struct { } fn renderReadBytes( + self: *Explorer, path: []const u8, content: []const u8, + hash: u64, allocator: std.mem.Allocator, out: *std.ArrayList(u8), opts: ReadRenderOptions, @@ -2547,13 +2916,11 @@ pub const Explorer = struct { const probe_len = @min(content.len, 8 * 1024); if (std.mem.indexOfScalar(u8, content[0..probe_len], 0) != null) { const w0 = cio.listWriter(out, allocator); - const hash_b = std.hash.Wyhash.hash(0, content); - try w0.print("binary file: {d} bytes hash:{x}\n", .{ content.len, hash_b }); + try w0.print("binary file: {d} bytes hash:{x}\n", .{ content.len, hash }); return; } try out.ensureUnusedCapacity(allocator, if (opts.line_start != null or opts.line_end != null or opts.compact) 2048 else @min(content.len + 64, 64 * 1024)); - const hash = std.hash.Wyhash.hash(0, content); var hash_buf: [16]u8 = undefined; const hash_str = std.fmt.bufPrint(&hash_buf, "{x}", .{hash}) catch ""; if (opts.if_hash) |prev| { @@ -2572,7 +2939,9 @@ pub const Explorer = struct { const start: u32 = if (opts.line_start) |n| @intCast(@min(@max(1, n), std.math.maxInt(u32))) else 1; const end: u32 = if (opts.line_end) |n| @intCast(@min(@max(1, n), std.math.maxInt(u32))) else std.math.maxInt(u32); const lang = detectLanguage(path); - try appendExtractedLines(content, start, end, true, opts.compact, lang, allocator, out); + if (!try self.line_offsets.appendRange(path, content, start, end, opts.compact, lang, "", allocator, out)) { + try appendExtractedLines(content, start, end, true, opts.compact, lang, "", allocator, out); + } } else { if (fullFileReadHint(content)) |hint| try out.appendSlice(allocator, hint); try out.appendSlice(allocator, content); @@ -2705,11 +3074,14 @@ pub const Explorer = struct { /// into the caller's buffer. Halves the allocation churn on the /// MCP codedb_tree path. pub fn renderTree(self: *Explorer, allocator: std.mem.Allocator, out: *std.ArrayList(u8), use_color: bool) !void { - const s = @import("style.zig").style(use_color); - self.mu.lockShared(); defer self.mu.unlockShared(); + const gen = self.search_gen.load(.acquire); + if (self.tree_render_cache.render(gen, use_color, allocator, out)) return; + const render_start = out.items.len; + const s = @import("style.zig").style(use_color); + const writer = cio.listWriter(out, allocator); var paths: std.ArrayList([]const u8) = .empty; @@ -2762,6 +3134,7 @@ pub const Explorer = struct { s.reset, }); } + self.tree_render_cache.put(gen, use_color, out.items[render_start..]); } pub fn findSymbol(self: *Explorer, name: []const u8, allocator: std.mem.Allocator) !?struct { path: []const u8, symbol: Symbol } { @@ -3109,6 +3482,32 @@ pub const Explorer = struct { } } } + } else if (spec.name != null and spec.prefix == null and spec.pattern == null) { + // Exact names are already hash-indexed. Avoid scanning and scoring + // every distinct symbol name for the overwhelmingly common MCP form + // `{ "name": "..." }`; appendOne preserves the existing result order. + const sym_name = spec.name.?; + if (self.symbol_index.get(sym_name)) |locs| { + for (locs.items) |loc| { + if (spec.kind) |k| if (loc.kind != k) continue; + var detail: ?[]const u8 = null; + if (self.outlines.getPtr(loc.path)) |outline| { + for (outline.symbols.items) |sym| { + if (sym.line_start == loc.line_start and std.mem.eql(u8, sym.name, sym_name)) { + detail = sym.detail; + break; + } + } + } + try appendOne(&candidates, allocator, spec.max_results, loc.path, .{ + .name = sym_name, + .kind = loc.kind, + .line_start = loc.line_start, + .line_end = loc.line_end, + .detail = detail, + }, 1.0); + } + } } else { var sym_iter = self.symbol_index.iterator(); while (sym_iter.next()) |entry| { @@ -5480,6 +5879,9 @@ pub const Explorer = struct { return; } + const gen = self.search_gen.load(.acquire); + if (self.word_render_cache.render(word, false, gen, allocator, out)) return; + const render_start = out.items.len; const hits = self.word_index.search(word); try out.ensureUnusedCapacity(allocator, 64 + hits.len * 48); const w = cio.listWriter(out, allocator); @@ -5487,18 +5889,18 @@ pub const Explorer = struct { for (hits) |h| { try w.print(" {s}:{d}\n", .{ self.word_index.hitPath(h), h.line_num }); } + self.word_render_cache.put(word, false, gen, out.items[render_start..]); } - pub const FuzzyMatch = struct { - path: []const u8, - score: f32, - }; + pub const FuzzyMatch = FuzzyFileMatch; pub fn fuzzyFindFiles(self: *Explorer, query: []const u8, allocator: std.mem.Allocator, max_results: usize) ![]const FuzzyMatch { if (query.len == 0) return &.{}; self.mu.lockShared(); defer self.mu.unlockShared(); + const gen = self.search_gen.load(.acquire); + if (self.fuzzy_file_cache.get(query, max_results, gen, allocator)) |cached| return cached; // Parse query: split on spaces, extract extension constraints (*.py, *.ts) var parts: std.ArrayList([]const u8) = .empty; @@ -5576,10 +5978,12 @@ pub const Explorer = struct { matches.items.len = max_results; } - return matches.toOwnedSlice(allocator) catch { + const result = matches.toOwnedSlice(allocator) catch { matches.deinit(allocator); return &.{}; }; + self.fuzzy_file_cache.put(query, max_results, gen, result); + return result; } pub fn renderExactFileFind(self: *Explorer, query: []const u8, allocator: std.mem.Allocator, out: *std.ArrayList(u8), max_results: usize) !usize { @@ -7269,6 +7673,7 @@ fn appendExtractedLines( line_numbers: bool, compact: bool, language: Language, + line_prefix: []const u8, allocator: std.mem.Allocator, out: *std.ArrayList(u8), ) !void { @@ -7281,7 +7686,7 @@ fn appendExtractedLines( if (line_num > end) break; if (compact and isCommentOrBlank(line, language)) continue; if (line_numbers) { - try w.print("{d:>5} | {s}\n", .{ line_num, line }); + try w.print("{s}{d:>5} | {s}\n", .{ line_prefix, line_num, line }); } else { try w.print("{s}\n", .{line}); } diff --git a/src/hot_cache.zig b/src/hot_cache.zig index cac1c128..788cfe76 100644 --- a/src/hot_cache.zig +++ b/src/hot_cache.zig @@ -4,6 +4,8 @@ const builtin = @import("builtin"); const AtomicCounter = if (builtin.cpu.arch == .wasm32) u32 else u64; /// Fixed-capacity CLOCK eviction cache for file contents. +/// Callers must provide external synchronization around get/put/remove/clear; +/// Explorer uses its shared/exclusive mutex for that contract. /// Keys are always owned (duped on put, freed on eviction/remove/clear/deinit). /// Values are owned by default (duped on put), but `putBorrowed` stores a value /// that aliases externally-owned memory (e.g. a retained mmap of the snapshot @@ -15,6 +17,7 @@ pub const ContentCache = struct { slots: []Slot, capacity: u32, count_: u32, + next_generation: u64, allocator: std.mem.Allocator, hits_: std.atomic.Value(AtomicCounter), misses_: std.atomic.Value(AtomicCounter), @@ -40,6 +43,7 @@ pub const ContentCache = struct { key_hash: u64, key: []const u8, value: []const u8, + generation: u64, ref_bit: bool, present: bool, /// When false, `value` aliases externally-owned memory and the cache @@ -51,6 +55,7 @@ pub const ContentCache = struct { .key_hash = 0, .key = &.{}, .value = &.{}, + .generation = 0, .ref_bit = false, .present = false, .value_owned = false, @@ -76,6 +81,7 @@ pub const ContentCache = struct { .slots = slots, .capacity = capacity, .count_ = 0, + .next_generation = 1, .allocator = allocator, .hits_ = std.atomic.Value(AtomicCounter).init(0), .misses_ = std.atomic.Value(AtomicCounter).init(0), @@ -95,6 +101,7 @@ pub const ContentCache = struct { .slots = slots, .capacity = capacity, .count_ = 0, + .next_generation = 1, .allocator = allocator, .hits_ = std.atomic.Value(AtomicCounter).init(0), .misses_ = std.atomic.Value(AtomicCounter).init(0), @@ -116,7 +123,13 @@ pub const ContentCache = struct { self.allocator.free(self.slots); } - pub fn get(self: *ContentCache, key: []const u8) ?[]const u8 { + pub const ValueRef = struct { + value: []const u8, + generation: u64, + value_owned: bool, + }; + + pub fn getWithGeneration(self: *ContentCache, key: []const u8) ?ValueRef { const h = hashKey(key); const base = @as(u32, @truncate(h)) % self.capacity; var i: u32 = 0; @@ -126,13 +139,18 @@ pub const ContentCache = struct { if (slot.present and slot.key_hash == h and std.mem.eql(u8, slot.key, key)) { slot.ref_bit = true; _ = self.hits_.fetchAdd(1, .monotonic); - return slot.value; + return .{ .value = slot.value, .generation = slot.generation, .value_owned = slot.value_owned }; } } _ = self.misses_.fetchAdd(1, .monotonic); return null; } + pub fn get(self: *ContentCache, key: []const u8) ?[]const u8 { + const ref = self.getWithGeneration(key) orelse return null; + return ref.value; + } + /// Insert key/value, duping both into the cache allocator. On collision past /// the probe limit, evicts a cold slot via CLOCK sweep and frees its memory. /// Insert key/value, duping both into the cache allocator. On collision past @@ -151,6 +169,13 @@ pub const ContentCache = struct { return self.putImpl(key, value, false); } + fn takeGeneration(self: *ContentCache) u64 { + const generation = self.next_generation; + if (generation == std.math.maxInt(u64)) @panic("ContentCache generation exhausted"); + self.next_generation = generation + 1; + return generation; + } + fn putImpl(self: *ContentCache, key: []const u8, value: []const u8, own: bool) !void { if (own and (value.len > self.max_entry_bytes or value.len > self.byte_budget)) { // Too large to cache; drop any stale entry for the key so get() @@ -180,6 +205,7 @@ pub const ContentCache = struct { } slot.value = new_value; slot.value_owned = own; + slot.generation = self.takeGeneration(); if (own) self.owned_bytes += value.len; slot.ref_bit = true; return; @@ -227,6 +253,7 @@ pub const ContentCache = struct { slot.key = duped_key; slot.value = new_value; slot.value_owned = own; + slot.generation = self.takeGeneration(); slot.ref_bit = true; slot.present = true; if (own) self.owned_bytes += value.len; @@ -381,6 +408,29 @@ test "ContentCache: put updates existing key in place" { try std.testing.expectEqual(@as(u32, 1), cache.len()); } +test "ContentCache: generations change when storage is replaced" { + var cache = try ContentCache.initAlloc(std.testing.allocator, 64); + defer cache.deinit(); + + try cache.put("key", "same"); + const first = cache.getWithGeneration("key").?; + try std.testing.expect(first.value_owned); + try cache.put("key", "size"); + const second = cache.getWithGeneration("key").?; + try std.testing.expect(first.generation != second.generation); + try std.testing.expectEqualStrings("size", second.value); + + cache.remove("key"); + try cache.put("key", "same"); + const third = cache.getWithGeneration("key").?; + try std.testing.expect(second.generation != third.generation); + + cache.clear(); + try cache.put("key", "same"); + const fourth = cache.getWithGeneration("key").?; + try std.testing.expect(third.generation != fourth.generation); +} + test "ContentCache: clear drops all entries" { var cache = try ContentCache.initAlloc(std.testing.allocator, 64); defer cache.deinit(); diff --git a/src/index.zig b/src/index.zig index 15016513..12127eb2 100644 --- a/src/index.zig +++ b/src/index.zig @@ -1262,28 +1262,41 @@ pub const TrigramIndex = struct { local.ensureTotalCapacity(estimated_unique) catch {}; if (content.len >= 3) { + // Keep overlapping raw and normalized windows as rolling locals. The + // old loop requested three loads and up to four normalizations at every + // position; rolling advances each value once without changing masks. + var c0 = content[0]; + var c1 = content[1]; + var c2 = content[2]; + var n0 = normalizeChar(c0); + var n1 = normalizeChar(c1); + var n2 = normalizeChar(c2); for (0..content.len - 2) |i| { - // Skip trigrams that are pure whitespace (terrible filters, ~12% of all occurrences) - const c0 = content[i]; - const c1 = content[i + 1]; - const c2 = content[i + 2]; - if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and - (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and - (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; + const has_next = i + 3 < content.len; + const c3 = if (has_next) content[i + 3] else 0; + const n3 = if (has_next) normalizeChar(c3) else 0; - const tri = packTrigram( - normalizeChar(c0), - normalizeChar(c1), - normalizeChar(c2), - ); - const gop = try local.getOrPut(tri); - if (!gop.found_existing) { - gop.value_ptr.* = PostingMask{}; - } - gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); - if (i + 3 < content.len) { - gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(normalizeChar(content[i + 3]) % 8); + // Skip trigrams that are pure whitespace (terrible filters, + // ~12% of all occurrences). + if (!((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and + (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and + (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r'))) + { + const tri = packTrigram(n0, n1, n2); + const gop = try local.getOrPut(tri); + if (!gop.found_existing) gop.value_ptr.* = PostingMask{}; + gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i & 7); + if (has_next) { + gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(n3 & 7); + } } + + c0 = c1; + c1 = c2; + c2 = c3; + n0 = n1; + n1 = n2; + n2 = n3; } } diff --git a/src/mcp.zig b/src/mcp.zig index 3fd7b038..9760488f 100644 --- a/src/mcp.zig +++ b/src/mcp.zig @@ -484,6 +484,37 @@ const ProjectCache = struct { } }; +fn assembleMcpContentEnvelope( + alloc: std.mem.Allocator, + summary: []const u8, + output: []const u8, + guidance: []const u8, + is_error: bool, + result: *std.ArrayList(u8), +) bool { + result.ensureTotalCapacity(alloc, output.len + summary.len + guidance.len + 256) catch {}; + result.appendSlice(alloc, "{\"content\":[") catch return false; + + if (summary.len > 0) { + result.appendSlice(alloc, "{\"type\":\"text\",\"annotations\":{\"audience\":[\"user\"]},\"text\":\"") catch return false; + mcpj.writeEscaped(alloc, result, summary); + result.appendSlice(alloc, "\"},") catch return false; + } + + result.appendSlice(alloc, "{\"type\":\"text\",\"annotations\":{\"audience\":[\"assistant\"]},\"text\":\"") catch return false; + mcpj.writeEscaped(alloc, result, output); + result.appendSlice(alloc, "\"}") catch return false; + + if (guidance.len > 0) { + result.appendSlice(alloc, ",{\"type\":\"text\",\"annotations\":{\"audience\":[\"user\"]},\"text\":\"") catch return false; + mcpj.writeEscaped(alloc, result, guidance); + result.appendSlice(alloc, "\"}") catch return false; + } + + result.appendSlice(alloc, if (is_error) "],\"isError\":true}" else "],\"isError\":false}") catch return false; + return true; +} + pub const BenchContext = struct { cache: ProjectCache, @@ -537,7 +568,7 @@ pub const BenchContext = struct { explorer: *Explorer, agents: *AgentRegistry, telem: *telemetry_mod.Telemetry, - ) struct { dispatch_ns: u64, response_bytes: usize } { + ) struct { dispatch_ns: u64, response_bytes: usize, response_hash: u64 } { var out: std.ArrayList(u8) = .empty; defer out.deinit(alloc); @@ -554,6 +585,15 @@ pub const BenchContext = struct { summary.appendSlice(alloc, if (is_error) MCP_RED ++ MCP_CROSS ++ " " ++ MCP_RESET else MCP_GREEN ++ MCP_CHECK ++ " " ++ MCP_RESET) catch {}; summary.appendSlice(alloc, mcpToolIcon(name)) catch {}; mcpGenerateSummary(alloc, name, args, out.items, is_error, &summary); + + // Normalize only the nondeterministic duration. Everything else in the + // client-visible MCP envelope participates in response parity. + var parity_summary: std.ArrayList(u8) = .empty; + defer parity_summary.deinit(alloc); + parity_summary.appendSlice(alloc, summary.items) catch {}; + var parity_dur_buf: [96]u8 = undefined; + parity_summary.appendSlice(alloc, mcpFormatDuration(&parity_dur_buf, 0)) catch {}; + var dur_buf: [96]u8 = undefined; summary.appendSlice(alloc, mcpFormatDuration(&dur_buf, elapsed)) catch {}; @@ -563,27 +603,33 @@ pub const BenchContext = struct { var result: std.ArrayList(u8) = .empty; defer result.deinit(alloc); - result.ensureTotalCapacity(alloc, out.items.len + summary.items.len + guidance.items.len + 256) catch {}; - result.appendSlice(alloc, "{\"content\":[") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = 0 }; - - if (summary.items.len > 0) { - result.appendSlice(alloc, "{\"type\":\"text\",\"text\":\"") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; - mcpj.writeEscaped(alloc, &result, summary.items); - result.appendSlice(alloc, "\"},") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; + if (!assembleMcpContentEnvelope(alloc, summary.items, out.items, guidance.items, is_error, &result)) { + return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = 0, .response_hash = 0 }; } - result.appendSlice(alloc, "{\"type\":\"text\",\"text\":\"") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; - mcpj.writeEscaped(alloc, &result, out.items); - result.appendSlice(alloc, "\"}") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; + const bench_id = std.json.Value{ .integer = 1 }; + var rpc_result: std.ArrayList(u8) = .empty; + defer rpc_result.deinit(alloc); + if (!assembleJsonRpcResult(alloc, bench_id, result.items, &rpc_result)) { + return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = 0, .response_hash = 0 }; + } - if (guidance.items.len > 0) { - result.appendSlice(alloc, ",{\"type\":\"text\",\"text\":\"") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; - mcpj.writeEscaped(alloc, &result, guidance.items); - result.appendSlice(alloc, "\"}") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; + var parity_result: std.ArrayList(u8) = .empty; + defer parity_result.deinit(alloc); + if (!assembleMcpContentEnvelope(alloc, parity_summary.items, out.items, guidance.items, is_error, &parity_result)) { + return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = rpc_result.items.len, .response_hash = 0 }; + } + var parity_rpc_result: std.ArrayList(u8) = .empty; + defer parity_rpc_result.deinit(alloc); + if (!assembleJsonRpcResult(alloc, bench_id, parity_result.items, &parity_rpc_result)) { + return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = rpc_result.items.len, .response_hash = 0 }; } - result.appendSlice(alloc, if (is_error) "],\"isError\":true}" else "],\"isError\":false}") catch return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; - return .{ .dispatch_ns = @intCast(elapsed), .response_bytes = result.items.len }; + return .{ + .dispatch_ns = @intCast(elapsed), + .response_bytes = rpc_result.items.len, + .response_hash = std.hash.Wyhash.hash(0, parity_rpc_result.items), + }; } }; @@ -1307,33 +1353,11 @@ fn handleCall( } // Assemble MCP content envelope (1 block in lean mode, up to 3 otherwise). + // BenchContext uses this same path so parity covers the full client-visible + // response shape, escaping, guidance, and isError value. var result: std.ArrayList(u8) = .empty; defer result.deinit(alloc); - result.ensureTotalCapacity(alloc, out.items.len + summary.items.len + guidance.items.len + 256) catch {}; - result.appendSlice(alloc, "{\"content\":[") catch return; - - // Block 1 (summary โ€” audience: user; spec-canonical signal that - // token-conscious clients can strip) - if (summary.items.len > 0) { - result.appendSlice(alloc, "{\"type\":\"text\",\"annotations\":{\"audience\":[\"user\"]},\"text\":\"") catch return; - mcpj.writeEscaped(alloc, &result, summary.items); - result.appendSlice(alloc, "\"},") catch return; - } - - // Block 2 (raw data โ€” audience: assistant; this is what the model - // actually consumes) - result.appendSlice(alloc, "{\"type\":\"text\",\"annotations\":{\"audience\":[\"assistant\"]},\"text\":\"") catch return; - mcpj.writeEscaped(alloc, &result, out.items); - result.appendSlice(alloc, "\"}") catch return; - - // Block 3 (guidance โ€” audience: user) - if (guidance.items.len > 0) { - result.appendSlice(alloc, ",{\"type\":\"text\",\"annotations\":{\"audience\":[\"user\"]},\"text\":\"") catch return; - mcpj.writeEscaped(alloc, &result, guidance.items); - result.appendSlice(alloc, "\"}") catch return; - } - - result.appendSlice(alloc, if (is_error) "],\"isError\":true}" else "],\"isError\":false}") catch return; + if (!assembleMcpContentEnvelope(alloc, summary.items, out.items, guidance.items, is_error, &result)) return; writeResult(alloc, stdout, id, result.items); } @@ -2628,35 +2652,8 @@ fn handleContext(io: std.Io, alloc: std.mem.Allocator, args: *const std.json.Obj wsr.print("- {s} ({s}) โ€” {s}:{d}\n", .{ sr.kw, sr.kind, sr.path, sr.line }) catch {}; wsl.print("- {s} ({s}) โ€” {s}:{d}\n", .{ sr.kw, sr.kind, sr.path, sr.line }) catch {}; if (inline_bodies) { - if (explorer.getContent(sr.path, A) catch null) |content| { - var cur_line: u32 = 1; - var i: usize = 0; - var line_start: ?usize = null; - var captured: u32 = 0; - const body_end: u32 = if (sr.line_end > sr.line) @min(sr.line_end, sr.line + 39) else sr.line; - const max_lines: u32 = body_end - sr.line + 1; - if (cur_line == sr.line) line_start = 0; - while (i < content.len and captured < max_lines) : (i += 1) { - if (content[i] == '\n') { - if (line_start) |ls| { - const line_end = i; - wsr.print(" {d:>5} | {s}\n", .{ cur_line, content[ls..line_end] }) catch {}; - captured += 1; - } - cur_line += 1; - if (cur_line >= sr.line and cur_line <= body_end) { - line_start = i + 1; - } else { - line_start = null; - } - } - } - if (line_start) |ls| { - if (captured < max_lines) { - wsr.print(" {d:>5} | {s}\n", .{ cur_line, content[ls..] }) catch {}; - } - } - } + const body_end: u32 = if (sr.line_end > sr.line) @min(sr.line_end, sr.line + 39) else sr.line; + _ = explorer.appendLineRange(sr.path, sr.line, body_end, " ", A, &sec_syms_rich) catch false; } } @@ -5306,22 +5303,33 @@ pub fn projectRelPath(path: []const u8, root: []const u8) ?[]const u8 { return rel; } +fn assembleJsonRpcResult(alloc: std.mem.Allocator, id: ?std.json.Value, result: []const u8, buf: *std.ArrayList(u8)) bool { + buf.ensureTotalCapacity(alloc, result.len + 64) catch {}; + buf.appendSlice(alloc, "{\"jsonrpc\":\"2.0\",\"id\":") catch return false; + appendId(alloc, buf, id); + buf.appendSlice(alloc, ",\"result\":") catch return false; + // MCP responses are normally compact JSON with no raw line breaks. Let + // std.mem's vectorized search prove that once, then copy the whole payload; + // retain the sanitizing fallback for any non-canonical producer. + if (std.mem.indexOfAny(u8, result, "\n\r") == null) { + buf.appendSlice(alloc, result) catch return false; + } else { + var i: usize = 0; + while (i < result.len) { + const start = i; + while (i < result.len and result[i] != '\n' and result[i] != '\r') : (i += 1) {} + if (i > start) buf.appendSlice(alloc, result[start..i]) catch return false; + if (i < result.len) i += 1; + } + } + buf.appendSlice(alloc, "}\n") catch return false; + return true; +} + fn writeResult(alloc: std.mem.Allocator, stdout: cio.File, id: ?std.json.Value, result: []const u8) void { var buf: std.ArrayList(u8) = .empty; defer buf.deinit(alloc); - buf.ensureTotalCapacity(alloc, result.len + 64) catch {}; - buf.appendSlice(alloc, "{\"jsonrpc\":\"2.0\",\"id\":") catch return; - appendId(alloc, &buf, id); - buf.appendSlice(alloc, ",\"result\":") catch return; - // Batch-copy non-newline runs instead of per-byte append. - var i: usize = 0; - while (i < result.len) { - const start = i; - while (i < result.len and result[i] != '\n' and result[i] != '\r') : (i += 1) {} - if (i > start) buf.appendSlice(alloc, result[start..i]) catch return; - if (i < result.len) i += 1; - } - buf.appendSlice(alloc, "}\n") catch return; + if (!assembleJsonRpcResult(alloc, id, result, &buf)) return; stdout.writeAll(buf.items) catch { stdout_broken.store(true, .release); return; diff --git a/src/release_info.zig b/src/release_info.zig index 0a6e3bb1..acee1dbd 100644 --- a/src/release_info.zig +++ b/src/release_info.zig @@ -1 +1 @@ -pub const semver = "0.2.5829"; +pub const semver = "0.2.5830"; diff --git a/src/test_explore.zig b/src/test_explore.zig index 9314a192..30acd918 100644 --- a/src/test_explore.zig +++ b/src/test_explore.zig @@ -2568,3 +2568,207 @@ test "issue-656: call graph is stale and dangling after a file edit" { defer if (after) |steps| testing.allocator.free(steps); try testing.expect(after != null); } + +test "render caches preserve output and invalidate on mutation" { + var ex = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer ex.deinit(); + try ex.indexFile("src/alpha.zig", "pub fn alpha() void {}\nconst cached_word = 1;\n"); + + var tree_first: std.ArrayList(u8) = .empty; + defer tree_first.deinit(testing.allocator); + var tree_cached: std.ArrayList(u8) = .empty; + defer tree_cached.deinit(testing.allocator); + try ex.renderTree(testing.allocator, &tree_first, false); + try ex.renderTree(testing.allocator, &tree_cached, false); + try testing.expectEqualSlices(u8, tree_first.items, tree_cached.items); + + var outline_first: std.ArrayList(u8) = .empty; + defer outline_first.deinit(testing.allocator); + var outline_cached: std.ArrayList(u8) = .empty; + defer outline_cached.deinit(testing.allocator); + try testing.expect(try ex.renderOutline("src/alpha.zig", testing.allocator, &outline_first, false)); + try testing.expect(try ex.renderOutline("src/alpha.zig", testing.allocator, &outline_cached, false)); + try testing.expectEqualSlices(u8, outline_first.items, outline_cached.items); + + var word_first: std.ArrayList(u8) = .empty; + defer word_first.deinit(testing.allocator); + var word_cached: std.ArrayList(u8) = .empty; + defer word_cached.deinit(testing.allocator); + try ex.renderWord("cached_word", testing.allocator, &word_first); + try ex.renderWord("cached_word", testing.allocator, &word_cached); + try testing.expectEqualSlices(u8, word_first.items, word_cached.items); + + try ex.indexFile("src/beta.zig", "pub fn beta() void {}\nconst cached_word = 2;\n"); + var tree_after: std.ArrayList(u8) = .empty; + defer tree_after.deinit(testing.allocator); + try ex.renderTree(testing.allocator, &tree_after, false); + try testing.expect(std.mem.indexOf(u8, tree_after.items, "beta.zig") != null); + + var word_after: std.ArrayList(u8) = .empty; + defer word_after.deinit(testing.allocator); + try ex.renderWord("cached_word", testing.allocator, &word_after); + try testing.expect(std.mem.indexOf(u8, word_after.items, "src/beta.zig") != null); + + try ex.indexFile("src/alpha.zig", "pub fn renamedAlpha() void {}\n"); + var outline_after: std.ArrayList(u8) = .empty; + defer outline_after.deinit(testing.allocator); + try testing.expect(try ex.renderOutline("src/alpha.zig", testing.allocator, &outline_after, false)); + try testing.expect(std.mem.indexOf(u8, outline_after.items, "renamedAlpha") != null); + try testing.expect(std.mem.indexOf(u8, outline_after.items, "function alpha") == null); +} + +test "render cache hits wait for Explorer mutation lock" { + var ex = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer ex.deinit(); + try ex.indexFile("src/alpha.zig", "pub fn alpha() void {}\n"); + + var tree_prime: std.ArrayList(u8) = .empty; + defer tree_prime.deinit(testing.allocator); + try ex.renderTree(testing.allocator, &tree_prime, false); + var outline_prime: std.ArrayList(u8) = .empty; + defer outline_prime.deinit(testing.allocator); + try testing.expect(try ex.renderOutline("src/alpha.zig", testing.allocator, &outline_prime, false)); + + const ReadCtx = struct { + ex: *Explorer, + outline: bool, + started: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + finished: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + failed: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + + fn run(ctx: *@This()) void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(std.heap.page_allocator); + ctx.started.store(true, .release); + if (ctx.outline) { + const found = ctx.ex.renderOutline("src/alpha.zig", std.heap.page_allocator, &out, false) catch { + ctx.failed.store(true, .release); + return; + }; + if (!found) ctx.failed.store(true, .release); + } else { + ctx.ex.renderTree(std.heap.page_allocator, &out, false) catch { + ctx.failed.store(true, .release); + return; + }; + } + ctx.finished.store(true, .release); + } + }; + + var tree_ctx = ReadCtx{ .ex = &ex, .outline = false }; + var outline_ctx = ReadCtx{ .ex = &ex, .outline = true }; + ex.mu.lock(); + const tree_thread = std.Thread.spawn(.{}, ReadCtx.run, .{&tree_ctx}) catch |err| { + ex.mu.unlock(); + return err; + }; + const outline_thread = std.Thread.spawn(.{}, ReadCtx.run, .{&outline_ctx}) catch |err| { + ex.mu.unlock(); + tree_thread.join(); + return err; + }; + while (!tree_ctx.started.load(.acquire) or !outline_ctx.started.load(.acquire)) cio.sleepMs(1); + cio.sleepMs(20); + const tree_blocked = !tree_ctx.finished.load(.acquire); + const outline_blocked = !outline_ctx.finished.load(.acquire); + ex.mu.unlock(); + tree_thread.join(); + outline_thread.join(); + + try testing.expect(tree_blocked); + try testing.expect(outline_blocked); + try testing.expect(!tree_ctx.failed.load(.acquire)); + try testing.expect(!outline_ctx.failed.load(.acquire)); +} + +test "cached deep reads and fuzzy finds invalidate exactly" { + var ex = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer ex.deinit(); + + var content: std.ArrayList(u8) = .empty; + defer content.deinit(testing.allocator); + const content_writer = cio.listWriter(&content, testing.allocator); + for (1..121) |line| try content_writer.print("line-{d}\n", .{line}); + try ex.indexFile("deep-alpha.zig", content.items); + + const opts = Explorer.ReadRenderOptions{ .line_start = 100, .line_end = 102 }; + var first: std.ArrayList(u8) = .empty; + defer first.deinit(testing.allocator); + var cached: std.ArrayList(u8) = .empty; + defer cached.deinit(testing.allocator); + try testing.expect(try ex.renderCachedRead("deep-alpha.zig", testing.allocator, &first, opts)); + try testing.expect(try ex.renderCachedRead("deep-alpha.zig", testing.allocator, &cached, opts)); + try testing.expectEqualSlices(u8, first.items, cached.items); + try testing.expect(std.mem.indexOf(u8, cached.items, " 100 | line-100") != null); + try testing.expect(std.mem.indexOf(u8, cached.items, " 103 |") == null); + + const fuzzy_first = try ex.fuzzyFindFiles("deep alfa", testing.allocator, 10); + defer testing.allocator.free(fuzzy_first); + const fuzzy_cached = try ex.fuzzyFindFiles("deep alfa", testing.allocator, 10); + defer testing.allocator.free(fuzzy_cached); + try testing.expectEqual(fuzzy_first.len, fuzzy_cached.len); + for (fuzzy_first, fuzzy_cached) |a, b| { + try testing.expectEqualStrings(a.path, b.path); + try testing.expectEqual(a.score, b.score); + } + + try ex.indexFile("deep-alfa-extra.zig", "pub fn extra() void {}\n"); + const fuzzy_after = try ex.fuzzyFindFiles("deep alfa", testing.allocator, 10); + defer testing.allocator.free(fuzzy_after); + try testing.expect(fuzzy_after.len > fuzzy_cached.len); + + content.clearRetainingCapacity(); + for (1..121) |line| try content_writer.print("changed-{d}\n", .{line}); + try ex.indexFile("deep-alpha.zig", content.items); + var changed: std.ArrayList(u8) = .empty; + defer changed.deinit(testing.allocator); + try testing.expect(try ex.renderCachedRead("deep-alpha.zig", testing.allocator, &changed, opts)); + try testing.expect(std.mem.indexOf(u8, changed.items, "changed-100") != null); + try testing.expect(std.mem.indexOf(u8, changed.items, "line-100") == null); +} + +test "content hashes are recomputed for mutable borrowed storage" { + var ex = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer ex.deinit(); + + var content = "const a = 1;\n".*; + try ex.contents.putBorrowed("hash.zig", content[0..]); + var first: std.ArrayList(u8) = .empty; + defer first.deinit(testing.allocator); + try testing.expect(try ex.renderCachedRead("hash.zig", testing.allocator, &first, .{})); + const hash_end = std.mem.indexOfScalar(u8, first.items, '\n').?; + const first_hash = first.items["hash:".len..hash_end]; + + @memcpy(content[0..], "const b = 2;\n"); + var changed: std.ArrayList(u8) = .empty; + defer changed.deinit(testing.allocator); + try testing.expect(try ex.renderCachedRead("hash.zig", testing.allocator, &changed, .{ .if_hash = first_hash })); + try testing.expect(!std.mem.startsWith(u8, changed.items, "unchanged:")); + try testing.expect(std.mem.indexOf(u8, changed.items, "const b = 2;") != null); +} + +test "disk-backed line ranges do not reuse offsets for changed content" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "fallback.zig", .data = "aa\nbb\ncc\n" }); + + var root_buf: [std.fs.max_path_bytes]u8 = undefined; + const root_len = try tmp.dir.realPathFile(io, ".", &root_buf); + var ex = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer ex.deinit(); + ex.setRoot(io, root_buf[0..root_len]); + + var first: std.ArrayList(u8) = .empty; + defer first.deinit(testing.allocator); + try testing.expect(try ex.appendLineRange("fallback.zig", 2, 3, " ", testing.allocator, &first)); + try testing.expectEqualStrings(" 2 | bb\n 3 | cc\n", first.items); + + // Keep the byte length constant while moving newline offsets. Temporary + // disk-read allocations may reuse the same address between calls. + try tmp.dir.writeFile(io, .{ .sub_path = "fallback.zig", .data = "a\nbbbb\nc\n" }); + var changed: std.ArrayList(u8) = .empty; + defer changed.deinit(testing.allocator); + try testing.expect(try ex.appendLineRange("fallback.zig", 2, 3, " ", testing.allocator, &changed)); + try testing.expectEqualStrings(" 2 | bbbb\n 3 | c\n", changed.items); +} diff --git a/src/test_index.zig b/src/test_index.zig index 5cf0aa87..308373e3 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -648,7 +648,27 @@ test "watcher: parallel initial scan matches sequential results" { } } -test "watcher: rolling trigram shards match canonical masks exactly" { +fn buildScalarTrigramMasks(content: []const u8, masks: *std.AutoHashMap(Trigram, PostingMask)) !void { + if (content.len < 3) return; + for (0..content.len - 2) |i| { + const c0 = content[i]; + const c1 = content[i + 1]; + const c2 = content[i + 2]; + if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and + (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and + (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; + + const tri = packTrigram(normalizeChar(c0), normalizeChar(c1), normalizeChar(c2)); + const gop = try masks.getOrPut(tri); + if (!gop.found_existing) gop.value_ptr.* = .{}; + gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); + if (i + 3 < content.len) { + gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(normalizeChar(content[i + 3]) % 8); + } + } +} + +test "rolling trigram indexes match scalar masks exactly" { var source = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer source.deinit(); var canonical = TrigramIndex.init(testing.allocator); @@ -660,14 +680,36 @@ test "watcher: rolling trigram shards match canonical masks exactly" { .{ .path = "edge/two.txt", .content = "Ab" }, .{ .path = "edge/three.txt", .content = "AbC" }, .{ .path = "edge/whitespace.txt", .content = " \t\nAbCdEf" }, + .{ .path = "edge/only-whitespace.txt", .content = " \t\n\r \t\n" }, .{ .path = "edge/rollover.txt", .content = "aaaaaaaaaaaaZ" }, .{ .path = "edge/mixed.txt", .content = "MiXeD-case final" }, + .{ .path = "edge/arbitrary-bytes.bin", .content = "\x00\x01AZ \t\n\r\xffabc" }, }; for (fixtures) |fixture| { try source.indexFile(fixture.path, fixture.content); try canonical.indexFile(fixture.path, fixture.content); } + // Keep a deliberately scalar reference independent from both rolling + // implementations. Verify every mask and ensure neither path adds keys. + for (fixtures) |fixture| { + var expected = std.AutoHashMap(Trigram, PostingMask).init(testing.allocator); + defer expected.deinit(); + try buildScalarTrigramMasks(fixture.content, &expected); + + var actual_count: usize = 0; + var actual_iter = canonical.index.iterator(); + while (actual_iter.next()) |entry| { + if (entry.value_ptr.get(fixture.path)) |actual_mask| { + actual_count += 1; + const expected_mask = expected.get(entry.key_ptr.*) orelse return error.TestUnexpectedResult; + try testing.expectEqual(expected_mask.loc_mask, actual_mask.loc_mask); + try testing.expectEqual(expected_mask.next_mask, actual_mask.next_mask); + } + } + try testing.expectEqual(expected.count(), actual_count); + } + const sharded = try watcher.buildTrigramsFromCache( &source.contents, testing.allocator,