From d4cdd41722676561766e2652777ac3ce6da82bd4 Mon Sep 17 00:00:00 2001 From: Codegraff Date: Sat, 11 Jul 2026 11:51:59 +0800 Subject: [PATCH 1/2] release: migrate 0.2.5829 to Zig 0.17 with parity guards Pin and verify the Zig 0.17 release toolchain, vendor the compatible nanoregex source, and internalize the small MCP JSON helpers. Keep the benchmark gate comparing each revision with its declared compiler. Optimize exact-word deduplication, Tier-0 aggregation, release startup, argv parsing, worker directory reuse, and rolling trigram extraction while retaining defensive fallbacks and adding differential retrieval tests. Document the migration, same-machine Zig 0.16 A/B results, full verification, and the remaining generic initial-scan gap. Closes #673 --- .github/copilot-instructions.md | 2 +- .github/workflows/bench-regression.yml | 70 +- .github/workflows/release-binaries.yml | 33 +- CHANGELOG.md | 88 +++ README.md | 5 +- build.zig | 43 +- build.zig.zon | 12 +- docs/agents.md | 2 +- docs/zig-0.17-migration.md | 448 ++++++++++++ npm/package.json | 2 +- src/adversarial_tests.zig | 4 +- src/bench.zig | 1 - src/cio.zig | 41 +- src/cli_proxy.zig | 8 +- src/codegraph.zig | 14 +- src/commands.zig | 2 +- src/explore.zig | 67 +- src/hot_cache.zig | 39 +- src/index.zig | 222 +++--- src/linter.zig | 4 +- src/main.zig | 38 +- src/mcp.zig | 14 +- src/mcp_json.zig | 97 +++ src/release_info.zig | 2 +- src/server.zig | 6 +- src/snapshot.zig | 2 +- src/telemetry.zig | 8 +- src/test_core.zig | 2 +- src/test_explore.zig | 143 ++-- src/test_index.zig | 229 +++++- src/test_mcp.zig | 58 +- src/test_parser.zig | 50 -- src/test_query.zig | 47 -- src/test_search.zig | 58 +- src/test_snapshot.zig | 48 +- src/wasm.zig | 28 +- src/watcher.zig | 116 +-- vendor/nanoregex/.gitignore | 12 + vendor/nanoregex/LICENSE | 21 + vendor/nanoregex/README.md | 136 ++++ vendor/nanoregex/UPSTREAM.md | 17 + vendor/nanoregex/build.zig | 165 +++++ vendor/nanoregex/build.zig.zon | 8 + vendor/nanoregex/src/ast.zig | 186 +++++ vendor/nanoregex/src/bench.zig | 145 ++++ vendor/nanoregex/src/bench_comptime.zig | 248 +++++++ vendor/nanoregex/src/comptime.zig | 546 ++++++++++++++ vendor/nanoregex/src/dfa.zig | 453 ++++++++++++ vendor/nanoregex/src/exec.zig | 469 ++++++++++++ vendor/nanoregex/src/minterm.zig | 191 +++++ vendor/nanoregex/src/nfa.zig | 449 ++++++++++++ vendor/nanoregex/src/parser.zig | 495 +++++++++++++ vendor/nanoregex/src/prefilter.zig | 517 ++++++++++++++ vendor/nanoregex/src/probe.zig | 87 +++ vendor/nanoregex/src/root.zig | 676 ++++++++++++++++++ .../tests/parity/fixtures/001_literal.txt | 3 + .../tests/parity/fixtures/002_dot_star.txt | 3 + .../tests/parity/fixtures/003_char_class.txt | 3 + .../tests/parity/fixtures/004_anchors.txt | 5 + .../tests/parity/fixtures/005_alternation.txt | 3 + .../parity/fixtures/006_shorthand_digit.txt | 3 + .../parity/fixtures/007_group_capture.txt | 3 + .../tests/parity/fixtures/008_lazy_star.txt | 3 + .../tests/parity/fixtures/009_lazy_plus.txt | 3 + .../parity/fixtures/010_nested_groups.txt | 3 + .../tests/parity/fixtures/011_escape_dot.txt | 3 + .../parity/fixtures/012_word_boundary.txt | 3 + .../parity/fixtures/013_dollar_multiline.txt | 6 + .../parity/fixtures/014_case_insensitive.txt | 3 + .../tests/parity/fixtures/015_dot_all.txt | 6 + .../parity/fixtures/016_counted_range.txt | 3 + .../fixtures/017_alternation_anchors.txt | 6 + .../fixtures/018_catastrophic_backtrack.txt | 3 + .../fixtures/019_char_class_negation.txt | 3 + .../fixtures/020_non_capturing_group.txt | 3 + .../fixtures/021_optional_quantifier.txt | 3 + .../fixtures/022_min_zero_quantifier.txt | 3 + .../tests/parity/fixtures/023_email_like.txt | 3 + .../parity/fixtures/024_string_anchor.txt | 4 + .../fixtures/025_function_def_pattern.txt | 5 + .../parity/fixtures/026_unbounded_min.txt | 3 + .../parity/fixtures/027_zero_width_loop.txt | 3 + vendor/nanoregex/tests/parity/run.sh | 89 +++ 83 files changed, 6480 insertions(+), 578 deletions(-) create mode 100644 docs/zig-0.17-migration.md create mode 100644 src/mcp_json.zig create mode 100644 vendor/nanoregex/.gitignore create mode 100644 vendor/nanoregex/LICENSE create mode 100644 vendor/nanoregex/README.md create mode 100644 vendor/nanoregex/UPSTREAM.md create mode 100644 vendor/nanoregex/build.zig create mode 100644 vendor/nanoregex/build.zig.zon create mode 100644 vendor/nanoregex/src/ast.zig create mode 100644 vendor/nanoregex/src/bench.zig create mode 100644 vendor/nanoregex/src/bench_comptime.zig create mode 100644 vendor/nanoregex/src/comptime.zig create mode 100644 vendor/nanoregex/src/dfa.zig create mode 100644 vendor/nanoregex/src/exec.zig create mode 100644 vendor/nanoregex/src/minterm.zig create mode 100644 vendor/nanoregex/src/nfa.zig create mode 100644 vendor/nanoregex/src/parser.zig create mode 100644 vendor/nanoregex/src/prefilter.zig create mode 100644 vendor/nanoregex/src/probe.zig create mode 100644 vendor/nanoregex/src/root.zig create mode 100644 vendor/nanoregex/tests/parity/fixtures/001_literal.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/003_char_class.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/004_anchors.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/005_alternation.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/023_email_like.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt create mode 100644 vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt create mode 100755 vendor/nanoregex/tests/parity/run.sh diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bcff91ca..5c0f76b8 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -2,7 +2,7 @@ ## Project -Zig 0.16.x code intelligence server. Tests live in `src/tests.zig`. Build and test with `zig build test`. +Zig 0.17.0-dev code intelligence server (pinned to `0.17.0-dev.813+2153f8143`). Tests live in `src/test_*.zig`. Build and test with `zig build test`. ## Rules diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index 1bb9bc9d..3b95cffd 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -7,6 +7,12 @@ permissions: contents: read pull-requests: write +env: + ZIG_VERSION: 0.17.0-dev.813+2153f8143 + ZIG_SHA256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 + LEGACY_ZIG_VERSION: 0.16.0 + LEGACY_ZIG_SHA256: 70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 + jobs: bench: runs-on: ubuntu-latest @@ -15,41 +21,61 @@ jobs: with: fetch-depth: 0 - - name: Install Zig + - name: Install pinned Zig toolchains run: | - curl -L https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz -o zig.tar.xz - tar -xf zig.tar.xz - echo "$PWD/zig-x86_64-linux-0.16.0" >> "$GITHUB_PATH" + set -euo pipefail + + head_archive="zig-x86_64-linux-${ZIG_VERSION}.tar.xz" + head_dir="zig-x86_64-linux-${ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/builds/${head_archive}" -o zig-head.tar.xz + echo "${ZIG_SHA256} zig-head.tar.xz" | sha256sum --check + tar -xf zig-head.tar.xz + test "$("$PWD/${head_dir}/zig" version)" = "$ZIG_VERSION" + + legacy_archive="zig-x86_64-linux-${LEGACY_ZIG_VERSION}.tar.xz" + legacy_dir="zig-x86_64-linux-${LEGACY_ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/download/${LEGACY_ZIG_VERSION}/${legacy_archive}" -o zig-legacy.tar.xz + echo "${LEGACY_ZIG_SHA256} zig-legacy.tar.xz" | sha256sum --check + tar -xf zig-legacy.tar.xz + test "$("$PWD/${legacy_dir}/zig" version)" = "$LEGACY_ZIG_VERSION" + + echo "$PWD/${head_dir}" >> "$GITHUB_PATH" - name: Benchmark head run: python3 scripts/run-bench-json.py bench-head.json - - name: Benchmark base + - name: Benchmark base with its declared compiler + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | - git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1 + set -euo pipefail + git fetch origin "$BASE_REF" --depth=1 git worktree add ../codedb-base FETCH_HEAD cd ../codedb-base - if python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json"; then - echo "have_base_json=true" >> "$GITHUB_ENV" - else - echo "have_base_json=false" >> "$GITHUB_ENV" - fi + + base_zig_version=$(sed -n 's/.*\.minimum_zig_version = "\([^"]*\)".*/\1/p' build.zig.zon) + case "$base_zig_version" in + "$ZIG_VERSION") + base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${ZIG_VERSION}" + ;; + "$LEGACY_ZIG_VERSION") + base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${LEGACY_ZIG_VERSION}" + ;; + *) + echo "unsupported base Zig version: ${base_zig_version:-missing}" >&2 + exit 1 + ;; + esac + test "$("$base_zig_dir/zig" version)" = "$base_zig_version" + PATH="$base_zig_dir:$PATH" \ + python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json" - name: Compare - if: env.have_base_json == 'true' run: | python3 scripts/compare-bench.py bench-base.json bench-head.json --threshold-pct 10 --markdown-out bench-report.md - - name: Bootstrap report - if: env.have_base_json != 'true' - run: | - cat > bench-report.md <<'EOF' - ## Benchmark Regression Report - - Skipped strict comparison because the base branch does not yet emit machine-readable benchmark JSON. - This PR introduces the JSON benchmark format that future PRs will compare against. - EOF - - name: Upload artifacts uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 00229911..6e83a8ce 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -12,7 +12,7 @@ permissions: contents: write env: - ZIG_VERSION: 0.16.0 + ZIG_VERSION: 0.17.0-dev.813+2153f8143 jobs: build: @@ -23,25 +23,25 @@ jobs: matrix: include: - runner: macos-15-intel + zig_host: x86_64-macos + zig_sha256: 3938c46ae4bca3c13f423b09503e3ef00bb4b7ef12b8bc1e5122ede366057a5b zig_target: x86_64-macos asset_name: codedb-darwin-x86_64 - zig_archive: zig-x86_64-macos-0.16.0.tar.xz - zig_dir: zig-x86_64-macos-0.16.0 - runner: macos-14 + zig_host: aarch64-macos + zig_sha256: 36673d2513afa4a96c86780648ba504beedd7f0451389091cf9d53e38d5b4840 zig_target: aarch64-macos asset_name: codedb-darwin-arm64 - zig_archive: zig-aarch64-macos-0.16.0.tar.xz - zig_dir: zig-aarch64-macos-0.16.0 - runner: ubuntu-24.04 + zig_host: x86_64-linux + zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 zig_target: x86_64-linux asset_name: codedb-linux-x86_64 - zig_archive: zig-x86_64-linux-0.16.0.tar.xz - zig_dir: zig-x86_64-linux-0.16.0 - runner: ubuntu-24.04 + zig_host: x86_64-linux + zig_sha256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 zig_target: aarch64-linux asset_name: codedb-linux-arm64 - zig_archive: zig-x86_64-linux-0.16.0.tar.xz - zig_dir: zig-x86_64-linux-0.16.0 env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.event.release.tag_name }} APPLE_CERTIFICATE_P12: ${{ secrets.APPLE_CERTIFICATE_P12 }} @@ -53,13 +53,22 @@ jobs: fetch-depth: 0 ref: ${{ env.RELEASE_TAG }} - - name: Install Zig + - name: Install pinned Zig shell: bash run: | set -euo pipefail - curl -L "https://ziglang.org/download/${ZIG_VERSION}/${{ matrix.zig_archive }}" -o zig.tar.xz + grep -Fq ".minimum_zig_version = \"${ZIG_VERSION}\"" build.zig.zon || { + echo "release tag does not require the workflow's pinned Zig ${ZIG_VERSION}" >&2 + exit 1 + } + archive="zig-${{ matrix.zig_host }}-${ZIG_VERSION}.tar.xz" + zig_dir="zig-${{ matrix.zig_host }}-${ZIG_VERSION}" + curl --fail --show-error --location --retry 3 \ + "https://ziglang.org/builds/${archive}" -o zig.tar.xz + echo "${{ matrix.zig_sha256 }} zig.tar.xz" | shasum -a 256 --check tar -xf zig.tar.xz - echo "$PWD/${{ matrix.zig_dir }}" >> "$GITHUB_PATH" + test "$("$PWD/${zig_dir}/zig" version)" = "$ZIG_VERSION" + echo "$PWD/${zig_dir}" >> "$GITHUB_PATH" - name: Import Apple signing certificate if: runner.os == 'macOS' && env.APPLE_CERTIFICATE_P12 != '' diff --git a/CHANGELOG.md b/CHANGELOG.md index 04e537aa..a1e7ae37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,94 @@ # Changelog +## 0.2.5829 - 2026-07-11 + +The release toolchain moves to pinned Zig `0.17.0-dev.813+2153f8143` without +changing codedb's retrieval semantics, while a focused performance pass makes +user-facing search faster than the previous Zig 0.16.0 build. The migration, +performance work, and retrieval-parity requirements are tracked in #673. + +### Zig 0.17 migration + +- **Compiler and release jobs are pinned to the exact tested development + snapshot.** `build.zig.zon` and binary-release CI select and assert + `0.17.0-dev.813+2153f8143`; transition benchmark CI selects each revision's + verified 0.16/0.17 compiler instead of silently skipping the base comparison. + Release downloads are verified before use. The build graph uses Zig 0.17 + passthrough/lazy-path APIs, and removed + language/library APIs were migrated without changing command behavior. +- **Published targets remain covered.** Native, Windows, and freestanding/WASM + paths use target-appropriate atomics and platform guards. Normal POSIX and + Windows argv are borrowed for process lifetime rather than copied again by + the command parser. +- **Dependencies are reproducible.** The Zig-0.17-compatible `nanoregex` source + is vendored with its license; the small protocol-neutral JSON helpers formerly + supplied by `mcp-zig` now live in `src/mcp_json.zig`, avoiding an otherwise + incompatible server-framework dependency. +- **Migration details are documented.** `docs/zig-0.17-migration.md` records the + removed APIs, dependency strategy, target-specific fixes, verification flow, + and the exact codedb file/subsystem inventory. + +### Faster than Zig 0.16, with unchanged retrieval + +Same-machine A/B used native macOS arm64 `ReleaseFast` binaries, clean +`428d8df` on Zig 0.16.0 as the baseline, and an immutable 631-file +`git archive` corpus. Direct tests ran 20 alternating A/B pairs with 200 +iterations per query. CLI tests used 5 warmups and 40 measured Hyperfine runs, +isolated homes/caches, and daemon/telemetry disabled. + +- **Cold single-token CLI search: 127.0ms → 105.7ms** — **1.20× faster** + (−16.8%). Warm snapshot-backed CLI search: 117.2ms → 102.8ms — **1.14× + faster** (−12.3%). Hyperfine reported outliers in both sets; the Zig 0.17 + build also had materially lower variance. +- **Exact-word lookup geometric mean: 3.217× faster.** Representative medians: + `config` 6607.5ns → 2452.5ns, `error` 12792.5ns → 3227.5ns, `request` + 6245ns → 2345ns, and `response` 10860ns → 2885ns. Pair-sorted postings now + compact adjacent duplicate `(doc_id, line)` hits linearly; malformed, legacy, + or fragmented postings retain full hash deduplication and first-occurrence + order through a detected fallback. +- **Tier-0 content retrieval avoids a per-query document map** when posting doc + IDs are nondecreasing. Any ordering break selects the previous direct-slot or + hash-map path, preserving defensive behavior for incrementally fragmented + indexes. Ranking weights, tier order, result caps, and tie-breakers are + unchanged. +- **Cold trigram construction reuses one directory handle per worker** and rolls + both raw and normalized byte windows, reducing overlapping loads and ASCII + normalization from roughly four per byte to one. Whitespace skipping, + `loc_mask`, `next_mask`, case folding, and final-trigram boundaries are + unchanged. +- **Release startup avoids unnecessary work.** Optimized builds execute directly + on the process stack instead of creating and immediately joining a worker + thread; Debug retains its 64MB-stack trampoline. Borrowed argv parsing removes + per-argument allocation/copy/free while retaining Windows bootstrap lifetime. + Completed word-index searches also retain their existing shared lock instead + of unlocking and immediately reacquiring it. + +No scoring, ranking, tokenization, parser, filtering, or result-rendering +formula changed in this pass. Direct benchmark hit counts are identical, and +parity tests cover sorted and malformed word postings, grouped and fragmented +Tier-0 aggregation, sequential versus parallel indexing, persisted/mmap index +round-trips, and lazy word-index rebuilds. + +The overall direct-query geometric mean is **1.405× faster**; search-only and +symbol groups are effectively flat to slightly faster (1.007× and 1.013×). +The remaining known gap is the generic full parse/serial-commit initial scan: +166ms → 169ms median (**1.8% slower**). Production cold single-token search uses +the optimized trigram scan path above. Closing the generic gap requires a +larger parser/commit ownership or pipeline redesign and is intentionally not +mixed into this behavior-preserving release. + +### Verification + +- uncached full Zig suite: 23/23 build steps; 892/896 tests passed, 4 platform skips +- focused suites include 181/181 index, 132/132 explore, 113/113 search, + and 152/156 MCP tests (4 platform skips) +- MCP E2E: 20/20 passed across roots handshake, explicit-root, no-roots, and + inline-argument scenarios +- immutable-corpus direct-query hit counts matched the Zig 0.16.0 baseline +- `git diff --check` and modified-file lint passed + + ## 0.2.5828 - 2026-07-05 Windows warm-daemon parity, a 2× faster `codedb_context`, OSTree (Fedora diff --git a/README.md b/README.md index 6b36a070..4eaf2a37 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Release License - Zig 0.16 + Zig 0.17.0-dev Alpha Ask DeepWiki
@@ -121,6 +121,7 @@ This replaces the `codedb` binary with the latest GitHub Release and keeps your - **[CLI reference](docs/cli.md)** — every command, every flag - **[Architecture](docs/architecture.md)** — engine internals, index layout - **[Benchmarks](docs/benchmarks.md)** — micro-benchmarks + agentic-eval results vs codegraph, FTS5, lean-ctx +- **[Zig 0.17.0-dev migration guide](docs/zig-0.17-migration.md)** — repeatable zigup workflow and API change recipes | Platform | Binary | Signed | |----------|--------|--------| @@ -454,7 +455,7 @@ rm -f codedb.snapshot # remove snapshot from current project only ## 🔨 Building from Source -**Requirements:** Zig 0.16+ +**Requirements:** Zig `0.17.0-dev.813+2153f8143` (the exact tested development snapshot). See the [migration guide](docs/zig-0.17-migration.md) for reproducible zigup setup. ```bash git clone https://github.com/justrach/codedb.git diff --git a/build.zig b/build.zig index f962257e..908ddddf 100644 --- a/build.zig +++ b/build.zig @@ -33,10 +33,6 @@ pub fn build(b: *std.Build) void { }), }); - // ── mcp-zig dependency ── - const mcp_dep = b.dependency("mcp_zig", .{}); - exe.root_module.addImport("mcp", mcp_dep.module("mcp")); - // ── nanoregex dependency ── const nanoregex_dep = b.dependency("nanoregex", .{}); exe.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); @@ -58,16 +54,15 @@ pub fn build(b: *std.Build) void { "--timestamp", "-s", identity, - b.getInstallPath(.bin, "codedb"), }); - codesign.step.dependOn(&install_exe.step); - b.getInstallStep().dependOn(&codesign.step); + codesign.addFileArg(exe.getEmittedBin()); + install_exe.step.dependOn(&codesign.step); } } const run_cmd = b.addRunArtifact(exe); run_cmd.step.dependOn(b.getInstallStep()); - if (b.args) |args| run_cmd.addArgs(args); + run_cmd.addPassthruArgs(); const run_step = b.step("run", "Run codedb daemon"); run_step.dependOn(&run_cmd.step); @@ -76,16 +71,16 @@ pub fn build(b: *std.Build) void { const test_filter = b.option([]const u8, "test-filter", "Only run tests whose name contains this substring"); const test_step = b.step("test", "Run all tests"); - const test_files = [_]struct { name: []const u8, path: []const u8, needs_mcp: bool, needs_nanoregex: bool }{ - .{ .name = "test-core", .path = "src/test_core.zig", .needs_mcp = false, .needs_nanoregex = false }, - .{ .name = "test-explore", .path = "src/test_explore.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-index", .path = "src/test_index.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-parser", .path = "src/test_parser.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-search", .path = "src/test_search.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-snapshot", .path = "src/test_snapshot.zig", .needs_mcp = false, .needs_nanoregex = true }, - .{ .name = "test-mcp", .path = "src/test_mcp.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-query", .path = "src/test_query.zig", .needs_mcp = true, .needs_nanoregex = true }, - .{ .name = "test-bench", .path = "src/test_bench.zig", .needs_mcp = false, .needs_nanoregex = true }, + const test_files = [_]struct { name: []const u8, path: []const u8, needs_nanoregex: bool }{ + .{ .name = "test-core", .path = "src/test_core.zig", .needs_nanoregex = false }, + .{ .name = "test-explore", .path = "src/test_explore.zig", .needs_nanoregex = true }, + .{ .name = "test-index", .path = "src/test_index.zig", .needs_nanoregex = true }, + .{ .name = "test-parser", .path = "src/test_parser.zig", .needs_nanoregex = true }, + .{ .name = "test-search", .path = "src/test_search.zig", .needs_nanoregex = true }, + .{ .name = "test-snapshot", .path = "src/test_snapshot.zig", .needs_nanoregex = true }, + .{ .name = "test-mcp", .path = "src/test_mcp.zig", .needs_nanoregex = true }, + .{ .name = "test-query", .path = "src/test_query.zig", .needs_nanoregex = true }, + .{ .name = "test-bench", .path = "src/test_bench.zig", .needs_nanoregex = true }, }; for (test_files) |tf| { @@ -97,7 +92,6 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - if (tf.needs_mcp) t.root_module.addImport("mcp", mcp_dep.module("mcp")); if (tf.needs_nanoregex) t.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); if (test_filter) |f| { const filters = b.allocator.alloc([]const u8, 1) catch @panic("oom"); @@ -111,7 +105,6 @@ pub fn build(b: *std.Build) void { individual_step.dependOn(&run.step); } - // ── Library tests (verify the module root compiles) ── const lib_tests = b.addTest(.{ .root_module = b.createModule(.{ @@ -135,7 +128,6 @@ pub fn build(b: *std.Build) void { adversarial_tests.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); test_step.dependOn(&b.addRunArtifact(adversarial_tests).step); - // ── Benchmarks ── const bench = b.addExecutable(.{ .name = "bench", @@ -147,9 +139,8 @@ pub fn build(b: *std.Build) void { }), }); const bench_run = b.addRunArtifact(bench); - bench.root_module.addImport("mcp", mcp_dep.module("mcp")); bench.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); - if (b.args) |args| bench_run.addArgs(args); + bench_run.addPassthruArgs(); const bench_step = b.step("bench", "Run benchmarks"); bench_step.dependOn(&bench_run.step); @@ -163,10 +154,9 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - bench_edge.root_module.addImport("mcp", mcp_dep.module("mcp")); bench_edge.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); const bench_edge_run = b.addRunArtifact(bench_edge); - if (b.args) |args| bench_edge_run.addArgs(args); + bench_edge_run.addPassthruArgs(); const bench_edge_step = b.step("bench-edge", "Run edge-case benchmarks (synthetic pathological corpus)"); bench_edge_step.dependOn(&bench_edge_run.step); @@ -180,10 +170,9 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - benchmark.root_module.addImport("mcp", mcp_dep.module("mcp")); benchmark.root_module.addImport("nanoregex", nanoregex_dep.module("nanoregex")); const benchmark_run = b.addRunArtifact(benchmark); - if (b.args) |args| benchmark_run.addArgs(args); + benchmark_run.addPassthruArgs(); const benchmark_step = b.step("benchmark", "Run repo benchmark (use -- --root /path/to/repo)"); benchmark_step.dependOn(&benchmark_run.step); diff --git a/build.zig.zon b/build.zig.zon index e19fabdb..63178473 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,20 +1,16 @@ .{ .name = .codedb2, .fingerprint = 0x6e18d96ca2a31757, - .version = "0.2.5826", - .minimum_zig_version = "0.16.0", + .version = "0.2.5829", + .minimum_zig_version = "0.17.0-dev.813+2153f8143", .dependencies = .{ - .mcp_zig = .{ - .url = "https://github.com/justrach/mcp-zig/archive/refs/heads/feature/7-zig-0-16-0-migration.tar.gz", - .hash = "mcp_zig-0.2.0-_PilzNJkAQADzH2t3vqpd_nl_W0ta-gDaumXKttuPyBy", - }, .nanoregex = .{ - .url = "git+https://github.com/justrach/nanoregex#736b46703454d5f37d3e46164fc91354386bb29c", - .hash = "nanoregex-0.0.1-EdkhcVDrAgB3eslIGGP7RK60162q26bVDA9GH44_iKnt", + .path = "vendor/nanoregex", }, }, .paths = .{ "src", + "vendor/nanoregex", "build.zig", "build.zig.zon", }, diff --git a/docs/agents.md b/docs/agents.md index ecbd3fdb..2b542f89 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -2,7 +2,7 @@ ## Project -Zig 0.16.x code intelligence server. Tests live in the split `src/test_*.zig` files — one binary per area (see `build.zig`). Build and test with `zig build test`; run a single binary with e.g. `zig build test-index`. +Zig 0.17.0-dev code intelligence server (pinned to `0.17.0-dev.813+2153f8143`). Tests live in the split `src/test_*.zig` files — one binary per area (see `build.zig`). Build and test with `zig build test`; run a single binary with e.g. `zig build test-index`. ## Rules diff --git a/docs/zig-0.17-migration.md b/docs/zig-0.17-migration.md new file mode 100644 index 00000000..2c79b80d --- /dev/null +++ b/docs/zig-0.17-migration.md @@ -0,0 +1,448 @@ +# Migrating a Zig project to 0.17.0-dev with zigup + +This guide records the migration process used for codedb and is intended to be reusable in other Zig repositories. + +The tested compiler is: + +```text +0.17.0-dev.813+2153f8143 +``` + +Zig development snapshots are moving targets. Pin the full version (including the build metadata) in CI and local setup while migrating. Upgrade to a newer snapshot deliberately, rerun the full matrix, and update the pin only after it passes. + +## 1. Establish a baseline + +Before changing compilers: + +```bash +git status --short +zig version +zig build +zig build test +``` + +Record existing failures and preserve unrelated working-tree changes. If the project has benchmarks, capture the benchmark-critical results before migration so regressions can be compared against a real baseline. + +## 2. Install and select the compiler with zigup + +This guide uses [`marler8997/zigup`](https://github.com/marler8997/zigup), not an unrelated tool with a similar name. On a clean machine, install a pinned zigup release from its [release assets](https://github.com/marler8997/zigup/releases), verify the downloaded asset against the checksum recorded by your organization, put the binary on `PATH`, and confirm `zigup help` works. Avoid piping a remote installer directly into a shell. + +Install/select the exact compiler used by codedb: + +```bash +zigup fetch 0.17.0-dev.813+2153f8143 +zigup default 0.17.0-dev.813+2153f8143 +zigup keep 0.17.0-dev.813+2153f8143 +zig version +``` + +`zigup default` displays the selected version when called without an argument. `zigup keep` is optional; it only prevents a later `zigup clean` from deleting the pinned compiler. Do not use `zigup --version`: zigup treats positional input as a requested Zig version rather than exposing that conventional flag. + +For exploratory work, `zigup master` selects the latest development compiler. Do not use a moving `master` selection in CI or release automation. + +In CI, download the exact host archive from `https://ziglang.org/builds/`, verify a repository-pinned SHA-256 before extraction, and assert that `zig version` equals the pin. A versioned URL alone does not protect a privileged release job from replaced or compromised bytes. + +## 3. Pin the package minimum + +Update `build.zig.zon`: + +```zig +.{ + // ... + .minimum_zig_version = "0.17.0-dev.813+2153f8143", +} +``` + +A full development build pin makes the tested compiler explicit. If a library intentionally supports a wider range of 0.17 snapshots, lower the minimum only after testing the oldest claimed snapshot. + +## 4. Migrate the build graph first + +Run `zig build` and fix `build.zig` errors before source errors. Zig configures dependency build scripts too, so an incompatible transitive `build.zig` can stop the build before your own source is compiled. + +### Forward arguments with `addPassthruArgs` + +The `Build.args` field was removed. + +Before: + +```zig +const run = b.addRunArtifact(exe); +if (b.args) |args| run.addArgs(args); +``` + +After: + +```zig +const run = b.addRunArtifact(exe); +run.addPassthruArgs(); +``` + +This preserves commands such as: + +```bash +zig build run -- --help +``` + +Apply the same change to benchmark and utility run steps. + +### Use lazy paths instead of `getInstallPath` + +`b.getInstallPath` was removed. Prefer artifact and lazy-path APIs so the build graph can track dependencies. + +For a command that operates on the compiled executable: + +```zig +const command = b.addSystemCommand(&.{ "tool", "--flag" }); +command.addFileArg(exe.getEmittedBin()); +install_artifact.step.dependOn(&command.step); +``` + +For codesigning, codedb signs the emitted artifact and makes installation depend on the signing step. Avoid constructing `zig-out` paths manually because `--prefix` and cross-target installation can change them. + +## 5. Replace removed repetition syntax + +The `value ** count` array/string repetition syntax is gone. Use `@splat` with a result type supplied by the destination. + +### Arrays + +Before: + +```zig +var flags: [256]bool = [_]bool{false} ** 256; +var matrix: [256][256]u64 = .{.{0} ** 256} ** 256; +``` + +After: + +```zig +var flags: [256]bool = @splat(false); +var matrix: [256][256]u64 = @splat(@splat(0)); +``` + +### Repeated byte data + +Before: + +```zig +const payload = "x" ** 300; +consume(payload); +``` + +After: + +```zig +const payload: [300]u8 = @splat('x'); +consume(&payload); +``` + +Zig 0.17 no longer implicitly coerces an array value to a slice in this case; use `&payload` when the callee expects `[]const u8`. + +### Inline fixed buffers + +Give `@splat` an explicit array type when no destination type is available: + +```zig +try writer.writeAll(&@as([40]u8, @splat(0))); +``` + +A useful first-pass search is: + +```bash +grep -R -n ' \*\* ' src --include='*.zig' +``` + +Review matches rather than replacing blindly: glob documentation and comments legitimately contain `**`. + +## 6. Replace `bufPrintZ` + +`std.fmt.bufPrintZ` became `std.fmt.bufPrintSentinel`: + +```zig +const path_z = try std.fmt.bufPrintSentinel(&buf, "{s}", .{path}, 0); +``` + +For a large codebase, a compatibility helper keeps the migration focused: + +```zig +pub fn bufPrintZ( + buf: []u8, + comptime fmt: []const u8, + args: anytype, +) std.fmt.BufPrintError![:0]u8 { + return std.fmt.bufPrintSentinel(buf, fmt, args, 0); +} +``` + +Call sites can then move to the helper without changing their return types. + +## 7. Stop calling unstable I/O vtable fields directly + +The direct `io.vtable.netRead` field is no longer available. Use the public stream operation: + +Before: + +```zig +var iov: [1][]u8 = .{dest}; +const n = try io.vtable.netRead(io.userdata, stream.socket.handle, &iov); +``` + +After: + +```zig +var iov: [1][]u8 = .{dest}; +const n = try stream.read(io, &iov); +``` + +The public method routes through `Io.operate` and remains compatible with the selected backend. Treat direct vtable access as an implementation detail unless the standard library explicitly documents it as public API. + +## 8. Update allocating writers and freestanding targets + +The unmanaged `ArrayList(u8).writer(allocator)` helper is no longer available. Use `std.Io.Writer.Allocating` when the output is fundamentally a byte stream: + +```zig +var out: std.Io.Writer.Allocating = .init(allocator); +defer out.deinit(); +const writer = &out.writer; +try writer.print("value={d}", .{value}); +const bytes = try out.toOwnedSlice(); +``` + +`toOwnedSlice()` resets the allocating writer, so the deferred `deinit()` is safe after ownership transfers. + +Compile every freestanding/WASM target separately. A native build can hide target-specific problems: + +- baseline `wasm32` supports atomic integers only up to 32 bits in this toolchain; use 32-bit counters/generations when wraparound is acceptable, or explicitly choose and validate a WebAssembly threads/atomics target; +- guard filesystem, subprocess, environment, mmap, clock, and pthread paths with `builtin.os.tag == .freestanding`; +- provide no-op locks only where the target is intentionally single-threaded; +- avoid reaching `std.posix` or libc declarations from exported WASM call graphs. + +Use a larger reference trace to locate an unexpected host-only dependency: + +```bash +zig build wasm -freference-trace=20 +``` + +## 9. Audit every dependency + +A project is not migrated if a clean checkout still depends on a package whose build script or source requires an older compiler. + +For each dependency: + +1. Run its own tests under the pinned compiler. +2. Prefer an upstream 0.17-compatible release or immutable commit. +3. If no compatible release exists, temporarily vendor or fork it. +4. Preserve its license and source history/attribution. +5. Point `build.zig.zon` at the reproducible source, not a patched local cache. +6. Add the vendored directory to the root package's `.paths` if releases must include it. + +Example local dependency: + +```zig +.dependencies = .{ + .some_library = .{ + .path = "vendor/some-library", + }, +}, +.paths = .{ + "src", + "vendor/some-library", + "build.zig", + "build.zig.zon", +}, +``` + +Never treat edits inside `.zig-cache`, a global package cache, or an ignored dependency cache as the final fix. They make only one workstation pass. + +For codedb, `nanoregex` is vendored until its upstream package is 0.17-compatible. The previous `mcp-zig` dependency supplied only small protocol-neutral JSON helpers and a root value type, so those helpers were moved into codedb instead of retaining a second server-framework dependency. + +## 10. Format narrowly while iterating + +During the error-fixing loop, format only touched files: + +```bash +zig fmt build.zig build.zig.zon src/file_you_changed.zig +``` + +A new Zig formatter can produce repository-wide canonical-format changes. Run `zig fmt src` only when that churn is intentional, and review the diff separately from semantic migration changes. + +## 11. Verify in layers + +Use a narrow-to-wide loop: + +```bash +# Build graph and main executable +zig build + +# Individual project steps while iterating +zig build test-core +zig build test-mcp + +# Full unit suite +zig build test + +# Optional targets your project publishes +zig build wasm +zig build -Doptimize=ReleaseFast +``` + +For codedb, MCP changes also require: + +```bash +zig build test +python3 scripts/e2e_mcp_test.py \ + --binary zig-out/bin/codedb \ + --project "$PWD" +``` + +The E2E suite checks root handshakes, normal explicit-root startup, and no-roots-client behavior. Run target/platform CI after local verification, especially Windows and cross-compiled release targets whose code may be skipped on the host. + +## 12. Diagnose nested build failures + +Tests that invoke `zig build` may report only that a child process exited nonzero. Run the nested command directly to expose the compiler diagnostic: + +```bash +zig build +zig build test-mcp +``` + +Also check for stale assumptions in test helpers, CI images, release workflows, editor configuration, and documentation. Search for the old version string and removed API names: + +```bash +grep -R -n '0\.16\|b\.args\|getInstallPath\|bufPrintZ\|vtable\.netRead' \ + --include='*.zig' --include='*.zon' --include='*.md' . +``` + +## Completion checklist + +- [ ] Exact Zig development snapshot selected and recorded +- [ ] `build.zig.zon` minimum updated +- [ ] Build graph compiles without deprecated/removed APIs +- [ ] Source repetition syntax migrated to `@splat` +- [ ] Sentinel formatting calls migrated +- [ ] Public I/O operations replace direct vtable access +- [ ] Every dependency builds from a clean, reproducible source +- [ ] `zig build` passes +- [ ] `zig build test` passes +- [ ] Optional targets and release modes pass +- [ ] Project-specific E2E tests pass +- [ ] CI, README, and contributor setup reference the pinned compiler +- [ ] Diff reviewed for unrelated formatter churn and pre-existing work preserved + +## codedb 0.2.5829 implementation record + +This section records the concrete release changes behind the reusable guidance +above. It is intentionally explicit so a future compiler upgrade can distinguish +compatibility edits from retrieval-sensitive performance work. + +### Toolchain, build, and release automation + +| Area | Files | Change | +|---|---|---| +| Compiler/package pin | `build.zig.zon`, `README.md`, `.github/copilot-instructions.md`, `docs/agents.md` | Require the exact tested `0.17.0-dev.813+2153f8143` snapshot. | +| Build graph | `build.zig` | Replace removed argument and install-path APIs, keep benchmark argument forwarding, and preserve codesign/install dependencies through emitted lazy paths. | +| CI and releases | `.github/workflows/bench-regression.yml`, `.github/workflows/release-binaries.yml` | Download verified pinned compilers and assert `zig version`. Release targets use the 0.17 snapshot; transition benchmarks select 0.16 or 0.17 from each revision's `minimum_zig_version` so the cross-compiler regression gate cannot silently skip its base. | +| Dependencies | `build.zig.zon`, `vendor/nanoregex/`, `src/mcp_json.zig` | Vendor the compatible regex implementation and internalize the small JSON protocol helpers formerly taken from `mcp-zig`. | + +### Runtime compatibility inventory + +- `src/cio.zig` adapts sentinel formatting and Zig I/O entry points while + retaining POSIX/Windows behavior. +- `src/main.zig`, `src/cli_proxy.zig`, `src/commands.zig`, `src/server.zig`, and + `src/mcp.zig` adapt argv and stream handling. Parsed arguments borrow the + process-lifetime bootstrap argv; they are not retained beyond that lifetime. +- `src/explore.zig`, `src/hot_cache.zig`, `src/codegraph.zig`, + `src/snapshot.zig`, and `src/telemetry.zig` use target-compatible atomics, + writers, and host-operation guards. Freestanding/WASM builds do not enter + filesystem, process, telemetry, or mmap paths. +- Removed repetition syntax was converted to typed `@splat` expressions across + runtime and tests. Large fixtures that only encoded compiler-stress behavior + were reduced or removed where Zig 0.17 no longer accepts the old construct; + retrieval assertions were retained in focused suites. +- `src/wasm.zig` uses a 32-bit generation on baseline `wasm32` and keeps native + 64-bit generations elsewhere, preserving cache-invalidation behavior within + each target's supported atomic widths. + +### Performance changes and invariants + +1. **Exact-word deduplication (`src/index.zig`).** Valid posting lists are sorted + by `(doc_id, line_num)`, so `searchDeduped` writes directly into one + caller-owned allocation and removes adjacent duplicates. It detects any + ordering break before returning and reruns the complete list through the old + hash-dedup contract. The fallback preserves first-occurrence ordering for + malformed, legacy, and incrementally fragmented data. +2. **Tier-0 document aggregation (`src/explore.zig`).** A nondecreasing posting + list guarantees one contiguous run per document, allowing direct result + construction without a slots array or hash map. Nonmonotonic lists use the + defensive pre-migration aggregation path. Scores, line selection, ranking, + and caps are not changed. +3. **Word-index lock lifetime (`src/explore.zig`).** The common completed-index + path keeps its original shared lock. Only an incomplete index releases the + lock for exclusive rebuild and reacquires it before lookup. +4. **Release startup (`src/main.zig`).** Debug keeps the 64MB worker-stack + trampoline. Optimized builds run the measured approximately 190KB frame on + the process stack, removing worker creation/join overhead. Mainstream native + process stacks have ample headroom; constrained-stack targets should retain + or reintroduce the trampoline after target-specific validation. +5. **Worker directory reuse (`src/watcher.zig`).** Initial-scan and trigram-shard + workers open the project root once and pass the directory handle through file + reads instead of reopening it per file. Every worker closes its own handle; + root-open failures follow the existing null-result/error publication paths. +6. **Rolling trigram extraction (`src/watcher.zig`).** Shared extraction rolls + `c0..c3` and normalized `n0..n3`, loading and normalizing each incoming byte + once. It preserves whitespace-triple suppression, case folding, location and + following-character masks, and the no-next-byte boundary on the final + trigram. +7. **Persistence invariants (`src/index.zig`).** Trigram posting insertion keeps + doc IDs sorted after ID reuse, and disk remapping follows ascending in-memory + IDs. Streamed writes preserve mmap merge/binary-search ordering requirements. + +### Reproducible A/B result + +Baseline and candidate: + +- baseline: clean `428d8df`, Zig 0.16.0, native arm64 `ReleaseFast`; +- candidate: Zig 0.17 migration plus the optimizations above, compiler + `0.17.0-dev.813+2153f8143`, native arm64 `ReleaseFast`; +- corpus: immutable 631-file `git archive` of the baseline, outside the working + repository; +- direct queries: 20 alternating A/B pairs, 200 iterations per query; +- CLI: Hyperfine, 5 warmups, 40 measured runs, isolated `HOME` and cache, + `CODEDB_ALLOW_TEMP=1`, `CODEDB_NO_CLI_DAEMON=1`, and + `CODEDB_NO_TELEMETRY=1`. + +| Layer | Zig 0.16.0 | Zig 0.17 optimized | Result | +|---|---:|---:|---:| +| Cold single-token CLI | 127.0 ± 26.0ms | 105.7 ± 10.7ms | 1.20× faster | +| Warm snapshot CLI | 117.2 ± 27.8ms | 102.8 ± 13.3ms | 1.14× faster | +| Exact-word geometric mean | — | — | 3.217× faster | +| All direct queries geometric mean | — | — | 1.405× faster | +| Search-only geometric mean | — | — | 1.007× faster | +| Symbol geometric mean | — | — | 1.013× faster | +| Generic full initial scan, median | 166ms | 169ms | 1.8% slower | + +Hyperfine identified outliers in both CLI samples, so means are reported with +standard deviation and should not be interpreted as deterministic single-run +latency. Direct-query hit counts were asserted equal. The generic initial-scan +gap is dominated by parse plus serial commit; changing that ownership/pipeline +is a separate, higher-risk project rather than a compatibility migration. + +### Retrieval-parity gates + +The release requires all of the following before tagging: + +- exact ordered output parity between grouped Tier-0 postings and semantically + equivalent fragmented postings that force the defensive fallback; +- duplicate suppression for naturally built and malformed/non-adjacent word + postings, including first-occurrence order; +- exact trigram candidate and posting-mask parity between canonical extraction + and worker-sharded rolling extraction, including 3-byte input, whitespace, + mixed case, location-bit rollover, and final `next_mask` boundaries; +- sequential/parallel initial scan and word-shard parity; +- heap, persisted, and mmap trigram candidate parity after file removal/doc-ID + reuse; +- lazy word-index rebuild parity with an eagerly complete index; +- the full Zig test suite and MCP E2E scenarios from the project instructions. + +These are retrieval-quality checks, not only crash or hit-count checks: ordered +paths, line numbers/text, scores where applicable, candidate sets, and mask +metadata must remain equal. diff --git a/npm/package.json b/npm/package.json index b89bd269..78a11d5d 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "codedeebee", - "version": "0.2.5828", + "version": "0.2.5829", "description": "Zig code intelligence MCP server — npx launcher for the codedb native binary", "license": "MIT", "author": "justrach", diff --git a/src/adversarial_tests.zig b/src/adversarial_tests.zig index 988cf474..2bc4258a 100644 --- a/src/adversarial_tests.zig +++ b/src/adversarial_tests.zig @@ -196,7 +196,7 @@ test "adversarial: extractSparseNgrams on all-same-character content" { try testing.expect(result.len > 0); // Verify every byte position is covered by at least one n-gram - var covered: [19]bool = .{false} ** 19; + var covered: [19]bool = @splat(false); for (result) |ng| { for (ng.pos..ng.pos + ng.len) |p| { if (p < 19) covered[p] = true; @@ -649,7 +649,7 @@ test "adversarial: setFrequencyTable changes pairWeight output" { const before = pairWeight('a', 'b'); // Create a custom table where 'a','b' has a very different weight - var custom: [256][256]u16 = .{.{0x5000} ** 256} ** 256; + var custom: [256][256]u16 = @splat(@splat(0x5000)); custom['a']['b'] = 0x0100; // very low setFrequencyTable(&custom); defer resetFrequencyTable(); diff --git a/src/bench.zig b/src/bench.zig index 11317b40..f5b6b355 100644 --- a/src/bench.zig +++ b/src/bench.zig @@ -192,7 +192,6 @@ fn copyCorpus(io: std.Io, allocator: std.mem.Allocator, repo_root: []const u8, t } } - fn makeTempCorpusDir(io: std.Io, buf: *[std.fs.max_path_bytes]u8) ![]const u8 { const base = cio.posixGetenv("TMPDIR") orelse "/tmp"; const ns = cio.nanoTimestamp(); diff --git a/src/cio.zig b/src/cio.zig index a54e3902..41c713bb 100644 --- a/src/cio.zig +++ b/src/cio.zig @@ -9,6 +9,7 @@ const std = @import("std"); const builtin = @import("builtin"); const is_windows = builtin.os.tag == .windows; +const is_freestanding = builtin.os.tag == .freestanding; // POSIX libc byte-I/O on integer fds. On Windows the CRT spells the same // primitives with a leading underscore; the cross-platform wrappers below pick @@ -60,7 +61,7 @@ fn getenv(name: [*:0]const u8) ?[*:0]const u8 { /// a write error instead of killing the process. No-op on Windows — there is no /// SIGPIPE; a broken pipe is reported through the write call's return value. pub fn ignoreSigpipe() void { - if (is_windows) { + if (is_windows or is_freestanding) { return; } else { var act: std.posix.Sigaction = .{ @@ -80,7 +81,7 @@ pub fn ignoreSigpipe() void { /// malfunctions. Called once at cli-daemon startup. On Windows the equivalent /// detachment is requested at spawn time (DETACHED_PROCESS), so this is a no-op. pub fn detachFromTerminal() void { - if (is_windows) { + if (is_windows or is_freestanding) { return; } else { _ = std.c.setsid(); @@ -97,6 +98,11 @@ pub fn detachFromTerminal() void { // ── Stdio ──────────────────────────────────────────────────────────────── +/// Zig 0.17 renamed std.fmt.bufPrintZ to bufPrintSentinel. +pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) std.fmt.BufPrintError![:0]u8 { + return std.fmt.bufPrintSentinel(buf, fmt, args, 0); +} + pub const File = struct { handle: c_int, @@ -174,6 +180,12 @@ pub const Mutex = if (is_windows) struct { pub fn tryLock(self: *Mutex) bool { return winsync.TryAcquireSRWLockExclusive(&self.inner) != 0; } +} else if (is_freestanding) struct { + pub fn lock(_: *Mutex) void {} + pub fn unlock(_: *Mutex) void {} + pub fn tryLock(_: *Mutex) bool { + return true; + } } else struct { inner: std.c.pthread_mutex_t = .{}, @@ -209,6 +221,17 @@ pub const RwLock = if (is_windows) struct { pub fn tryLockShared(self: *RwLock) bool { return winsync.TryAcquireSRWLockShared(&self.inner) != 0; } +} else if (is_freestanding) struct { + pub fn lock(_: *RwLock) void {} + pub fn unlock(_: *RwLock) void {} + pub fn lockShared(_: *RwLock) void {} + pub fn unlockShared(_: *RwLock) void {} + pub fn tryLock(_: *RwLock) bool { + return true; + } + pub fn tryLockShared(_: *RwLock) bool { + return true; + } } else struct { inner: std.c.pthread_rwlock_t = .{}, @@ -236,6 +259,7 @@ pub const RwLock = if (is_windows) struct { /// Wall-clock nanoseconds since the Unix epoch. pub fn nanoTimestamp() i128 { + if (is_freestanding) return 0; if (is_windows) { var ft: winsync.FILETIME = undefined; winsync.GetSystemTimePreciseAsFileTime(&ft); @@ -255,6 +279,7 @@ pub fn milliTimestamp() i64 { /// Monotonic tick source for Timer (raw counter units, not nanoseconds). fn monoTicks() u64 { + if (is_freestanding) return 0; if (is_windows) { var ctr: i64 = undefined; _ = winsync.QueryPerformanceCounter(&ctr); @@ -305,8 +330,10 @@ pub fn randU64() u64 { const now: u128 = @bitCast(nanoTimestamp()); const ns: u64 = @truncate(now); const sec: u64 = @truncate(now / 1_000_000_000); - const tid: u64 = @intCast(std.Thread.getCurrentId()); - const pid: u64 = if (is_windows) + const tid: u64 = if (is_freestanding) 0 else @intCast(std.Thread.getCurrentId()); + const pid: u64 = if (is_freestanding) + 0 + else if (is_windows) @intCast(winsync.GetCurrentProcessId()) else @intCast(std.c.getpid()); @@ -321,16 +348,18 @@ pub fn randU64() u64 { } pub fn currentProcessId() u32 { + if (is_freestanding) return 0; if (is_windows) return @intCast(winsync.GetCurrentProcessId()); return @intCast(std.c.getpid()); } pub fn userId() usize { - if (is_windows) return 0; + if (is_windows or is_freestanding) return 0; return @intCast(std.c.getuid()); } pub fn sleepMs(ms: u64) void { + if (is_freestanding) return; if (is_windows) { winsync.Sleep(@intCast(@min(ms, @as(u64, std.math.maxInt(u32))))); } else { @@ -363,6 +392,7 @@ pub fn readFd(fd: c_int, buf: []u8) isize { } pub fn posixGetenv(name: []const u8) ?[]const u8 { + if (is_freestanding) return null; var buf: [256]u8 = undefined; if (name.len >= buf.len) return null; @memcpy(buf[0..name.len], name); @@ -990,6 +1020,7 @@ pub fn mmapReadonly(handle: anytype, len: usize) MmapError![]align(std.heap.page /// Release a view previously returned by mmapReadonly. pub fn munmap(data: []align(std.heap.page_size_min) const u8) void { + if (is_freestanding) return; if (is_windows) { _ = winmap.UnmapViewOfFile(data.ptr); } else { diff --git a/src/cli_proxy.zig b/src/cli_proxy.zig index d0713079..45e5fa9e 100644 --- a/src/cli_proxy.zig +++ b/src/cli_proxy.zig @@ -154,7 +154,7 @@ fn cliPipeMetadataPath(allocator: std.mem.Allocator, data_dir: []const u8) ![]u8 } fn cliRandomPipeName(buf: []u8) ?[:0]const u8 { - return std.fmt.bufPrintZ(buf, "\\\\.\\pipe\\codedb-{d}-{x:0>16}-{x:0>16}", .{ + return cio.bufPrintZ(buf, "\\\\.\\pipe\\codedb-{d}-{x:0>16}-{x:0>16}", .{ cio.currentProcessId(), secureRandomU64(), secureRandomU64(), @@ -450,7 +450,7 @@ const DaemonLock = if (builtin.os.tag == .windows) win.HANDLE else c_int; pub fn daemonLockTryAcquire(data_dir: []const u8) ?DaemonLock { var buf: [std.fs.max_path_bytes]u8 = undefined; - const p = std.fmt.bufPrintZ(&buf, "{s}/cli-daemon.lock", .{data_dir}) catch return null; + const p = cio.bufPrintZ(&buf, "{s}/cli-daemon.lock", .{data_dir}) catch return null; if (builtin.os.tag == .windows) { const path_w = windowsPathZ(std.heap.page_allocator, p) orelse return null; defer std.heap.page_allocator.free(path_w); @@ -556,7 +556,7 @@ fn cliSendYieldRequest(sa: SockAddr) void { /// Returns null if `shutdown` is set while waiting. pub fn cliAcquireListener(sock_path: []const u8, retry: bool, retry_interval_ms: u64, shutdown: *std.atomic.Value(bool)) ?c_int { var z_buf: [128]u8 = undefined; - const sock_path_z = std.fmt.bufPrintZ(&z_buf, "{s}", .{sock_path}) catch return null; + const sock_path_z = cio.bufPrintZ(&z_buf, "{s}", .{sock_path}) catch return null; const sa = cliFillSockaddr(sock_path) orelse return null; var yield_sent = false; while (true) { @@ -706,7 +706,7 @@ fn cliDaemonListenPosix(io: std.Io, allocator: std.mem.Allocator, explorer: *Exp return; }; var path_z_buf: [128]u8 = undefined; - const sock_path_z = std.fmt.bufPrintZ(&path_z_buf, "{s}", .{sock_path}) catch { + const sock_path_z = cio.bufPrintZ(&path_z_buf, "{s}", .{sock_path}) catch { shutdown.store(true, .release); return; }; diff --git a/src/codegraph.zig b/src/codegraph.zig index 687692a1..fc93e0a7 100644 --- a/src/codegraph.zig +++ b/src/codegraph.zig @@ -36,13 +36,13 @@ pub const FuncInput = struct { /// languages); over-filtering a rare real call only loses an additive boost. fn isCallKeyword(name: []const u8) bool { const kws = [_][]const u8{ - "if", "else", "for", "while", "switch", "return", "catch", - "try", "defer", "errdefer", "and", "or", "orelse", "sizeof", - "typeof", "do", "case", "when", "match", "with", "in", - "not", "is", "await", "yield", "throw", "new", "delete", - "fn", "func", "function", "def", "class", "struct", "enum", - "union", "const", "var", "let", "static", "assert", "where", - "select", "from", "foreach", "using", "unless", "until", "elif", + "if", "else", "for", "while", "switch", "return", "catch", + "try", "defer", "errdefer", "and", "or", "orelse", "sizeof", + "typeof", "do", "case", "when", "match", "with", "in", + "not", "is", "await", "yield", "throw", "new", "delete", + "fn", "func", "function", "def", "class", "struct", "enum", + "union", "const", "var", "let", "static", "assert", "where", + "select", "from", "foreach", "using", "unless", "until", "elif", }; for (kws) |kw| if (std.mem.eql(u8, name, kw)) return true; return false; diff --git a/src/commands.zig b/src/commands.zig index 8a9244bf..84d31484 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -323,7 +323,7 @@ pub fn runCliDaemon(ctx: *RunCtx) !void { var sock_buf: [128]u8 = undefined; if (cliSocketPath(&sock_buf, abs_root)) |sock_path| { var sock_z_buf: [128]u8 = undefined; - if (std.fmt.bufPrintZ(&sock_z_buf, "{s}", .{sock_path})) |sock_z| { + if (cio.bufPrintZ(&sock_z_buf, "{s}", .{sock_path})) |sock_z| { _ = std.c.unlink(sock_z.ptr); } else |_| {} } diff --git a/src/explore.zig b/src/explore.zig index 5ef95ee9..8e017169 100644 --- a/src/explore.zig +++ b/src/explore.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const ContentCache = @import("hot_cache.zig").ContentCache; const nanoregex = @import("nanoregex"); const cio = @import("cio.zig"); @@ -12,6 +13,8 @@ const SparseNgramIndex = idx.SparseNgramIndex; const codegraph = @import("codegraph.zig"); const git = @import("git.zig"); +const SearchGeneration = if (builtin.cpu.arch == .wasm32) u32 else u64; + /// Fast hash context for u32-keyed maps on hot paths (ranked search aggregation). /// Zig's AutoHashMap runs the 4 key bytes through Wyhash even for an integer key; /// std.hash.int is a couple of multiply/shift/xor ops with full avalanche. Pure @@ -1351,7 +1354,7 @@ pub const Explorer = struct { plain_render_cache: PlainRenderCache, /// Bumped (atomically — searches run under the SHARED lock) by every /// mutation that can change search results; see bumpSearchGen callers. - search_gen: std.atomic.Value(u64) = std.atomic.Value(u64).init(0), + search_gen: std.atomic.Value(SearchGeneration) = std.atomic.Value(SearchGeneration).init(0), /// Buffers adopted from snapshot loads (the raw outline_state section). /// Restored FileOutlines borrow their import/symbol strings as slices into /// these (see FileOutline.borrows_strings), so they must outlive every @@ -2515,6 +2518,7 @@ pub const Explorer = struct { if (self.contents.get(path)) |cached| { return .{ .data = cached, .owned = false, .allocator = allocator }; } + if (builtin.os.tag == .freestanding) return null; const io = self.io orelse return null; const dir = self.root_dir orelse std.Io.Dir.cwd(); const data = dir.readFileAlloc(io, path, allocator, .limited(64 * 1024 * 1024)) catch return null; @@ -3497,7 +3501,24 @@ pub const Explorer = struct { const SLOT_NONE = std.math.maxInt(u32); const SLOT_INVALID = SLOT_NONE - 1; const ndocs = self.word_index.id_to_path.items.len; - const use_slots = ndocs > 0 and (ndocs <= 4096 or (ndocs <= 65536 and word_hits.len >= 512)); + + // Fresh and persisted indexes keep postings in nondecreasing doc-id + // order, which guarantees one contiguous run per document. That is + // the overwhelmingly common read path, so avoid allocating and + // populating a dedup table for it. Incremental swap-removal from an + // older/mutated index can fragment a document into multiple runs; + // detect any ordering break and retain the defensive table below. + var postings_grouped = true; + var previous_doc = word_hits[0].doc_id; + for (word_hits[1..]) |hit| { + if (hit.doc_id < previous_doc) { + postings_grouped = false; + break; + } + previous_doc = hit.doc_id; + } + + const use_slots = !postings_grouped and ndocs > 0 and (ndocs <= 4096 or (ndocs <= 65536 and word_hits.len >= 512)); var slots: []u32 = &.{}; defer if (slots.len > 0) allocator.free(slots); var idx_by_doc = std.AutoHashMap(u32, u32).init(allocator); @@ -3505,7 +3526,7 @@ pub const Explorer = struct { if (use_slots) { slots = try allocator.alloc(u32, ndocs); @memset(slots, SLOT_NONE); - } else { + } else if (!postings_grouped) { idx_by_doc.ensureTotalCapacity(@intCast(@min(word_hits.len, 1024))) catch {}; } @@ -3523,6 +3544,7 @@ pub const Explorer = struct { defer run_start = run_end; var cur: u32 = blk: { + if (postings_grouped) break :blk SLOT_NONE; if (use_slots) { if (doc_id >= ndocs) break :blk SLOT_INVALID; break :blk slots[doc_id]; @@ -3569,10 +3591,12 @@ pub const Explorer = struct { cur = SLOT_INVALID; }; } - if (use_slots) { - if (doc_id < ndocs) slots[doc_id] = cur; - } else { - idx_by_doc.put(doc_id, cur) catch {}; + if (!postings_grouped) { + if (use_slots) { + if (doc_id < ndocs) slots[doc_id] = cur; + } else { + idx_by_doc.put(doc_id, cur) catch {}; + } } } if (cur != SLOT_INVALID) { @@ -4451,6 +4475,7 @@ pub const Explorer = struct { /// Caps query at 256 bytes, results at 50 entries, file at 10 MB /// (rotates by truncate-clobber). fn appendRerankTrace(self: *const Explorer, query: []const u8, results: []const SearchResult) void { + if (builtin.os.tag == .freestanding) return; const path = self.rerank_trace_path orelse return; const io_inst = self.io orelse return; @@ -4671,6 +4696,7 @@ pub const Explorer = struct { /// git repo — the attempted flag stops re-shelling. Mirrors /// ensureCallGraph: call while holding at least a shared lock on `mu`. fn ensureCoChange(self: *Explorer) void { + if (builtin.os.tag == .freestanding) return; if (self.co_change != null or self.co_change_attempted) return; self.cochange_build_mu.lock(); defer self.cochange_build_mu.unlock(); @@ -5342,15 +5368,18 @@ pub const Explorer = struct { /// Search for a word using the inverted word index. O(1) lookup. pub fn searchWord(self: *Explorer, word: []const u8, allocator: std.mem.Allocator) ![]const idx.WordHit { self.mu.lockShared(); + var shared_locked = true; + defer if (shared_locked) self.mu.unlockShared(); + const needs_rebuild = !self.word_index_complete and (self.contents.len() > 0 or (self.io != null and self.root_dir != null)); - self.mu.unlockShared(); if (needs_rebuild) { + self.mu.unlockShared(); + shared_locked = false; try self.rebuildWordIndex(); + self.mu.lockShared(); + shared_locked = true; } - - self.mu.lockShared(); - defer self.mu.unlockShared(); // #569: a multi-word query can never be a single index token — fall back // to per-token matching so phrase-shaped queries don't silently dead-end. if (std.mem.indexOfScalar(u8, word, ' ') != null) { @@ -7529,7 +7558,7 @@ pub fn regexMatch(haystack: []const u8, pattern: []const u8) bool { /// the SIMD anchor inside indexOfCaseInsensitive — lower is rarer. Anchor /// choice never affects correctness, only how often candidates verify. const code_char_freq: [256]u8 = blk: { - var t = [_]u8{3} ** 256; + var t: [256]u8 = @splat(3); const ranks = "zqjxkvbywgpfmucdlhrsnioate"; for (ranks, 0..) |c, i| t[c] = @intCast(i + 4); for ('0'..'9' + 1) |c| t[c] = 5; @@ -8599,7 +8628,7 @@ fn fuzzyFilenameStart(path: []const u8) usize { // chars can be rejected in O(path) without the DP — and would fail the LCS check // anyway, so no result changes. pub fn fuzzyPresenceReject(query: []const u8, path: []const u8) bool { - var present = [_]bool{false} ** 256; + var present: [256]bool = @splat(false); for (path) |c| present[toLowerByte(c)] = true; var hits: usize = 0; for (query) |c| { @@ -8729,9 +8758,9 @@ pub fn fuzzyScoreBatch(query: []const u8, paths: []const []const u8, out_best: * const BoolV = @Vector(FZ_LANES, bool); const count = paths.len; - var lens: [FZ_LANES]usize = .{0} ** FZ_LANES; - var len_arr: [FZ_LANES]u32 = .{0} ** FZ_LANES; - var fname_arr: [FZ_LANES]u32 = .{0} ** FZ_LANES; + var lens: [FZ_LANES]usize = @splat(0); + var len_arr: [FZ_LANES]u32 = @splat(0); + var fname_arr: [FZ_LANES]u32 = @splat(0); var max_len: usize = 0; for (0..count) |l| { lens[l] = paths[l].len; @@ -8748,9 +8777,9 @@ pub fn fuzzyScoreBatch(query: []const u8, paths: []const []const u8, out_best: * var po_t: [FUZZY_MAX_PATH]U8V = undefined; var wb_t: [FUZZY_MAX_PATH]BoolV = undefined; for (0..max_len) |j| { - var lo: [FZ_LANES]u8 = .{0} ** FZ_LANES; - var orig: [FZ_LANES]u8 = .{0} ** FZ_LANES; - var wbf: [FZ_LANES]bool = .{false} ** FZ_LANES; + var lo: [FZ_LANES]u8 = @splat(0); + var orig: [FZ_LANES]u8 = @splat(0); + var wbf: [FZ_LANES]bool = @splat(false); for (0..count) |l| { if (j < lens[l]) { const c = paths[l][j]; diff --git a/src/hot_cache.zig b/src/hot_cache.zig index 5010b14c..cac1c128 100644 --- a/src/hot_cache.zig +++ b/src/hot_cache.zig @@ -1,4 +1,7 @@ const std = @import("std"); +const builtin = @import("builtin"); + +const AtomicCounter = if (builtin.cpu.arch == .wasm32) u32 else u64; /// Fixed-capacity CLOCK eviction cache for file contents. /// Keys are always owned (duped on put, freed on eviction/remove/clear/deinit). @@ -13,9 +16,9 @@ pub const ContentCache = struct { capacity: u32, count_: u32, allocator: std.mem.Allocator, - hits_: std.atomic.Value(u64), - misses_: std.atomic.Value(u64), - evictions_: std.atomic.Value(u64), + hits_: std.atomic.Value(AtomicCounter), + misses_: std.atomic.Value(AtomicCounter), + evictions_: std.atomic.Value(AtomicCounter), /// Total bytes of cache-owned values (borrowed values are exempt). owned_bytes: usize, /// CLOCK hand for the global budget sweep (window eviction has its own). @@ -74,9 +77,9 @@ pub const ContentCache = struct { .capacity = capacity, .count_ = 0, .allocator = allocator, - .hits_ = std.atomic.Value(u64).init(0), - .misses_ = std.atomic.Value(u64).init(0), - .evictions_ = std.atomic.Value(u64).init(0), + .hits_ = std.atomic.Value(AtomicCounter).init(0), + .misses_ = std.atomic.Value(AtomicCounter).init(0), + .evictions_ = std.atomic.Value(AtomicCounter).init(0), .owned_bytes = 0, .sweep_hand = 0, .byte_budget = DEFAULT_BYTE_BUDGET, @@ -93,9 +96,9 @@ pub const ContentCache = struct { .capacity = capacity, .count_ = 0, .allocator = allocator, - .hits_ = std.atomic.Value(u64).init(0), - .misses_ = std.atomic.Value(u64).init(0), - .evictions_ = std.atomic.Value(u64).init(0), + .hits_ = std.atomic.Value(AtomicCounter).init(0), + .misses_ = std.atomic.Value(AtomicCounter).init(0), + .evictions_ = std.atomic.Value(AtomicCounter).init(0), .owned_bytes = 0, .sweep_hand = 0, .byte_budget = DEFAULT_BYTE_BUDGET, @@ -473,13 +476,13 @@ test "ContentCache: byte budget evicts owned values until the new value fits" { cache.byte_budget = 1000; cache.max_entry_bytes = 1000; - const v300 = "x" ** 300; - try cache.put("a", v300); - try cache.put("b", v300); - try cache.put("c", v300); + const v300: [300]u8 = @splat('x'); + try cache.put("a", &v300); + try cache.put("b", &v300); + try cache.put("c", &v300); try std.testing.expectEqual(@as(usize, 900), cache.stats().owned_bytes); - try cache.put("d", v300); + try cache.put("d", &v300); const s = cache.stats(); try std.testing.expect(s.owned_bytes <= 1000); try std.testing.expect(s.evictions >= 1); @@ -494,8 +497,8 @@ test "ContentCache: per-entry ceiling refuses oversized values and drops the sta try cache.put("k", "small"); try std.testing.expect(cache.get("k") != null); - const big = "y" ** 200; - try cache.put("k", big); + const big: [200]u8 = @splat('y'); + try cache.put("k", &big); try std.testing.expect(cache.get("k") == null); try std.testing.expectEqual(@as(usize, 0), cache.stats().owned_bytes); } @@ -505,8 +508,8 @@ test "ContentCache: borrowed values are exempt from the byte budget" { defer cache.deinit(); cache.byte_budget = 10; - const big = "z" ** 1000; - try cache.putBorrowed("snap", big); + const big: [1000]u8 = @splat('z'); + try cache.putBorrowed("snap", &big); try std.testing.expect(cache.get("snap") != null); try std.testing.expectEqual(@as(usize, 0), cache.stats().owned_bytes); try std.testing.expectEqual(@as(u64, 0), cache.stats().evictions); diff --git a/src/index.zig b/src/index.zig index c7e3632b..15016513 100644 --- a/src/index.zig +++ b/src/index.zig @@ -517,34 +517,46 @@ pub const WordIndex = struct { self.mmap_data = null; } - /// Look up hits, returning results allocated by the caller. - /// Deduplicates by (path, line_num). + /// Look up hits, returning results allocated by the caller and deduplicated + /// by (doc_id, line_num). Valid posting lists are pair-sorted, so their only + /// possible duplicates are adjacent and compact in one allocation without a + /// hash table. Retain a hash fallback for malformed, legacy, or incrementally + /// swap-removed lists whose ordering has been broken. pub fn searchDeduped(self: *WordIndex, word: []const u8, allocator: std.mem.Allocator) ![]const WordHit { const hits = self.search(word); - if (hits.len == 0) return try allocator.alloc(WordHit, 0); - if (hits.len == 1) { - var out = try allocator.alloc(WordHit, 1); - out[0] = hits[0]; - return out; + var result = try allocator.alloc(WordHit, hits.len); + errdefer allocator.free(result); + + var result_len: usize = 0; + var sorted = true; + for (hits) |hit| { + if (result_len > 0) { + const previous = result[result_len - 1]; + if (hit.doc_id < previous.doc_id or + (hit.doc_id == previous.doc_id and hit.line_num < previous.line_num)) + { + sorted = false; + break; + } + if (hit.doc_id == previous.doc_id and hit.line_num == previous.line_num) continue; + } + result[result_len] = hit; + result_len += 1; } + if (sorted) return try allocator.realloc(result, result_len); const DedupKey = struct { doc_id: u32, line_num: u32 }; var seen = std.AutoHashMap(DedupKey, void).init(allocator); defer seen.deinit(); try seen.ensureTotalCapacity(@intCast(hits.len)); - - var result: std.ArrayList(WordHit) = .empty; - errdefer result.deinit(allocator); - try result.ensureTotalCapacity(allocator, hits.len); - + result_len = 0; for (hits) |hit| { - const key = DedupKey{ .doc_id = hit.doc_id, .line_num = hit.line_num }; - const gop = try seen.getOrPut(key); - if (!gop.found_existing) { - result.appendAssumeCapacity(hit); - } + const gop = try seen.getOrPut(.{ .doc_id = hit.doc_id, .line_num = hit.line_num }); + if (gop.found_existing) continue; + result[result_len] = hit; + result_len += 1; } - return result.toOwnedSlice(allocator); + return try allocator.realloc(result, result_len); } /// Collect all hits for index keys that begin with `prefix_raw` (normalized internally to @@ -700,7 +712,7 @@ pub const WordIndex = struct { var file_buf: [256 * 1024]u8 = undefined; var writer = file.writer(io, &file_buf); - var header_buf: [51]u8 = [_]u8{0} ** 51; + var header_buf: [51]u8 = @splat(0); @memcpy(header_buf[0..4], &DISK_MAGIC); std.mem.writeInt(u16, header_buf[4..6], DISK_FORMAT_VERSION, .little); std.mem.writeInt(u32, header_buf[6..10], @intCast(file_table.items.len), .little); @@ -1401,11 +1413,12 @@ pub const TrigramIndex = struct { if (!idx_gop.found_existing) { idx_gop.value_ptr.* = .{ .path_to_id = &self.path_to_id }; } - try idx_gop.value_ptr.items.append(self.allocator, .{ - .doc_id = doc_id, - .next_mask = mask.next_mask, - .loc_mask = mask.loc_mask, - }); + // removeFile can free and then reuse a lower doc_id, so append + // would violate PostingList's sorted-doc-ID invariant. Mmap-backed + // persistence relies on that ordering for its merge/binary searches. + const posting = try idx_gop.value_ptr.getOrAddPosting(self.allocator, doc_id); + posting.next_mask = mask.next_mask; + posting.loc_mask = mask.loc_mask; try tri_list.append(self.allocator, tri); } try self.file_trigrams.put(path, tri_list); @@ -1717,40 +1730,27 @@ pub const TrigramIndex = struct { }; /// Write the current in-memory index to disk in a two-file format. - /// Files are written atomically (write to tmp, then rename). + /// Each file is replaced via tmp + rename; publishing the pair is not transactional. pub fn writeToDisk(self: *TrigramIndex, io: std.Io, dir_path: []const u8, git_head: ?[40]u8) !void { - // Step 1: Build file table from path_to_id (reuse existing doc IDs for consistency) + // Step 1: Build the file table in ascending in-memory doc_id order. Posting + // lists are sorted by doc_id, so this makes their remapped disk file_ids + // sorted too — an invariant required by the mmap merge and binary searches. + // Retain the existing tracked-file selection while avoiding an intermediate + // path→disk-ID map and its second lookup pass. var file_table: std.ArrayList([]const u8) = .empty; defer file_table.deinit(self.allocator); - var disk_path_to_id = std.StringHashMap(u32).init(self.allocator); - defer disk_path_to_id.deinit(); - - if (self.file_trigrams.count() > 0) { - var ft_iter = self.file_trigrams.keyIterator(); - while (ft_iter.next()) |path_ptr| { - const id: u32 = @intCast(file_table.items.len); - try file_table.append(self.allocator, path_ptr.*); - try disk_path_to_id.put(path_ptr.*, id); - } - } else { - for (self.id_to_path.items) |path| { - if (path.len == 0) continue; - const id: u32 = @intCast(file_table.items.len); - try file_table.append(self.allocator, path); - try disk_path_to_id.put(path, id); - } - } - - // Precompute doc_id -> disk file_id ONCE (one path hash per doc), so the - // Step-3 posting loop does an array load instead of hashing the full path - // string per posting (tens of millions of times on a large repo). Mirrors - // the WordIndex doc_to_disk fix (#475). const doc_to_disk = try self.allocator.alloc(u32, self.id_to_path.items.len); defer self.allocator.free(doc_to_disk); @memset(doc_to_disk, std.math.maxInt(u32)); + const use_tracked_files = self.file_trigrams.count() > 0; for (self.id_to_path.items, 0..) |path, doc_idx| { if (path.len == 0) continue; - doc_to_disk[doc_idx] = disk_path_to_id.get(path) orelse std.math.maxInt(u32); + if (use_tracked_files and !self.file_trigrams.contains(path)) continue; + if (path.len > std.math.maxInt(u16)) return error.NameTooLong; + if (file_table.items.len >= std.math.maxInt(u32)) return error.IndexTooLarge; + const disk_id: u32 = @intCast(file_table.items.len); + try file_table.append(self.allocator, path); + doc_to_disk[doc_idx] = disk_id; } const file_count: u32 = @intCast(file_table.items.len); @@ -1770,28 +1770,46 @@ pub const TrigramIndex = struct { } }.lt); - // Step 3: Stream postings to the temporary file while retaining only - // the compact lookup table. Materializing every DiskPosting first can - // briefly double the memory required by a large cold-built index and - // then makes the allocator copy the whole postings blob again on grow. - var lookup_entries: std.ArrayList(LookupEntry) = .empty; - defer lookup_entries.deinit(self.allocator); + // Step 3: The sorted trigram list determines the lookup header count. + // Stream both payloads in fixed batches so auxiliary persistence memory + // remains bounded by the two batches rather than index cardinality. + if (trigrams_sorted.items.len > std.math.maxInt(u32)) return error.IndexTooLarge; + const lookup_entry_count: u32 = @intCast(trigrams_sorted.items.len); - // Step 4: Write postings file atomically (random suffix prevents collisions) + // Step 4: Write postings and lookup files to random-suffixed temporaries. const post_rand: u64 = cio.randU64(); const postings_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.postings.{x}.tmp", .{ dir_path, post_rand }); defer self.allocator.free(postings_tmp); const postings_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.postings", .{dir_path}); defer self.allocator.free(postings_final); + const lk_rand: u64 = cio.randU64(); + const lookup_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup.{x}.tmp", .{ dir_path, lk_rand }); + defer self.allocator.free(lookup_tmp); + const lookup_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup", .{dir_path}); + defer self.allocator.free(lookup_final); { - const file = try std.Io.Dir.cwd().createFile(io, postings_tmp, .{}); - defer file.close(io); + const postings_file = try std.Io.Dir.cwd().createFile(io, postings_tmp, .{}); + defer postings_file.close(io); + const lookup_file = try std.Io.Dir.cwd().createFile(io, lookup_tmp, .{}); + defer lookup_file.close(io); var pw_buf: [256 * 1024]u8 = undefined; - var pw = file.writer(io, &pw_buf); + var pw = postings_file.writer(io, &pw_buf); + var lw_buf: [64 * 1024]u8 = undefined; + var lw = lookup_file.writer(io, &lw_buf); - // Header v3: magic(4) + version(2) + file_count(4) + head_len(1) + head(40) = 51 bytes + // Lookup header: magic(4) + version(2) + pad(2) + entry_count(4) = 12 bytes + try lw.interface.writeAll(&LOOKUP_MAGIC); + var lookup_version_buf: [2]u8 = undefined; + std.mem.writeInt(u16, &lookup_version_buf, FORMAT_VERSION, .little); + try lw.interface.writeAll(&lookup_version_buf); + try lw.interface.writeAll(&[_]u8{ 0, 0 }); + var lookup_count_buf: [4]u8 = undefined; + std.mem.writeInt(u32, &lookup_count_buf, lookup_entry_count, .little); + try lw.interface.writeAll(&lookup_count_buf); + + // Postings header v3: magic(4) + version(2) + file_count(4) + head_len(1) + head(40) = 51 bytes try pw.interface.writeAll(&POSTINGS_MAGIC); var ver_buf: [2]u8 = undefined; std.mem.writeInt(u16, &ver_buf, FORMAT_VERSION, .little); @@ -1805,7 +1823,7 @@ pub const TrigramIndex = struct { try pw.interface.writeAll(&head); } else { try pw.interface.writeAll(&.{0}); - try pw.interface.writeAll(&([_]u8{0} ** 40)); + try pw.interface.writeAll(&@as([40]u8, @splat(0))); } // File table: for each file, path_len(u16) + path bytes @@ -1820,10 +1838,12 @@ pub const TrigramIndex = struct { // too so the inner loop does not dispatch one virtual write per // posting. The batch is fixed-size, keeping peak extra memory flat. var posting_batch: [4096]DiskPosting = undefined; - var batch_len: usize = 0; + var posting_batch_len: usize = 0; + var lookup_batch: [4096]LookupEntry = undefined; + var lookup_batch_len: usize = 0; var postings_len: usize = 0; for (trigrams_sorted.items) |tri| { - const posting_list = self.index.getPtr(tri) orelse continue; + const posting_list = self.index.getPtr(tri) orelse unreachable; const offset: u32 = @intCast(postings_len); var count: u32 = 0; for (posting_list.items.items) |p| { @@ -1835,62 +1855,40 @@ pub const TrigramIndex = struct { // Reject instead of wrapping once its representable range // is full; a corrupt index would otherwise be persisted. if (postings_len >= std.math.maxInt(u32) or count >= std.math.maxInt(u32)) return error.IndexTooLarge; - posting_batch[batch_len] = .{ + posting_batch[posting_batch_len] = .{ .file_id = fid, .next_mask = p.next_mask, .loc_mask = p.loc_mask, }; - batch_len += 1; + posting_batch_len += 1; postings_len += 1; count += 1; - if (batch_len == posting_batch.len) { - try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..batch_len])); - batch_len = 0; + if (posting_batch_len == posting_batch.len) { + try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..posting_batch_len])); + posting_batch_len = 0; } } - try lookup_entries.append(self.allocator, .{ + lookup_batch[lookup_batch_len] = .{ .trigram = @as(u32, tri), .offset = offset, .count = count, - }); + }; + lookup_batch_len += 1; + if (lookup_batch_len == lookup_batch.len) { + try lw.interface.writeAll(std.mem.sliceAsBytes(lookup_batch[0..lookup_batch_len])); + lookup_batch_len = 0; + } } - if (batch_len > 0) { - try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..batch_len])); + if (posting_batch_len > 0) { + try pw.interface.writeAll(std.mem.sliceAsBytes(posting_batch[0..posting_batch_len])); + } + if (lookup_batch_len > 0) { + try lw.interface.writeAll(std.mem.sliceAsBytes(lookup_batch[0..lookup_batch_len])); } try pw.interface.flush(); - } - try std.Io.Dir.cwd().rename(postings_tmp, std.Io.Dir.cwd(), postings_final, io); - - // Step 5: Write lookup file atomically (random suffix prevents collisions) - const lk_rand: u64 = cio.randU64(); - const lookup_tmp = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup.{x}.tmp", .{ dir_path, lk_rand }); - defer self.allocator.free(lookup_tmp); - const lookup_final = try std.fmt.allocPrint(self.allocator, "{s}/trigram.lookup", .{dir_path}); - defer self.allocator.free(lookup_final); - - { - const file = try std.Io.Dir.cwd().createFile(io, lookup_tmp, .{}); - defer file.close(io); - - var lw_buf: [64 * 1024]u8 = undefined; - var lw = file.writer(io, &lw_buf); - - // Header: magic(4) + version(2) + pad(2) + entry_count(4) = 12 bytes - try lw.interface.writeAll(&LOOKUP_MAGIC); - var ver_buf2: [2]u8 = undefined; - std.mem.writeInt(u16, &ver_buf2, FORMAT_VERSION, .little); - try lw.interface.writeAll(&ver_buf2); - var pad_buf: [2]u8 = .{ 0, 0 }; - try lw.interface.writeAll(&pad_buf); - var ec_buf: [4]u8 = undefined; - std.mem.writeInt(u32, &ec_buf, @intCast(lookup_entries.items.len), .little); - try lw.interface.writeAll(&ec_buf); - - // Entries (already aligned at 12 bytes each) - const entry_bytes = std.mem.sliceAsBytes(lookup_entries.items); - try lw.interface.writeAll(entry_bytes); try lw.interface.flush(); } + try std.Io.Dir.cwd().rename(postings_tmp, std.Io.Dir.cwd(), postings_final, io); try std.Io.Dir.cwd().rename(lookup_tmp, std.Io.Dir.cwd(), lookup_final, io); } @@ -3146,7 +3144,7 @@ pub const MAX_NGRAM_LEN: usize = 16; /// Rare pairs → HIGH weight (they become n-gram boundaries). /// All unspecified pairs default to 0xFE00 (rare = high weight). pub const default_pair_freq: [256][256]u16 = blk: { - var table: [256][256]u16 = .{.{0xFE00} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0xFE00)); // English bigrams (lowercase) — common in identifiers and prose table['t']['h'] = 0x1000; table['h']['e'] = 0x1000; @@ -3244,7 +3242,7 @@ pub fn resetFrequencyTable() void { /// Build a per-project frequency table by counting byte-pair occurrences in /// `content`, then inverting counts to weights (common → low, rare → high). pub fn buildFrequencyTable(content: []const u8) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); if (content.len >= 2) { for (0..content.len - 1) |i| { counts[content[i]][content[i + 1]] += 1; @@ -3257,7 +3255,7 @@ pub fn buildFrequencyTable(content: []const u8) [256][256]u16 { /// Zero extra memory — counts pairs within each slice, skipping cross-slice /// boundaries (negligible loss for large corpora). pub fn buildFrequencyTableFromSlices(slices: []const []const u8) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); for (slices) |content| { if (content.len < 2) continue; for (0..content.len - 1) |i| { @@ -3270,7 +3268,7 @@ pub fn buildFrequencyTableFromSlices(slices: []const []const u8) [256][256]u16 { /// Build a frequency table by streaming over a StringHashMap of content. /// Iterates file-by-file — no concatenation, zero extra memory. pub fn buildFrequencyTableFromMap(contents: *ContentCache) [256][256]u16 { - var counts: [256][256]u64 = .{.{0} ** 256} ** 256; + var counts: [256][256]u64 = @splat(@splat(0)); var iter = contents.iterator(); while (iter.next()) |entry| { const content = entry.value_ptr.*; @@ -3288,7 +3286,7 @@ pub fn buildFrequencyTableFromMap(contents: *ContentCache) [256][256]u16 { pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: std.mem.Allocator, worker_count: usize) ![256][256]u16 { const n_files = contents.count(); if (n_files == 0) { - const zero: [256][256]u64 = .{.{0} ** 256} ** 256; + const zero: [256][256]u64 = @splat(@splat(0)); return finishFrequencyTable(&zero); } @@ -3301,7 +3299,7 @@ pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: st if (content.len >= 2) slices.appendAssumeCapacity(content); } if (slices.items.len == 0) { - const zero: [256][256]u64 = .{.{0} ** 256} ** 256; + const zero: [256][256]u64 = @splat(@splat(0)); return finishFrequencyTable(&zero); } @@ -3332,7 +3330,7 @@ pub fn buildFrequencyTableFromMapParallel(contents: *ContentCache, allocator: st for (threads[0..spawned]) |t| t.join(); // Sum worker tables. - var merged: Counts = .{.{0} ** 256} ** 256; + var merged: Counts = @splat(@splat(0)); for (worker_counts) |*wc| { for (0..256) |a| { for (0..256) |b| { @@ -3359,7 +3357,7 @@ fn finishFrequencyTable(counts: *const [256][256]u64) [256][256]u16 { } } // Invert: count 0 → 0xFE00 (rare, high); max_count → 0x1000 (common, low). - var table: [256][256]u16 = .{.{0xFE00} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0xFE00)); for (0..256) |a| { for (0..256) |b| { const c = counts[a][b]; diff --git a/src/linter.zig b/src/linter.zig index 918997ec..332101c5 100644 --- a/src/linter.zig +++ b/src/linter.zig @@ -528,8 +528,8 @@ pub const DiagnosticsCache = struct { mu: cio.Mutex = .{}, alloc: std.mem.Allocator, - entries: [MAX]?Entry = .{null} ** MAX, - pending: [MAX]?[]u8 = .{null} ** MAX, + entries: [MAX]?Entry = @splat(null), + pending: [MAX]?[]u8 = @splat(null), inflight: usize = 0, pub fn init(alloc: std.mem.Allocator) DiagnosticsCache { diff --git a/src/main.zig b/src/main.zig index 3e3fee82..555e29e0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -39,11 +39,11 @@ const mcpRootIsImplicitCwd = cli_args.mcpRootIsImplicitCwd; const mcpRootAcceptsEnv = cli_args.mcpRootAcceptsEnv; const isHelpRequest = cli_args.isHelpRequest; -/// The real entry point. In Debug builds, Zig may merge all command-branch +/// The real entry point. In Debug builds, Zig may merge all command-branch /// stack frames into one producing a frame that overflows the default OS stack, -/// so we trampoline through a thread with an explicit 64 MB stack. -/// In optimised builds the merged frame is ~190 KB, so 8 MB is ample and -/// avoids triggering Rosetta 2's 64 MB stack allocation bug on x86_64-macos. +/// so those builds trampoline through a thread with an explicit 64 MB stack. +/// Optimized builds have a ~190 KB frame and run directly on the process stack, +/// avoiding an OS thread create/join on every CLI invocation. /// /// #504: must have a non-error-union return type. A Zig binary with /// `pub fn main(...) !void` ad-hoc-signed and run via Rosetta (or, in the @@ -59,7 +59,11 @@ pub fn main(init: std.process.Init.Minimal) void { const argv = cio.bootstrapArgs(init.args); cio.setProcessArgs(argv); if (handleFastPath(argv)) return; - mainTrampoline() catch |err| { + if (builtin.mode != .Debug) { + mainInner(argv); + return; + } + mainTrampoline(argv) catch |err| { // Surface the failure on stderr so users see something even if the // worker thread crashes during startup. var buf: [256]u8 = undefined; @@ -70,9 +74,8 @@ pub fn main(init: std.process.Init.Minimal) void { }; } -fn mainTrampoline() !void { - const stack_size: usize = if (builtin.mode == .Debug) 64 * 1024 * 1024 else 8 * 1024 * 1024; - const thread = try std.Thread.spawn(.{ .stack_size = stack_size }, mainInner, .{}); +fn mainTrampoline(argv: []const [*:0]const u8) !void { + const thread = try std.Thread.spawn(.{ .stack_size = 64 * 1024 * 1024 }, mainInner, .{argv}); thread.join(); } @@ -107,8 +110,8 @@ fn handleFastPath(argv: []const [*:0]const u8) bool { return false; } -fn mainInner() void { - mainImpl() catch |err| { +fn mainInner(argv: []const [*:0]const u8) void { + mainImpl(argv) catch |err| { std.debug.print("fatal: {s}\n", .{@errorName(err)}); std.process.exit(1); }; @@ -127,7 +130,7 @@ const loadUserConfig = bootstrap.loadUserConfig; const getDataDir = bootstrap.getDataDir; const commands = @import("commands.zig"); -fn mainImpl() !void { +fn mainImpl(argv: []const [*:0]const u8) !void { // Use c_allocator (libc malloc) — better page reclamation than GPA const allocator = std.heap.c_allocator; cio.ignoreSigpipe(); @@ -145,9 +148,6 @@ fn mainImpl() !void { var out = Out{ .file = stdout, .alloc = allocator }; defer out.flush(); - const raw_args = try cio.argsAlloc(allocator); - defer cio.argsFree(allocator, raw_args); - // Extract --config-file= / --config-file before positional // arg parsing so a leading `--config-file=X` isn't misread as the root. // See #101, #102. @@ -156,15 +156,15 @@ fn mainImpl() !void { const args = blk: { var filtered: std.ArrayList([]const u8) = .empty; errdefer filtered.deinit(allocator); - try filtered.append(allocator, raw_args[0]); + try filtered.append(allocator, std.mem.span(argv[0])); var i: usize = 1; - while (i < raw_args.len) : (i += 1) { - const a = raw_args[i]; + while (i < argv.len) : (i += 1) { + const a = std.mem.span(argv[i]); if (std.mem.startsWith(u8, a, "--config-file=")) { explicit_config = a["--config-file=".len..]; continue; - } else if (std.mem.eql(u8, a, "--config-file") and i + 1 < raw_args.len) { - explicit_config = raw_args[i + 1]; + } else if (std.mem.eql(u8, a, "--config-file") and i + 1 < argv.len) { + explicit_config = std.mem.span(argv[i + 1]); i += 1; continue; } else if (std.mem.eql(u8, a, "--no-telemetry")) { diff --git a/src/mcp.zig b/src/mcp.zig index d4a852be..3fd7b038 100644 --- a/src/mcp.zig +++ b/src/mcp.zig @@ -2,13 +2,15 @@ const cio = @import("cio.zig"); // // Exposes codedb's exploration + edit engine as MCP tools. -// Uses mcp-zig for protocol utilities; adds roots support for workspace awareness. +// Implements the JSON-RPC transport directly and adds roots support for workspace awareness. const std = @import("std"); const testing = std.testing; -const mcp_lib = @import("mcp"); -const mcpj = mcp_lib.json; -pub const Root = mcp_lib.mcp.Root; +const mcpj = @import("mcp_json.zig"); +pub const Root = struct { + uri: []u8, + name: []u8, +}; const Store = @import("store.zig").Store; const explore_mod = @import("explore.zig"); const Explorer = explore_mod.Explorer; @@ -329,7 +331,7 @@ const ProjectCache = struct { return .{ .mu = .{}, .alloc = alloc_, - .entries = [_]?*Entry{null} ** MAX_CACHED, + .entries = @splat(null), .default_path = default_path_, .default_snapshot_cache = .{}, .default_deps_cache = .{}, @@ -857,7 +859,7 @@ pub const ConvergenceGovernor = struct { pub const HISTORY = 8; // ring-buffer window of recent calls pub const WARN_AT = 3; // same signature this many times in the window -> nudge - sigs: [HISTORY]u64 = [_]u64{0} ** HISTORY, + sigs: [HISTORY]u64 = @splat(0), head: usize = 0, /// Record a call signature and return how many times it has occurred within diff --git a/src/mcp_json.zig b/src/mcp_json.zig new file mode 100644 index 00000000..13c04f74 --- /dev/null +++ b/src/mcp_json.zig @@ -0,0 +1,97 @@ +//! Small JSON-RPC helpers used by codedb's MCP transport. +//! +//! These were formerly supplied by mcp-zig. Keeping the protocol-neutral +//! helpers local avoids coupling codedb's build to a second server framework. + +const std = @import("std"); + +pub const MAX_LINE = 1024 * 1024; + +/// Read one newline-terminated message from a buffered reader. +/// The caller owns the returned slice. +pub fn readLineBuf(alloc: std.mem.Allocator, reader: *std.Io.Reader) ?[]u8 { + var line: std.ArrayList(u8) = .empty; + while (true) { + const byte = reader.takeByte() catch { + if (line.items.len == 0) { + line.deinit(alloc); + return null; + } + return line.toOwnedSlice(alloc) catch null; + }; + if (byte == '\n') return line.toOwnedSlice(alloc) catch null; + line.append(alloc, byte) catch { + line.deinit(alloc); + return null; + }; + if (line.items.len > MAX_LINE) { + line.deinit(alloc); + return null; + } + } +} + +pub fn getStr(obj: *const std.json.ObjectMap, key: []const u8) ?[]const u8 { + return switch (obj.get(key) orelse return null) { + .string => |s| s, + else => null, + }; +} + +pub fn getInt(obj: *const std.json.ObjectMap, key: []const u8) ?i64 { + return switch (obj.get(key) orelse return null) { + .integer => |n| n, + else => null, + }; +} + +pub fn getBool(obj: *const std.json.ObjectMap, key: []const u8) bool { + return switch (obj.get(key) orelse return false) { + .bool => |b| b, + else => false, + }; +} + +pub fn eql(a: []const u8, b: []const u8) bool { + return std.mem.eql(u8, a, b); +} + +/// Append a JSON-escaped string without adding surrounding quotes. +pub fn writeEscaped(alloc: std.mem.Allocator, out: *std.ArrayList(u8), s: []const u8) void { + const Vec = @Vector(16, u8); + var i: usize = 0; + + while (i < s.len) { + const start = i; + while (i + 16 <= s.len) { + const chunk: Vec = s[i..][0..16].*; + const lo = chunk < @as(Vec, @splat(@as(u8, 0x20))); + const dq = chunk == @as(Vec, @splat(@as(u8, '"'))); + const bs = chunk == @as(Vec, @splat(@as(u8, '\\'))); + if (@reduce(.Or, lo | dq | bs)) break; + i += 16; + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c < 0x20 or c == '"' or c == '\\') break; + } + + if (i > start) out.appendSlice(alloc, s[start..i]) catch return; + if (i >= s.len) break; + + const c = s[i]; + switch (c) { + '"' => out.appendSlice(alloc, "\\\"") catch return, + '\\' => out.appendSlice(alloc, "\\\\") catch return, + '\n' => out.appendSlice(alloc, "\\n") catch return, + '\r' => out.appendSlice(alloc, "\\r") catch return, + '\t' => out.appendSlice(alloc, "\\t") catch return, + else => { + const hex = "0123456789abcdef"; + const escaped = [6]u8{ '\\', 'u', '0', '0', hex[c >> 4], hex[c & 0x0f] }; + out.appendSlice(alloc, &escaped) catch return; + }, + } + i += 1; + } +} diff --git a/src/release_info.zig b/src/release_info.zig index 79197512..0a6e3bb1 100644 --- a/src/release_info.zig +++ b/src/release_info.zig @@ -1 +1 @@ -pub const semver = "0.2.5828"; +pub const semver = "0.2.5829"; diff --git a/src/server.zig b/src/server.zig index d262ffed..9531150e 100644 --- a/src/server.zig +++ b/src/server.zig @@ -687,13 +687,11 @@ fn handleConnection( // ── Transport helpers ─────────────────────────────────────────── /// Read some bytes from the TCP stream into `dest`. Returns 0 on clean EOF. -/// This is the direct-vtable analogue of the old `conn.stream.read`: no -/// buffered reader state, so each call issues one `netRead` syscall. +/// This uses the low-level stream API without allocating buffered reader state. fn readSome(io: std.Io, stream: std.Io.net.Stream, dest: []u8) !usize { if (dest.len == 0) return 0; var iov: [1][]u8 = .{dest}; - const n = try io.vtable.netRead(io.userdata, stream.socket.handle, &iov); - return n; + return stream.read(io, &iov); } // ── Response helpers ──────────────────────────────────────────── diff --git a/src/snapshot.zig b/src/snapshot.zig index 14a88098..8039aba9 100644 --- a/src/snapshot.zig +++ b/src/snapshot.zig @@ -354,7 +354,7 @@ pub fn writeSnapshot( if (git_head) |head| { try fw.writeAll(&head); } else { - try fw.writeAll(&([_]u8{0x00} ** 40)); + try fw.writeAll(&@as([40]u8, @splat(0x00))); } var sc_buf: [4]u8 = undefined; diff --git a/src/telemetry.zig b/src/telemetry.zig index af80234d..3fc2e8c6 100644 --- a/src/telemetry.zig +++ b/src/telemetry.zig @@ -37,7 +37,7 @@ pub const Event = struct { pub const Kind = union(enum) { tool_call: struct { - tool: [32]u8 = .{0} ** 32, + tool: [32]u8 = @splat(0), tool_len: u8 = 0, latency_ns: i128, err: bool, @@ -339,9 +339,9 @@ pub const Telemetry = struct { }, .search_breakdown => |sb| { try w.print(",\"event_type\":\"search_breakdown\",\"tier0_ns\":{d},\"tier05_ns\":{d},\"tier1_ns\":{d},\"tier2_ns\":{d},\"tier3_ns\":{d},\"tier4_ns\":{d},\"tier5_ns\":{d},\"rerank_ns\":{d},\"tier_reached\":{d},\"candidates\":{d},\"results\":{d}", .{ - sb.tier0_ns, sb.tier05_ns, sb.tier1_ns, - sb.tier2_ns, sb.tier3_ns, sb.tier4_ns, - sb.tier5_ns, sb.rerank_ns, sb.tier_reached, + sb.tier0_ns, sb.tier05_ns, sb.tier1_ns, + sb.tier2_ns, sb.tier3_ns, sb.tier4_ns, + sb.tier5_ns, sb.rerank_ns, sb.tier_reached, sb.candidate_count, sb.result_count, }); }, diff --git a/src/test_core.zig b/src/test_core.zig index a9bc2a6d..a46f8784 100644 --- a/src/test_core.zig +++ b/src/test_core.zig @@ -1048,7 +1048,7 @@ test "issue-584: ContentCache probe-window — overflow inserts, holes, and dupl var bufs: [5][16]u8 = undefined; var lens: [5]usize = undefined; var nkeys: usize = 0; - var counts = [_]u8{0} ** 64; + var counts: [64]u8 = @splat(0); var pick: u32 = 0; var i: usize = 0; while (i < 4096) : (i += 1) { diff --git a/src/test_explore.zig b/src/test_explore.zig index f3e63b76..9314a192 100644 --- a/src/test_explore.zig +++ b/src/test_explore.zig @@ -139,7 +139,6 @@ const watcher = @import("watcher.zig"); const git_mod = @import("git.zig"); const snapshot_mod = @import("snapshot.zig"); - test "word tokenizer" { var tok = WordTokenizer{ .buf = "pub fn main() !void {" }; const w1 = tok.next().?; @@ -153,7 +152,6 @@ test "word tokenizer" { try testing.expect(tok.next() == null); } - test "word index: index and search" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -175,7 +173,6 @@ test "word index: index and search" { try testing.expect(const_hits[0].line_num == 2); } - test "word index: re-index clears old entries" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -188,7 +185,6 @@ test "word index: re-index clears old entries" { try testing.expect(wi.search("new_func").len > 0); } - test "word index: removeFile" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -200,7 +196,6 @@ test "word index: removeFile" { try testing.expect(wi.search("hello").len == 0); } - test "word index: deduped search" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -213,7 +208,6 @@ test "word index: deduped search" { try testing.expect(hits.len == 1); } - test "explorer: sparse ngram index integrated into searchContent" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -227,7 +221,6 @@ test "explorer: sparse ngram index integrated into searchContent" { try testing.expectEqualStrings("src/alpha.zig", results[0].path); } - test "explorer: searchContent finds query embedded in longer identifier" { // Verify that searchContent correctly finds files whose content contains // the query string. The sparse index (sliding-window) and trigram index @@ -248,7 +241,6 @@ test "explorer: searchContent finds query embedded in longer identifier" { try testing.expect(found); } - test "explorer: packed outline clone owns one string backing" { const expected_mask = explore.FileOutline.nameLenBit("packedSymbol".len); var packed_outline = blk: { @@ -317,7 +309,6 @@ test "explorer: index file and get outline" { try testing.expect(outline.symbols.items.len == 3); } - test "explorer: findSymbol" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -331,7 +322,6 @@ test "explorer: findSymbol" { try testing.expectEqualStrings("a.zig", result.?.path); } - test "explorer: findAllSymbols returns multiple" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -345,7 +335,6 @@ test "explorer: findAllSymbols returns multiple" { try testing.expect(results.len == 2); } - test "explorer: searchContent with trigram acceleration" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -368,7 +357,6 @@ test "explorer: searchContent with trigram acceleration" { try testing.expect(results[0].line_num == 1); } - test "explorer: searchWord via inverted index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -382,6 +370,52 @@ test "explorer: searchWord via inverted index" { try testing.expectEqualStrings("math.zig", explorer.word_index.hitPath(hits[0])); } +test "explorer: searchWord lazy rebuild matches eager index" { + var eager = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer eager.deinit(); + var lazy = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer lazy.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "src/a.zig", .content = "pub fn alpha() void {}\n// alpha again\n" }, + .{ .path = "src/b.zig", .content = "const alpha = 1;\n" }, + }; + for (fixtures) |fixture| { + try eager.indexFile(fixture.path, fixture.content); + try lazy.indexFile(fixture.path, fixture.content); + } + lazy.markWordIndexIncomplete(false); + + const expected = try eager.searchWord("alpha", testing.allocator); + defer testing.allocator.free(expected); + const rebuilt = try lazy.searchWord("alpha", testing.allocator); + defer testing.allocator.free(rebuilt); + try testing.expectEqual(expected.len, rebuilt.len); + // Rebuild walks the content hash map, so internal doc IDs and posting order + // are intentionally unspecified. Compare the complete semantic hit set. + for (expected) |a| { + var found = false; + for (rebuilt) |b| { + if (a.line_num == b.line_num and + std.mem.eql(u8, eager.word_index.hitPath(a), lazy.word_index.hitPath(b))) + { + found = true; + break; + } + } + try testing.expect(found); + } + + // The second call takes the optimized already-complete shared-lock path and + // must preserve the rebuilt index's exact sequence. + const warm = try lazy.searchWord("alpha", testing.allocator); + defer testing.allocator.free(warm); + try testing.expectEqual(rebuilt.len, warm.len); + for (rebuilt, warm) |a, b| { + try testing.expectEqual(a.doc_id, b.doc_id); + try testing.expectEqual(a.line_num, b.line_num); + } +} test "explorer: removeFile cleans up everything" { var arena = std.heap.ArenaAllocator.init(testing.allocator); @@ -397,7 +431,6 @@ test "explorer: removeFile cleans up everything" { try testing.expect((try explorer.findSymbol("doStuff", testing.allocator)) == null); } - test "explorer: python parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -415,7 +448,6 @@ test "explorer: python parser" { try testing.expect(outline.symbols.items.len == 3); // import, class, def } - test "explorer: typescript parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -432,7 +464,6 @@ test "explorer: typescript parser" { try testing.expect(outline.symbols.items.len >= 3); } - test "explorer: reindex OOM keeps prior outline reachable" { // Use a real allocator for the explorer so the first indexFile always succeeds. // We can't use FailingAllocator for the whole explorer because deinit would crash. @@ -475,7 +506,6 @@ test "explorer: reindex OOM keeps prior outline reachable" { try testing.expect(new_results.len == 1); } - test "explorer: getOutline clone OOM preserves source outline" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -510,7 +540,6 @@ test "explorer: getOutline clone OOM preserves source outline" { try testing.expect(induced_oom); } - test "explorer: outline copy survives source removal" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -526,7 +555,6 @@ test "explorer: outline copy survives source removal" { try testing.expect(outline.symbols.items.len > 0); } - test "explorer: removeFile frees owned map key" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -544,7 +572,6 @@ test "explorer: removeFile frees owned map key" { try testing.expect(explorer.dep_graph.count() == 0); } - test "word index: removeFile prunes empty buckets" { var wi = WordIndex.init(testing.allocator); defer wi.deinit(); @@ -558,7 +585,6 @@ test "word index: removeFile prunes empty buckets" { try testing.expect(wi.search("uniqueWordOnlyHere").len == 0); } - test "extractLines: basic range with line numbers" { const content = "line1\nline2\nline3\nline4\nline5"; const result = try extractLines(content, 2, 4, true, false, .unknown, testing.allocator); @@ -570,7 +596,6 @@ test "extractLines: basic range with line numbers" { try testing.expect(std.mem.indexOf(u8, result, "line5") == null); } - test "extractLines: start beyond file returns empty" { const content = "line1\nline2"; const result = try extractLines(content, 10, 20, true, false, .unknown, testing.allocator); @@ -578,7 +603,6 @@ test "extractLines: start beyond file returns empty" { try testing.expect(result.len == 0); } - test "extractLines: compact skips comments and blanks" { const content = "fn main() void {}\n// this is a comment\n\n return 0;\n}"; const result = try extractLines(content, 1, 5, false, true, .zig, testing.allocator); @@ -589,7 +613,6 @@ test "extractLines: compact skips comments and blanks" { try testing.expect(std.mem.indexOf(u8, result, "return 0") != null); } - test "isCommentOrBlank: detects language-specific comments" { try testing.expect(isCommentOrBlank(" // zig comment", .zig)); try testing.expect(isCommentOrBlank(" # python comment", .python)); @@ -603,7 +626,6 @@ test "isCommentOrBlank: detects language-specific comments" { try testing.expect(!isCommentOrBlank("// comment", .unknown)); } - test "explorer: getSymbolBody returns source lines" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -620,7 +642,6 @@ test "explorer: getSymbolBody returns source lines" { } } - test "explorer: getSymbolBody returns null for unknown file" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -630,7 +651,6 @@ test "explorer: getSymbolBody returns null for unknown file" { try testing.expect(body == null); } - test "explorer: searchContentWithScope annotates results" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -657,7 +677,6 @@ test "explorer: searchContentWithScope annotates results" { try testing.expectEqualStrings("handleAuth", results[0].scope_name.?); } - test "explorer: searchContentWithScope no scope for standalone line" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -680,7 +699,6 @@ test "explorer: searchContentWithScope no scope for standalone line" { try testing.expect(results[0].scope_name == null); } - test "content hash: Wyhash produces consistent hash" { const content = "pub fn main() void {}"; const hash1 = std.hash.Wyhash.hash(0, content); @@ -691,7 +709,6 @@ test "content hash: Wyhash produces consistent hash" { try testing.expect(hash1 != hash3); } - test "detectLanguage: public access and correct detection" { try testing.expect(explore.detectLanguage("src/main.zig") == .zig); try testing.expect(explore.detectLanguage("app.py") == .python); @@ -699,7 +716,6 @@ test "detectLanguage: public access and correct detection" { try testing.expect(explore.detectLanguage("style.css") == .css); } - test "extractLines: without line numbers" { const content = "alpha\nbeta\ngamma"; const result = try extractLines(content, 1, 3, false, false, .unknown, testing.allocator); @@ -707,7 +723,6 @@ test "extractLines: without line numbers" { try testing.expectEqualStrings("alpha\nbeta\ngamma\n", result); } - test "extractLines: start only reads to EOF" { const content = "a\nb\nc\nd\ne"; const result = try extractLines(content, 3, std.math.maxInt(u32), true, false, .unknown, testing.allocator); @@ -719,7 +734,6 @@ test "extractLines: start only reads to EOF" { try testing.expect(std.mem.indexOf(u8, result, "| b") == null); } - test "extractLines: end beyond file clamps to EOF" { const content = "x\ny\nz"; const result = try extractLines(content, 2, 999, true, false, .unknown, testing.allocator); @@ -730,7 +744,6 @@ test "extractLines: end beyond file clamps to EOF" { try testing.expect(std.mem.count(u8, result, "\n") == 2); } - test "extractLines: single line range (start == end)" { const content = "one\ntwo\nthree"; const result = try extractLines(content, 2, 2, true, false, .unknown, testing.allocator); @@ -739,7 +752,6 @@ test "extractLines: single line range (start == end)" { try testing.expect(std.mem.count(u8, result, "\n") == 1); } - test "extractLines: empty content returns single empty line" { const result = try extractLines("", 1, 10, true, false, .unknown, testing.allocator); defer testing.allocator.free(result); @@ -747,7 +759,6 @@ test "extractLines: empty content returns single empty line" { try testing.expect(result.len > 0); } - test "extractLines: compact with Python comments" { const content = "# comment\nimport os\n\ndef hello():\n # inline comment\n print('hi')"; const result = try extractLines(content, 1, 6, false, true, .python, testing.allocator); @@ -759,7 +770,6 @@ test "extractLines: compact with Python comments" { try testing.expect(std.mem.indexOf(u8, result, "print('hi')") != null); } - test "extractLines: compact with JS/TS comments" { const content = "// header\nconst x = 1;\n/* block */\n* star line\nexport default x;"; const result = try extractLines(content, 1, 5, false, true, .typescript, testing.allocator); @@ -771,26 +781,22 @@ test "extractLines: compact with JS/TS comments" { try testing.expect(std.mem.indexOf(u8, result, "export default x;") != null); } - test "isCommentOrBlank: rust double-slash" { try testing.expect(isCommentOrBlank(" // rust comment", .rust)); try testing.expect(!isCommentOrBlank(" let x = 1;", .rust)); } - test "isCommentOrBlank: go double-slash" { try testing.expect(isCommentOrBlank(" // go comment", .go_lang)); try testing.expect(!isCommentOrBlank(" func main() {", .go_lang)); } - test "isCommentOrBlank: dart comments" { try testing.expect(isCommentOrBlank(" // dart comment", .dart)); try testing.expect(isCommentOrBlank(" /* dart block comment */", .dart)); try testing.expect(!isCommentOrBlank(" class WidgetBuilder {}", .dart)); } - test "isCommentOrBlank: cpp block and line comments" { try testing.expect(isCommentOrBlank(" // cpp line comment", .cpp)); try testing.expect(isCommentOrBlank(" /* cpp block comment */", .cpp)); @@ -798,7 +804,6 @@ test "isCommentOrBlank: cpp block and line comments" { try testing.expect(!isCommentOrBlank(" int x = 0;", .cpp)); } - test "isCommentOrBlank: detected extension language comments" { try testing.expect(isCommentOrBlank(" // java line comment", .java)); try testing.expect(isCommentOrBlank(" // kotlin line comment", .kotlin)); @@ -817,21 +822,18 @@ test "isCommentOrBlank: detected extension language comments" { try testing.expect(!isCommentOrBlank(" SELECT * FROM users;", .sql)); } - test "isCommentOrBlank: tabs and mixed whitespace" { try testing.expect(isCommentOrBlank("\t\t// tabbed comment", .zig)); try testing.expect(isCommentOrBlank(" \t \t ", .zig)); try testing.expect(isCommentOrBlank("\t", .python)); } - test "isCommentOrBlank: markdown and json never strip" { try testing.expect(!isCommentOrBlank("# heading", .markdown)); try testing.expect(!isCommentOrBlank("// not a comment in json", .json)); try testing.expect(!isCommentOrBlank("# not a comment in yaml", .yaml)); } - test "explorer: getSymbolBody multi-line range" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -853,7 +855,6 @@ test "explorer: getSymbolBody multi-line range" { } } - test "explorer: getSymbolBody range beyond file length" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -870,7 +871,6 @@ test "explorer: getSymbolBody range beyond file length" { } } - test "explorer: searchContentWithScope across multiple files" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -896,7 +896,6 @@ test "explorer: searchContentWithScope across multiple files" { } } - test "explorer: searchContentWithScope respects max_results" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -917,7 +916,6 @@ test "explorer: searchContentWithScope respects max_results" { try testing.expect(results.len == 2); } - test "explorer: searchContentWithScope no results for missing query" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -938,7 +936,6 @@ test "explorer: searchContentWithScope no results for missing query" { try testing.expect(results.len == 0); } - test "content hash: format as hex string" { const content = "hello world"; const hash = std.hash.Wyhash.hash(0, content); @@ -954,14 +951,12 @@ test "content hash: format as hex string" { try testing.expectEqualStrings(hex, hex2); } - test "content hash: empty content hashes consistently" { const h1 = std.hash.Wyhash.hash(0, ""); const h2 = std.hash.Wyhash.hash(0, ""); try testing.expect(h1 == h2); } - test "detectLanguage: all supported extensions" { try testing.expect(explore.detectLanguage("main.zig") == .zig); try testing.expect(explore.detectLanguage("lib.c") == .c); @@ -1003,7 +998,6 @@ test "detectLanguage: all supported extensions" { try testing.expect(explore.detectLanguage("no_ext") == .unknown); } - test "explorer: getSymbolBody with line number format" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1024,7 +1018,6 @@ test "explorer: getSymbolBody with line number format" { } } - test "extractLines: compact preserves brace-only lines" { const content = "fn main() void {\n // comment\n doWork();\n}"; const result = try extractLines(content, 1, 4, false, true, .zig, testing.allocator); @@ -1035,7 +1028,6 @@ test "extractLines: compact preserves brace-only lines" { try testing.expect(std.mem.indexOf(u8, result, "// comment") == null); } - test "extractLines: compact on all-comment file returns empty" { const content = "// comment 1\n// comment 2\n// comment 3"; const result = try extractLines(content, 1, 3, false, true, .zig, testing.allocator); @@ -1043,7 +1035,6 @@ test "extractLines: compact on all-comment file returns empty" { try testing.expect(result.len == 0); } - test "explorer: searchContentRegex end-to-end" { var explorer_inst = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_inst.deinit(); @@ -1073,7 +1064,6 @@ test "explorer: searchContentRegex end-to-end" { try testing.expect(found2); } - test "explorer: searchContentRegex no match" { var explorer_inst = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_inst.deinit(); @@ -1086,7 +1076,6 @@ test "explorer: searchContentRegex no match" { try testing.expectEqual(@as(usize, 0), results.len); } - test "git: getGitHead returns 40-char hex SHA in a git repo" { if (builtin.os.tag == .windows) return error.SkipZigTest; @@ -1100,14 +1089,12 @@ test "git: getGitHead returns 40-char hex SHA in a git repo" { } } - test "git: getGitHead returns null for non-git directory" { // /tmp is not a git repo const head = try git_mod.getGitHead("/tmp", testing.allocator); try testing.expect(head == null); } - test "thread-safe: concurrent TrigramIndex.candidates() with per-thread allocators" { var ti = TrigramIndex.init(testing.allocator); defer ti.deinit(); @@ -1139,7 +1126,6 @@ test "thread-safe: concurrent TrigramIndex.candidates() with per-thread allocato try testing.expectEqual(@as(u32, 0), ctx.errors.load(.monotonic)); } - test "thread-safe: concurrent SparseNgramIndex.candidates() with per-thread allocators" { var sni = SparseNgramIndex.init(testing.allocator); defer sni.deinit(); @@ -1169,7 +1155,6 @@ test "thread-safe: concurrent SparseNgramIndex.candidates() with per-thread allo for (threads) |t| t.join(); } - test "issue-43: trigram_index swap in scanBg races with concurrent MCP queries" { // Regression: the scanBg disk-load path must serialize trigram_index swaps // with readers by taking exp.mu.lock() before replacing the index. @@ -1200,7 +1185,6 @@ test "issue-43: trigram_index swap in scanBg races with concurrent MCP queries" try testing.expect(!raced); } - test "issue-116: getGitHead returns valid SHA for git repos" { const git = @import("git.zig"); @@ -1215,7 +1199,6 @@ test "issue-116: getGitHead returns valid SHA for git repos" { } } - test "issue-224: codedb_symbol body=true returns full body — line_end populated" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1245,7 +1228,6 @@ test "issue-224: codedb_symbol body=true returns full body — line_end populate try testing.expect(std.mem.indexOf(u8, body, "return a + b;") != null); } - test "issue-224: Python def line_end covers full body" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1268,14 +1250,12 @@ test "issue-224: Python def line_end covers full body" { try testing.expectEqual(@as(u32, 3), sym.line_end); } - test "issue-108: detectLanguage handles .tf and .tfvars" { try testing.expectEqual(Language.hcl, explore.detectLanguage("main.tf")); try testing.expectEqual(Language.hcl, explore.detectLanguage("prod.tfvars")); try testing.expectEqual(Language.hcl, explore.detectLanguage("config.hcl")); } - test "issue-215: detectLanguage handles .r and .R" { try testing.expectEqual(Language.r, explore.detectLanguage("script.r")); try testing.expectEqual(Language.r, explore.detectLanguage("analysis.R")); @@ -1286,7 +1266,6 @@ test "issue-532: detectLanguage handles .res and .resi" { try testing.expectEqualStrings("rescript", @tagName(explore.detectLanguage("src/User.resi"))); } - test "dep-graph: reverse index gives O(1) imported_by lookup" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1320,7 +1299,6 @@ test "dep-graph: reverse index gives O(1) imported_by lookup" { try testing.expectEqualStrings("main.zig", imported_by2[0]); } - test "dep-graph: setDeps removes old reverse edges" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1359,7 +1337,6 @@ test "dep-graph: setDeps removes old reverse edges" { try testing.expectEqual(@as(usize, 1), utils_deps.len); } - test "dep-graph: transitive dependents via BFS" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1395,7 +1372,6 @@ test "dep-graph: transitive dependents via BFS" { try testing.expectEqualStrings("store.zig", shallow[0]); } - test "dep-graph: transitive dependencies (forward BFS)" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1430,7 +1406,6 @@ test "dep-graph: transitive dependencies (forward BFS)" { try testing.expectEqual(@as(usize, 2), deps_shallow.len); } - test "dep-graph: remove cleans forward and reverse edges" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1459,7 +1434,6 @@ test "dep-graph: remove cleans forward and reverse edges" { try testing.expectEqualStrings("server.zig", imported_by[0]); } - test "dep-graph: cycle does not cause infinite BFS" { var graph = DependencyGraph.init(testing.allocator); defer graph.deinit(); @@ -1495,7 +1469,6 @@ test "dep-graph: cycle does not cause infinite BFS" { try testing.expectEqual(@as(usize, 2), fwd.len); } - test "dep-graph: Explorer integration — getImportedBy uses reverse index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1513,7 +1486,6 @@ test "dep-graph: Explorer integration — getImportedBy uses reverse index" { try testing.expectEqual(@as(usize, 2), deps.len); } - test "dep-graph: Explorer transitive dependents" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1532,7 +1504,6 @@ test "dep-graph: Explorer transitive dependents" { try testing.expectEqual(@as(usize, 2), blast.len); } - test "issue-445: dep-graph dedupes multi-aliased forward imports" { // A file that imports the same dep under multiple aliases // const idx = @import("index.zig"); @@ -1564,7 +1535,6 @@ test "issue-445: dep-graph dedupes multi-aliased forward imports" { try testing.expectEqualStrings("index.zig", fwd[0]); } - test "symbol-index: O(1) findSymbol via symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1597,7 +1567,6 @@ test "symbol-index: O(1) findSymbol via symbol_index" { try testing.expectEqual(@as(usize, 2), all.len); } - test "symbol-index: removeFile cleans symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1616,7 +1585,6 @@ test "symbol-index: removeFile cleans symbol_index" { try testing.expect(after == null); } - test "symbol-index: re-index updates symbol_index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1641,7 +1609,6 @@ test "symbol-index: re-index updates symbol_index" { if (r3.?.symbol.detail) |d| testing.allocator.free(d); } - test "word-index: splitIdentifier snake_case" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1657,7 +1624,6 @@ test "word-index: splitIdentifier snake_case" { try testing.expectEqualStrings("put", out.items[2]); } - test "word-index: splitIdentifier camelCase" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1672,7 +1638,6 @@ test "word-index: splitIdentifier camelCase" { try testing.expectEqualStrings("token", out.items[1]); } - test "word-index: splitIdentifier acronym (HTTPHandler)" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1687,7 +1652,6 @@ test "word-index: splitIdentifier acronym (HTTPHandler)" { try testing.expectEqualStrings("handler", out.items[1]); } - test "word-index: splitIdentifier simple word emits itself" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1701,7 +1665,6 @@ test "word-index: splitIdentifier simple word emits itself" { try testing.expectEqualStrings("handler", out.items[0]); } - test "word-index: sub-token search finds camelCase components" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1735,7 +1698,6 @@ test "word-index: sub-token search finds camelCase components" { try testing.expectEqualStrings("b.zig", r2[0].path); } - test "word-index: sub-token search finds snake_case components" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1766,7 +1728,6 @@ test "word-index: sub-token search finds snake_case components" { try testing.expect(r2.len >= 1); } - test "word-index: case-insensitive lookup finds exact identifiers" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1786,7 +1747,6 @@ test "word-index: case-insensitive lookup finds exact identifiers" { try testing.expectEqual(@as(usize, 1), r1.len); } - test "word-index: searchPrefix finds extensions of a prefix" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1801,7 +1761,6 @@ test "word-index: searchPrefix finds extensions of a prefix" { try testing.expect(hits.len >= 1); } - test "word-index: searchPrefix skips exact match (Tier 0 responsibility)" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1822,7 +1781,6 @@ test "word-index: searchPrefix skips exact match (Tier 0 responsibility)" { try testing.expect(hits_prefix.len >= 1); } - test "word-index: searchPrefix respects max_results cap" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1843,7 +1801,6 @@ test "word-index: searchPrefix respects max_results cap" { try testing.expect(hits.len > 0); } - test "integration: Tier 0.5 prefix expansion finds partial identifier" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1864,7 +1821,6 @@ test "integration: Tier 0.5 prefix expansion finds partial identifier" { try testing.expect(results.len >= 1); } - test "issue-389: FilteredWalker yields symlinked source files" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -1900,7 +1856,6 @@ test "issue-389: FilteredWalker yields symlinked source files" { try testing.expect(explorer.contents.contains("src/alias.zig")); } - test "issue-405: FilteredWalker walks directory symlinks safely (cycle + escape)" { if (builtin.os.tag == .windows) return error.SkipZigTest; @@ -1967,7 +1922,6 @@ test "issue-405: FilteredWalker walks directory symlinks safely (cycle + escape) } } - test "issue-405: cleanupStaleTmpFiles deletes in-flight sibling tmp files" { // BUG: snapshot.zig:cleanupStaleTmpFiles deletes ANY file matching // `*.tmp` in the snapshot directory with no age guard. @@ -2025,7 +1979,6 @@ test "issue-405: cleanupStaleTmpFiles deletes in-flight sibling tmp files" { }; } - test "issue-409: snapshot .env prefix filter wrongly excludes .envoy/.environment files" { // BUG: snapshot.zig:isSensitivePath uses // if (basename.len >= 4 and std.mem.eql(u8, basename[0..4], ".env")) return true; @@ -2075,7 +2028,6 @@ test "issue-409: snapshot .env prefix filter wrongly excludes .envoy/.environmen try testing.expect(exp2.outlines.contains(".envoy.json")); } - test "issue-208: content cache evicts cold entries under pressure" { const ContentCache = @import("hot_cache.zig").ContentCache; const cap = 50; @@ -2119,8 +2071,6 @@ test "issue-208: content cache evicts cold entries under pressure" { try testing.expect(s.evictions > 0); } - - test "explorer: renderSkeleton elides bodies, keeps signatures" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -2154,7 +2104,6 @@ test "explorer: renderSkeleton elides bodies, keeps signatures" { try testing.expect(std.mem.indexOf(u8, s, "codedb_outline sk.zig") != null); } - test "explorer: multi-line signature gets correct line_end (findBraceEnd paren-awareness)" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); diff --git a/src/test_index.zig b/src/test_index.zig index c21482b2..6537ac3f 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -391,7 +391,7 @@ test "frequency table: little-endian byte order on disk" { const dir_path_len = try tmp_dir.dir.realPathFile(io, ".", &dir_buf); const dir_path = dir_buf[0..dir_path_len]; - var table: [256][256]u16 = .{.{0} ** 256} ** 256; + var table: [256][256]u16 = @splat(@splat(0)); table[0][0] = 0x1234; // little-endian on disk: 0x34, 0x12 table[0][1] = 0xABCD; // little-endian on disk: 0xCD, 0xAB try writeFrequencyTable(io, &table, dir_path); @@ -416,7 +416,7 @@ test "frequency table: little-endian byte order on disk" { test "setFrequencyTable / resetFrequencyTable: pairWeight output changes" { // Build a table where 'th' is rare (high weight) — opposite of default. - var custom: [256][256]u16 = .{.{0x1000} ** 256} ** 256; // all common + var custom: [256][256]u16 = @splat(@splat(0x1000)); // all common custom['q']['x'] = 0xFE00; // make 'qx' rare const before_th = pairWeight('t', 'h'); @@ -541,6 +541,47 @@ test "watcher: queue event copies path bytes" { try testing.expect(event.seq == 99); } +test "word index: posting construction suppresses duplicate line hits" { + var wi = WordIndex.init(testing.allocator); + defer wi.deinit(); + + // Repeated source tokens and identifier splitting can present the same + // normalized term more than once, but each (document, line) is one hit. + try wi.indexFile("a.zig", "alpha alpha alpha_value alpha\nalpha\n"); + try wi.indexFile("b.zig", "alpha alpha\n"); + + const raw = wi.search("alpha"); + try testing.expectEqual(@as(usize, 3), raw.len); + try testing.expectEqual(@as(u32, 1), raw[0].line_num); + try testing.expectEqual(@as(u32, 2), raw[1].line_num); + try testing.expectEqual(@as(u32, 1), raw[2].line_num); + + const owned = try wi.searchDeduped("ALPHA", testing.allocator); + defer testing.allocator.free(owned); + try testing.expectEqual(raw.len, owned.len); + for (raw, owned) |expected, actual| { + try testing.expectEqual(expected.doc_id, actual.doc_id); + try testing.expectEqual(expected.line_num, actual.line_num); + } +} + +test "word index: searchDeduped handles non-adjacent malformed postings" { + var wi = WordIndex.init(testing.allocator); + defer wi.deinit(); + + try wi.indexFile("a.zig", "alpha\n"); + try wi.indexFile("b.zig", "alpha\n"); + const alpha = wi.index.getPtr("alpha") orelse return error.TestUnexpectedResult; + // Simulate a legacy/corrupt unsorted list with a repeated first posting. + try alpha.append(testing.allocator, alpha.items[0]); + + const hits = try wi.searchDeduped("alpha", testing.allocator); + defer testing.allocator.free(hits); + try testing.expectEqual(@as(usize, 2), hits.len); + try testing.expectEqualStrings("a.zig", wi.hitPath(hits[0])); + try testing.expectEqualStrings("b.zig", wi.hitPath(hits[1])); +} + test "watcher: parallel initial scan matches sequential results" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -583,6 +624,51 @@ test "watcher: parallel initial scan matches sequential results" { try testing.expectEqual(explorer_seq.outlines.count(), explorer_par.outlines.count()); } +test "watcher: rolling trigram shards match canonical masks exactly" { + var source = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer source.deinit(); + var canonical = TrigramIndex.init(testing.allocator); + defer canonical.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "edge/empty.txt", .content = "" }, + .{ .path = "edge/one.txt", .content = "A" }, + .{ .path = "edge/two.txt", .content = "Ab" }, + .{ .path = "edge/three.txt", .content = "AbC" }, + .{ .path = "edge/whitespace.txt", .content = " \t\nAbCdEf" }, + .{ .path = "edge/rollover.txt", .content = "aaaaaaaaaaaaZ" }, + .{ .path = "edge/mixed.txt", .content = "MiXeD-case final" }, + }; + for (fixtures) |fixture| { + try source.indexFile(fixture.path, fixture.content); + try canonical.indexFile(fixture.path, fixture.content); + } + + const sharded = try watcher.buildTrigramsFromCache( + &source.contents, + testing.allocator, + testing.allocator, + 4, + ); + defer { + sharded.deinit(); + testing.allocator.destroy(sharded); + } + + try testing.expectEqual(canonical.index.count(), sharded.index.count()); + var tri_iter = canonical.index.iterator(); + while (tri_iter.next()) |entry| { + const actual_postings = sharded.index.get(entry.key_ptr.*) orelse return error.TestUnexpectedResult; + try testing.expectEqual(entry.value_ptr.count(), actual_postings.count()); + for (entry.value_ptr.items.items) |expected_posting| { + const path = canonical.id_to_path.items[expected_posting.doc_id]; + const actual_mask = actual_postings.get(path) orelse return error.TestUnexpectedResult; + try testing.expectEqual(expected_posting.loc_mask, actual_mask.loc_mask); + try testing.expectEqual(expected_posting.next_mask, actual_mask.next_mask); + } + } +} + test "watcher: parallel word-index shards match sequential (skip_file_words)" { // Exercises the per-worker WordIndex shard + serial mergeShard path // (use_shards requires word_index.enabled and skip_file_words). Asserts the @@ -2030,6 +2116,145 @@ test "disk index: streamed postings cross a batch boundary" { try testing.expectEqual(@as(usize, 4097), candidates.?.len); } +test "disk index: streamed lookup crosses a batch boundary" { + const alloc = testing.allocator; + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"; + var ti = TrigramIndex.init(alloc); + ti.owns_paths = true; + defer ti.deinit(); + + // 4,097 distinct trigrams forces one full lookup batch and a one-entry + // tail. Check entries before, at, and after the boundary with both readers. + for (0..4097) |i| { + const content = [3]u8{ + alphabet[i % alphabet.len], + alphabet[(i / alphabet.len) % alphabet.len], + alphabet[(i / (alphabet.len * alphabet.len)) % alphabet.len], + }; + const path = try std.fmt.allocPrint(alloc, "src/lookup-{d}.zig", .{i}); + errdefer alloc.free(path); + try ti.indexFile(path, &content); + alloc.free(path); + } + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + const loaded = TrigramIndex.readFromDisk(io, dir_path, alloc); + try testing.expect(loaded != null); + var heap_index = loaded.?; + defer heap_index.deinit(); + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + + for ([_]usize{ 0, 4095, 4096 }) |i| { + const query = [3]u8{ + alphabet[i % alphabet.len], + alphabet[(i / alphabet.len) % alphabet.len], + alphabet[(i / (alphabet.len * alphabet.len)) % alphabet.len], + }; + const expected_path = try std.fmt.allocPrint(alloc, "src/lookup-{d}.zig", .{i}); + defer alloc.free(expected_path); + + const heap_candidates = heap_index.candidates(&query, alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 1), heap_candidates.?.len); + try testing.expectEqualStrings(expected_path, heap_candidates.?[0]); + + const mmap_candidates = mmap_index.candidates(&query, alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 1), mmap_candidates.?.len); + try testing.expectEqualStrings(expected_path, mmap_candidates.?[0]); + } +} + +test "disk index: mmap candidate parity survives persisted doc ID remapping" { + const alloc = testing.allocator; + var ti = TrigramIndex.init(alloc); + defer ti.deinit(); + + // Each short file contributes one distinct query trigram; only the final + // file contains the whole query. This catches disk file IDs that do not + // preserve the sorted posting-list order required by mmap intersections. + try ti.indexFile("src/abc.zig", "abc"); + try ti.indexFile("src/bcd.zig", "bcd"); + try ti.indexFile("src/cde.zig", "cde"); + try ti.indexFile("src/def.zig", "def"); + try ti.indexFile("src/efg.zig", "efg"); + try ti.indexFile("src/fgh.zig", "fgh"); + try ti.indexFile("src/ghi.zig", "ghi"); + try ti.indexFile("src/full.zig", "abcdefghi"); + + const heap_candidates = ti.candidates("abcdefghi", alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 1), heap_candidates.?.len); + try testing.expectEqualStrings("src/full.zig", heap_candidates.?[0]); + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + const mmap_candidates = mmap_index.candidates("abcdefghi", alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 1), mmap_candidates.?.len); + try testing.expectEqualStrings(heap_candidates.?[0], mmap_candidates.?[0]); +} + +test "trigram index: insertExtracted preserves mmap posting order after doc ID reuse" { + const alloc = testing.allocator; + var ti = TrigramIndex.init(alloc); + defer ti.deinit(); + + var extracted = TrigramIndex.extractTrigrams("abcdefghi", alloc); + defer extracted.deinit(); + try ti.insertExtracted("src/a.zig", &extracted); + try ti.insertExtracted("src/b.zig", &extracted); + try ti.insertExtracted("src/c.zig", &extracted); + ti.removeFile("src/a.zig"); + try ti.insertExtracted("src/a.zig", &extracted); + + const heap_candidates = ti.candidates("abcdefghi", alloc); + defer if (heap_candidates) |items| alloc.free(items); + try testing.expect(heap_candidates != null); + try testing.expectEqual(@as(usize, 3), heap_candidates.?.len); + try testing.expectEqualStrings("src/a.zig", heap_candidates.?[0]); + try testing.expectEqualStrings("src/b.zig", heap_candidates.?[1]); + try testing.expectEqualStrings("src/c.zig", heap_candidates.?[2]); + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_path_len = try tmp.dir.realPathFile(io, ".", &path_buf); + const dir_path = path_buf[0..dir_path_len]; + try ti.writeToDisk(io, dir_path, null); + + var mmap_index = MmapTrigramIndex.initFromDisk(io, dir_path, alloc) orelse + return error.MmapInitFailed; + defer mmap_index.deinit(); + const mmap_candidates = mmap_index.candidates("abcdefghi", alloc); + defer if (mmap_candidates) |items| alloc.free(items); + try testing.expect(mmap_candidates != null); + try testing.expectEqual(@as(usize, 3), mmap_candidates.?.len); + try testing.expectEqualStrings("src/a.zig", mmap_candidates.?[0]); + try testing.expectEqualStrings("src/b.zig", mmap_candidates.?[1]); + try testing.expectEqualStrings("src/c.zig", mmap_candidates.?[2]); +} + test "disk index: readFromDisk returns null for missing files" { const loaded = TrigramIndex.readFromDisk(io, "/tmp/codedb_nonexistent_dir_12345", testing.allocator); try testing.expect(loaded == null); diff --git a/src/test_mcp.zig b/src/test_mcp.zig index e0aea984..f84ca1df 100644 --- a/src/test_mcp.zig +++ b/src/test_mcp.zig @@ -29,10 +29,62 @@ const cli_proxy_mod = @import("cli_proxy.zig"); const bootstrap_mod = @import("bootstrap.zig"); const background_mod = @import("background.zig"); const commands_mod = @import("commands.zig"); +const mcp_json = @import("mcp_json.zig"); comptime { _ = @import("config.zig"); } +test "mcp json: line reader preserves newline and EOF framing" { + var reader: std.Io.Reader = .fixed("first\nlast"); + + const first = mcp_json.readLineBuf(testing.allocator, &reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(first); + try testing.expectEqualStrings("first", first); + + const last = mcp_json.readLineBuf(testing.allocator, &reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(last); + try testing.expectEqualStrings("last", last); + try testing.expect(mcp_json.readLineBuf(testing.allocator, &reader) == null); +} + +test "mcp json: line reader enforces maximum message size" { + const exact = try testing.allocator.alloc(u8, mcp_json.MAX_LINE); + defer testing.allocator.free(exact); + @memset(exact, 'x'); + var exact_reader: std.Io.Reader = .fixed(exact); + const accepted = mcp_json.readLineBuf(testing.allocator, &exact_reader) orelse return error.TestUnexpectedResult; + defer testing.allocator.free(accepted); + try testing.expectEqual(exact.len, accepted.len); + + const oversized = try testing.allocator.alloc(u8, mcp_json.MAX_LINE + 1); + defer testing.allocator.free(oversized); + @memset(oversized, 'y'); + var oversized_reader: std.Io.Reader = .fixed(oversized); + try testing.expect(mcp_json.readLineBuf(testing.allocator, &oversized_reader) == null); +} + +test "mcp json: typed fields and escaping preserve protocol behavior" { + const parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, + \\{"name":"codedb","count":7,"enabled":true,"other":null} + , .{}); + defer parsed.deinit(); + const object = &parsed.value.object; + try testing.expectEqualStrings("codedb", mcp_json.getStr(object, "name").?); + try testing.expectEqual(@as(i64, 7), mcp_json.getInt(object, "count").?); + try testing.expect(mcp_json.getBool(object, "enabled")); + try testing.expect(mcp_json.getStr(object, "count") == null); + try testing.expect(mcp_json.getInt(object, "name") == null); + try testing.expect(!mcp_json.getBool(object, "missing")); + + var escaped: std.ArrayList(u8) = .empty; + defer escaped.deinit(testing.allocator); + mcp_json.writeEscaped(testing.allocator, &escaped, "plain \"quote\" \\ slash\n\r\t\x00\x08\x0c\x1f"); + try testing.expectEqualStrings( + "plain \\\"quote\\\" \\\\ slash\\n\\r\\t\\u0000\\u0008\\u000c\\u001f", + escaped.items, + ); +} + fn zigExe() []const u8 { return "zig"; } @@ -3002,7 +3054,7 @@ test "issue-619: serve/mcp daemon reclaims the cli socket after the owner exits" var pbuf: [128]u8 = undefined; const sock_path = cli_proxy_mod.cliSocketPath(&pbuf, abs_root).?; var zbuf: [128]u8 = undefined; - const sock_z = try std.fmt.bufPrintZ(&zbuf, "{s}", .{sock_path}); + const sock_z = try cio.bufPrintZ(&zbuf, "{s}", .{sock_path}); _ = std.c.unlink(sock_z.ptr); defer _ = std.c.unlink(sock_z.ptr); @@ -3097,7 +3149,7 @@ const CliHandoverOwnerSim = struct { // socket up entirely rather than waiting for an idle timeout. _ = std.c.close(self.fd); var zbuf: [128]u8 = undefined; - if (std.fmt.bufPrintZ(&zbuf, "{s}", .{self.sock_path})) |z| { + if (cio.bufPrintZ(&zbuf, "{s}", .{self.sock_path})) |z| { _ = std.c.unlink(z.ptr); } else |_| {} self.sd.store(true, .release); @@ -3112,7 +3164,7 @@ test "issue-619: serve/mcp daemon signals a live owner to yield instead of waiti var pbuf: [128]u8 = undefined; const sock_path = cli_proxy_mod.cliSocketPath(&pbuf, abs_root).?; var zbuf: [128]u8 = undefined; - const sock_z = try std.fmt.bufPrintZ(&zbuf, "{s}", .{sock_path}); + const sock_z = try cio.bufPrintZ(&zbuf, "{s}", .{sock_path}); _ = std.c.unlink(sock_z.ptr); defer _ = std.c.unlink(sock_z.ptr); diff --git a/src/test_parser.zig b/src/test_parser.zig index 5cf5c740..d24c78ba 100644 --- a/src/test_parser.zig +++ b/src/test_parser.zig @@ -9,7 +9,6 @@ const SymbolKind = explore.SymbolKind; const DependencyGraph = explore.DependencyGraph; const Store = @import("store.zig").Store; - fn expectOutlineSymbol(outline: *const explore.FileOutline, name: []const u8, kind: SymbolKind) !void { for (outline.symbols.items) |sym| { if (std.mem.eql(u8, sym.name, name) and sym.kind == kind) return; @@ -17,7 +16,6 @@ fn expectOutlineSymbol(outline: *const explore.FileOutline, name: []const u8, ki return error.TestUnexpectedResult; } - fn expectOutlineImport(outline: *const explore.FileOutline, import_path: []const u8) !void { for (outline.imports.items) |imp| { if (std.mem.eql(u8, imp, import_path)) return; @@ -25,7 +23,6 @@ fn expectOutlineImport(outline: *const explore.FileOutline, import_path: []const return error.TestUnexpectedResult; } - test "issue-1: multi-line TS/JS import paths captured for dep graph" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -240,7 +237,6 @@ test "issue-301: Dart / Flutter parser" { try testing.expect(std.mem.indexOf(u8, tree, "home_screen.dart dart") != null); } - test "issue-php-1: PHP class definition herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -268,7 +264,6 @@ test "issue-php-1: PHP class definition herkend" { try testing.expect(found); } - test "issue-php-2: PHP methode binnen class herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -298,7 +293,6 @@ test "issue-php-2: PHP methode binnen class herkend" { try testing.expectEqual(@as(usize, 2), method_count); } - test "issue-php-3: PHP top-level functie herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -325,7 +319,6 @@ test "issue-php-3: PHP top-level functie herkend" { try testing.expectEqual(@as(usize, 2), fn_count); } - test "issue-php-4: PHP interface herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -352,7 +345,6 @@ test "issue-php-4: PHP interface herkend" { try testing.expect(found); } - test "issue-php-5: PHP trait herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -381,7 +373,6 @@ test "issue-php-5: PHP trait herkend" { try testing.expect(found); } - test "issue-php-6: PHP use-import omgezet naar pad in dep_graph" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -401,7 +392,6 @@ test "issue-php-6: PHP use-import omgezet naar pad in dep_graph" { try testing.expectEqualStrings("illuminate/Support/Facades/DB.php", outline.imports.items[1]); } - test "issue-php-7: PHP commentaarregels worden overgeslagen" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -425,7 +415,6 @@ test "issue-php-7: PHP commentaarregels worden overgeslagen" { try testing.expect(outline.symbols.items[0].kind == .class_def); } - test "issue-php-8: PHP function after class is top-level, not method" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -458,7 +447,6 @@ test "issue-php-8: PHP function after class is top-level, not method" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-9: PHP 8.1 enum herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -487,7 +475,6 @@ test "issue-php-9: PHP 8.1 enum herkend" { try testing.expect(found_method); } - test "issue-php-10: PHP grouped use-statement parsed into individual imports" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -509,7 +496,6 @@ test "issue-php-10: PHP grouped use-statement parsed into individual imports" { try testing.expectEqualStrings("app/Models/Role.php", outline.imports.items[2]); } - test "issue-php-11: PHP readonly class herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -535,7 +521,6 @@ test "issue-php-11: PHP readonly class herkend" { try testing.expect(found); } - test "issue-php-12: PHP class and public constants herkend" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -561,7 +546,6 @@ test "issue-php-12: PHP class and public constants herkend" { try testing.expectEqual(@as(usize, 3), constant_count); } - test "issue-php-13: PHP nested braces in methods do not break class tracking" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -603,7 +587,6 @@ test "issue-php-13: PHP nested braces in methods do not break class tracking" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-14: PHP multi-line block comments do not produce symbols" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -640,7 +623,6 @@ test "issue-php-14: PHP multi-line block comments do not produce symbols" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-15: PHP use-as alias stripped from import path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -658,7 +640,6 @@ test "issue-php-15: PHP use-as alias stripped from import path" { try testing.expectEqualStrings("app/Models/User.php", outline.imports.items[0]); } - test "issue-php-16: PHP escaped quotes do not end string mode" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -696,7 +677,6 @@ test "issue-php-16: PHP escaped quotes do not end string mode" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-17: PHP code after block comment terminator is parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -722,7 +702,6 @@ test "issue-php-17: PHP code after block comment terminator is parsed" { try testing.expectEqual(@as(usize, 1), function_count); } - test "issue-php-18: PHP use-as alias case-insensitive" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -743,7 +722,6 @@ test "issue-php-18: PHP use-as alias case-insensitive" { try testing.expectEqualStrings("app/Services/Logger.php", outline.imports.items[2]); } - test "issue-111: Python triple-quote docstrings not parsed as code" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -768,7 +746,6 @@ test "issue-111: Python triple-quote docstrings not parsed as code" { try testing.expect(func_count == 1); } - test "issue-112: Python import-as alias stripped from dep path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -785,7 +762,6 @@ test "issue-112: Python import-as alias stripped from dep path" { try testing.expect(deps.len == 1); } - test "issue-113: TypeScript block comments not parsed as code" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -807,7 +783,6 @@ test "issue-113: TypeScript block comments not parsed as code" { try testing.expect(func_count == 1); } - test "issue-114: TypeScript import-as alias does not affect dep path" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -823,7 +798,6 @@ test "issue-114: TypeScript import-as alias does not affect dep path" { try testing.expectEqualStrings("./mod", outline.imports.items[0]); } - test "issue-151: Go func and type definitions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -867,7 +841,6 @@ test "issue-151: Go func and type definitions" { try testing.expect(interface_count == 1); // Handler } - test "issue-151: Ruby class, module, and def" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -903,7 +876,6 @@ test "issue-151: Ruby class, module, and def" { try testing.expect(outline.imports.items.len == 2); // json + ./helpers } - test "issue-151: Ruby =begin/=end comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -929,7 +901,6 @@ test "issue-151: Ruby =begin/=end comments skipped" { try testing.expect(func_count == 1); // only real_method } - test "issue-151: Go block comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -953,7 +924,6 @@ test "issue-151: Go block comments skipped" { try testing.expect(func_count == 1); // only realFunc } - test "issue-301: Dart block comments skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -980,7 +950,6 @@ test "issue-301: Dart block comments skipped" { try testing.expectEqual(@as(usize, 0), func_count); } - test "issue-179: block comment does not produce phantom symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1002,7 +971,6 @@ test "issue-179: block comment does not produce phantom symbols" { try testing.expect(!found_fake); } - test "issue-179: code after single-line /* */ comment is parsed" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1021,7 +989,6 @@ test "issue-179: code after single-line /* */ comment is parsed" { try testing.expect(found); } - test "issue-179: Python docstring with text does not leak symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1069,7 +1036,6 @@ test "issue-518: Python class is labeled class_def, not struct_def" { try expectOutlineSymbol(&outline, "Widget", .class_def); } - test "issue-108: HCL resource block parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1086,7 +1052,6 @@ test "issue-108: HCL resource block parsed" { try testing.expectEqual(SymbolKind.struct_def, results[0].symbol.kind); } - test "issue-108: HCL variable and output parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1110,7 +1075,6 @@ test "issue-108: HCL variable and output parsed" { try testing.expectEqual(SymbolKind.constant, outs[0].symbol.kind); } - test "issue-108: HCL module and provider parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1132,7 +1096,6 @@ test "issue-108: HCL module and provider parsed" { try testing.expect(mods.len == 1); } - test "issue-108: HCL comment lines skipped" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1148,7 +1111,6 @@ test "issue-108: HCL comment lines skipped" { try testing.expect(results.len == 1); } - test "issue-215: R function assignment parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1165,7 +1127,6 @@ test "issue-215: R function assignment parsed" { try testing.expectEqual(SymbolKind.function, results[0].symbol.kind); } - test "issue-215: R library import parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1179,7 +1140,6 @@ test "issue-215: R library import parsed" { try testing.expectEqual(@as(usize, 2), outline.imports.items.len); } - test "issue-215: R setClass parsed" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1198,7 +1158,6 @@ test "issue-215: R setClass parsed" { try testing.expect(a2.len == 1); } - test "issue-319: C parser extracts includes macros types and functions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1276,7 +1235,6 @@ test "issue-319: C parser extracts includes macros types and functions" { try testing.expectEqual(SymbolKind.function, alloc_item[0].symbol.kind); } - test "issue-319: C parser avoids comments strings prototypes and macro calls" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1323,7 +1281,6 @@ test "issue-319: C parser avoids comments strings prototypes and macro calls" { try testing.expectEqual(@as(usize, 0), handler.len); } - test "issue-321: common detected extensions produce outlines" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1639,7 +1596,6 @@ test "issue-321: common detected extensions produce outlines" { try testing.expectEqual(@as(usize, 1), r0.len); } - test "issue-179: Python inline docstring does not leak symbols" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1661,7 +1617,6 @@ test "issue-179: Python inline docstring does not leak symbols" { try testing.expectEqual(@as(usize, 0), fake.len); } - test "issue-179: Python multi-line docstring with def inside" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1687,7 +1642,6 @@ test "issue-179: Python multi-line docstring with def inside" { try testing.expectEqual(@as(usize, 0), inner.len); } - test "issue-331: C parser does not index indented call sites as functions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1726,7 +1680,6 @@ test "issue-331: C parser does not index indented call sites as functions" { try testing.expect(found_real); } - test "issue-331: C parser finds nginx-style split-line definitions" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1760,7 +1713,6 @@ test "issue-331: C parser finds nginx-style split-line definitions" { try testing.expect(found_create); } - test "issue-392: Swift parser" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1872,7 +1824,6 @@ test "issue-532: ReScript parser" { try expectOutlineImport(&outline, "Belt"); } - // ─── audit (2026-06-09): latent-issue sweep — parser/deps fixes ─── // src/explore.zig parseDelimitedImport — Kotlin/Swift `import X as Y` stored "X as Y" as @@ -1945,7 +1896,6 @@ test "audit: Python comma import records all modules" { try expectOutlineImport(&outline, "beta.py"); } - // renderImportedBy (the MCP codedb_deps reverse path) kept the unconditional // basename fallback when getImportedBy gained the ambiguity guard — the same // bare import is still attributed to every same-basename file through the diff --git a/src/test_query.zig b/src/test_query.zig index 87545cb6..9e87b056 100644 --- a/src/test_query.zig +++ b/src/test_query.zig @@ -13,12 +13,10 @@ const Language = explore.Language; const SymbolKind = explore.SymbolKind; const mcp_mod = @import("mcp.zig"); - const fuzzyScore = @import("explore.zig").fuzzyScore; const AgentRegistry = @import("agent.zig").AgentRegistry; const edit_mod = @import("edit.zig"); - test "issue-360: edit rejects mismatched if_hash and leaves file untouched" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -53,7 +51,6 @@ test "issue-360: edit rejects mismatched if_hash and leaves file untouched" { try testing.expectEqualStrings(original, after_bytes); } - test "issue-360: edit response reports hex hash matching codedb_read" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -87,7 +84,6 @@ test "issue-360: edit response reports hex hash matching codedb_read" { try testing.expectEqual(expected_hash, result.new_hash); } - test "issue-360: edit dry_run returns diff preview and leaves file untouched" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -134,7 +130,6 @@ test "issue-360: edit dry_run returns diff preview and leaves file untouched" { try testing.expect(std.mem.indexOf(u8, preview, "+BETA") != null); } - test "issue-163: fuzzy exact match scores highest" { const exact = fuzzyScore("main.zig", "src/main.zig"); const partial = fuzzyScore("main.zig", "src/main_helper.zig"); @@ -143,21 +138,18 @@ test "issue-163: fuzzy exact match scores highest" { try testing.expect(exact.? > partial.?); } - test "issue-163: fuzzy subsequence match works" { const score = fuzzyScore("authmid", "src/auth_middleware.py"); try testing.expect(score != null); try testing.expect(score.? > 0); } - test "issue-163: fuzzy typo-tolerant (missing char)" { // "auth_midlware" missing the 'd' in middleware — should still match via subsequence const score = fuzzyScore("auth_midlware", "src/auth_middleware.py"); try testing.expect(score != null); } - test "issue-163: fuzzy word boundary bonus" { // "auth" at word boundary should score higher than "auth" buried in a word const boundary = fuzzyScore("auth", "src/auth_handler.py"); @@ -167,7 +159,6 @@ test "issue-163: fuzzy word boundary bonus" { try testing.expect(boundary.? > buried.?); } - test "issue-163: fuzzy filename ranks above directory" { // "test" in filename portion should score higher than "test" only in directory const in_name = fuzzyScore("test", "src/test_auth.py"); @@ -177,7 +168,6 @@ test "issue-163: fuzzy filename ranks above directory" { try testing.expect(in_name.? > in_dir.?); } - test "issue-163: fuzzy no match returns null" { const score = fuzzyScore("zzzzxyz", "src/main.zig"); try testing.expect(score == null); @@ -196,7 +186,6 @@ test "issue-518: fuzzy find has a subsequence floor — garbage queries return n try testing.expect(fuzzyScore("auth_midlware", "src/auth_middleware.py") != null); } - test "issue-163: fuzzyFindFiles via Explorer" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -214,7 +203,6 @@ test "issue-163: fuzzyFindFiles via Explorer" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_middleware") != null); } - test "issue-163: multi-part query matches both parts" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -231,7 +219,6 @@ test "issue-163: multi-part query matches both parts" { try testing.expect(std.mem.indexOf(u8, results[0].path, "middleware") != null); } - test "issue-163: extension constraint filters results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -250,7 +237,6 @@ test "issue-163: extension constraint filters results" { } } - test "issue-163: special entry point files get bonus" { const score_main = fuzzyScore("main", "src/main.zig"); const score_regular = fuzzyScore("main", "src/maintain.zig"); @@ -260,7 +246,6 @@ test "issue-163: special entry point files get bonus" { try testing.expect(score_main.? > score_regular.?); } - test "issue-163: transpositions handled by Smith-Waterman" { // These all failed with the old subsequence matcher try testing.expect(fuzzyScore("mpc", "src/mcp.zig") != null); @@ -269,7 +254,6 @@ test "issue-163: transpositions handled by Smith-Waterman" { try testing.expect(fuzzyScore("indxe", "src/index.zig") != null); } - test "issue-168: query pipeline find → limit produces file set" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -295,7 +279,6 @@ test "issue-168: query pipeline find → limit produces file set" { try testing.expect(found_handler); } - test "issue-168: query pipeline search returns matching lines" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -316,7 +299,6 @@ test "issue-168: query pipeline search returns matching lines" { try testing.expect(std.mem.indexOf(u8, results[0].path, "main.zig") != null); } - test "issue-168: query pipeline filter by extension" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -335,7 +317,6 @@ test "issue-168: query pipeline filter by extension" { } } - test "issue-168: query pipeline outline returns symbols" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -347,7 +328,6 @@ test "issue-168: query pipeline outline returns symbols" { try testing.expect(outline.symbols.items.len >= 2); } - test "issue-168: query pipeline chained find → filter narrows results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -368,7 +348,6 @@ test "issue-168: query pipeline chained find → filter narrows results" { try testing.expect(py_only.len < all.len); // filtered set is smaller } - test "issue-168: query pipeline handles empty results gracefully" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -381,7 +360,6 @@ test "issue-168: query pipeline handles empty results gracefully" { try testing.expectEqual(@as(usize, 0), results.len); } - test "issue-168: recall — find + filter preserves only matching extension" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -403,7 +381,6 @@ test "issue-168: recall — find + filter preserves only matching extension" { for (py) |r| try testing.expect(std.mem.endsWith(u8, r.path, ".py")); } - test "issue-168: recall — search finds content across multiple files" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -436,7 +413,6 @@ test "issue-168: recall — search finds content across multiple files" { try testing.expect(!found_c); } - test "issue-168: recall — fuzzy find ranks exact matches highest" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -457,7 +433,6 @@ test "issue-168: recall — fuzzy find ranks exact matches highest" { } } - test "issue-168: recall — multi-part query intersection" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -475,7 +450,6 @@ test "issue-168: recall — multi-part query intersection" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_controller") != null); } - test "issue-168: recall — transposition tolerance in pipeline" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -492,7 +466,6 @@ test "issue-168: recall — transposition tolerance in pipeline" { try testing.expect(std.mem.indexOf(u8, results[0].path, "middleware") != null); } - test "auto-retry: delimiter stripping finds results" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -506,7 +479,6 @@ test "auto-retry: delimiter stripping finds results" { try testing.expect(std.mem.indexOf(u8, results[0].path, "auth_middleware") != null); } - test "per-file truncation: max 5 matches per file in output" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -533,7 +505,6 @@ test "per-file truncation: max 5 matches per file in output" { try testing.expect(results.len >= 10); } - test "issue-359: globPaths matches files by glob pattern" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -570,7 +541,6 @@ test "issue-359: globPaths matches files by glob pattern" { } } - test "issue-359: lsDir returns immediate children with file metadata" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -624,7 +594,6 @@ test "issue-359: lsDir returns immediate children with file metadata" { try testing.expectEqual(@as(usize, 2), file_count); } - test "issue-359: mcp.globMatch backtracks across **/* boundary" { // Pipeline filter (codedb_query) calls mcp.globMatch on each path. The // iterative version forgot the outer ** position when it entered the @@ -641,7 +610,6 @@ test "issue-359: mcp.globMatch backtracks across **/* boundary" { try testing.expect(!mcp_mod.globMatch("docs/*.md", "src/mcp.zig")); } - test "issue-511: glob supports brace alternatives" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -662,7 +630,6 @@ test "issue-511: glob supports brace alternatives" { try testing.expect(!mcp_mod.globMatch("src/{mcp,explore}.zig", "src/main.zig")); } - test "issue-359: globPaths recall — every matching path survives at every depth" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -712,7 +679,6 @@ test "issue-359: globPaths recall — every matching path survives at every dept try testing.expect(!mcp_mod.globMatch("src/**/*.zig", "tests/h.zig")); } - test "issue-359/360: retrieval recall — search/word/symbol/fuzzy/glob/deps all return ground truth" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -854,7 +820,6 @@ test "issue-359/360: retrieval recall — search/word/symbol/fuzzy/glob/deps all } } - test "issue-356-1: codedb_query returns partial results when a step fails" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -895,7 +860,6 @@ test "issue-356-1: codedb_query returns partial results when a step fails" { try testing.expect(std.mem.indexOf(u8, out.items, "failed_at: 1") != null); } - test "issue-356-2: codedb_outline suggests fuzzy alternatives for non-indexed paths" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -930,7 +894,6 @@ test "issue-356-2: codedb_outline suggests fuzzy alternatives for non-indexed pa try testing.expect(std.mem.indexOf(u8, out.items, "src/main.zig") != null); } - test "issue-356-3: codedb_query surfaces received keys on missing-arg errors" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -965,7 +928,6 @@ test "issue-356-3: codedb_query surfaces received keys on missing-arg errors" { try testing.expect(std.mem.indexOf(u8, out.items, "q") != null); } - test "issue-356-p2: codedb_outline missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -995,7 +957,6 @@ test "issue-356-p2: codedb_outline missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "file_path") != null); } - test "issue-356-p2: codedb_symbol missing name surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1025,7 +986,6 @@ test "issue-356-p2: codedb_symbol missing name surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "symbol") != null); } - test "issue-356-p2: codedb_search missing query surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1054,7 +1014,6 @@ test "issue-356-p2: codedb_search missing query surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_word missing word surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1083,7 +1042,6 @@ test "issue-356-p2: codedb_word missing word surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_read missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1112,7 +1070,6 @@ test "issue-356-p2: codedb_read missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p2: codedb_deps missing path surfaces received keys" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1141,7 +1098,6 @@ test "issue-356-p2: codedb_deps missing path surfaces received keys" { try testing.expect(std.mem.indexOf(u8, out.items, "received keys") != null); } - test "issue-356-p3: codedb_query emits per-stage summary tail on success" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1180,7 +1136,6 @@ test "issue-356-p3: codedb_query emits per-stage summary tail on success" { try testing.expect(std.mem.indexOf(u8, out.items, "1: sort") != null); } - test "issue-356-p3: codedb_outline includes actionable hint when parser fails" { var explorer = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer.deinit(); @@ -1216,7 +1171,6 @@ test "issue-356-p3: codedb_outline includes actionable hint when parser fails" { try testing.expect(std.mem.indexOf(u8, out.items, "codedb_index") != null); } - test "issue-356-p3: codedb_read appends fuzzy suggestions when path is unreadable" { const tmp_io = testing.io; var tmp = testing.tmpDir(.{}); @@ -1264,7 +1218,6 @@ test "issue-356-p3: codedb_read appends fuzzy suggestions when path is unreadabl try testing.expect(std.mem.indexOf(u8, out.items, "src/main.zig") != null); } - test "issue-558: codedb_query filter must filter (or error) — never silently pass every file" { // Live repro: [{find},{filter,"pattern":"src/index.zig"},{limit,n:3}] // returned all 50 find results — filter only reads 'ext'/'glob', so an diff --git a/src/test_search.zig b/src/test_search.zig index bf4fa13e..6cc370d6 100644 --- a/src/test_search.zig +++ b/src/test_search.zig @@ -507,6 +507,61 @@ test "issue-393: BM25 ranking surfaces high-density file before single-mention f } } +test "searchContent Tier-0 grouped postings match fragmented fallback exactly" { + var grouped = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer grouped.deinit(); + var fragmented = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); + defer fragmented.deinit(); + + const fixtures = [_]struct { path: []const u8, content: []const u8 }{ + .{ .path = "src/a.zig", .content = "const needle = 1;\n// needle second occurrence\n" }, + .{ .path = "docs/b.md", .content = "needle documentation\n" }, + .{ .path = "src/c.zig", .content = "fn needle() void {}\n" }, + .{ .path = "src/noise.zig", .content = "const unrelated = true;\n" }, + }; + for (fixtures) |fixture| { + try grouped.indexFile(fixture.path, fixture.content); + try fragmented.indexFile(fixture.path, fixture.content); + } + + // Natural postings are grouped by document: A1, A2, B1, C1. Reorder the + // equivalent control list to A1, B1, A2, C1 so searchContent must take its + // defensive slots/hash aggregation path. No semantic hit is added or lost. + const postings = fragmented.word_index.index.getPtr("needle") orelse return error.TestUnexpectedResult; + try testing.expectEqual(@as(usize, 4), postings.items.len); + const a_second = postings.items[1]; + postings.items[1] = postings.items[2]; + postings.items[2] = a_second; + try testing.expect(postings.items[1].doc_id > postings.items[2].doc_id); + + // A cap of two forces both paths to make the same ordering/truncation + // decision rather than merely comparing an unbounded candidate set. + const expected = try grouped.searchContent("needle", testing.allocator, 2); + defer { + for (expected) |result| { + testing.allocator.free(result.path); + testing.allocator.free(result.line_text); + } + testing.allocator.free(expected); + } + const actual = try fragmented.searchContent("needle", testing.allocator, 2); + defer { + for (actual) |result| { + testing.allocator.free(result.path); + testing.allocator.free(result.line_text); + } + testing.allocator.free(actual); + } + + try testing.expectEqual(expected.len, actual.len); + for (expected, actual) |a, b| { + try testing.expectEqualStrings(a.path, b.path); + try testing.expectEqual(a.line_num, b.line_num); + try testing.expectEqualStrings(a.line_text, b.line_text); + try testing.expectEqual(a.score, b.score); + } +} + test "issue-400: BM25 ranks both-terms file above single-term files" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1907,8 +1962,7 @@ test "def-first: renderPlainSearch surfaces the definition line before mentions // A comment mentioning `gadget` appears BEFORE its definition; several call // sites follow. Line-order rendering would show the line-1 comment first. - try explorer.indexFile("src/g.zig", - "// gadget helper\nconst a = gadget;\nconst b = gadget;\npub fn gadget() void {}\ngadget();\ngadget();\n"); + try explorer.indexFile("src/g.zig", "// gadget helper\nconst a = gadget;\nconst b = gadget;\npub fn gadget() void {}\ngadget();\ngadget();\n"); var out: std.ArrayList(u8) = .empty; defer out.deinit(testing.allocator); diff --git a/src/test_snapshot.zig b/src/test_snapshot.zig index 434a4e8e..891c16d5 100644 --- a/src/test_snapshot.zig +++ b/src/test_snapshot.zig @@ -17,7 +17,6 @@ const git_mod = @import("git.zig"); const AgentRegistry = @import("agent.zig").AgentRegistry; const edit_mod = @import("edit.zig"); - test "issue-625: in-tree snapshot is added to .git/info/exclude" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -110,7 +109,6 @@ test "issue-35: edits immediately update explorer and snapshot output" { try testing.expect(std.mem.indexOf(u8, after_snap, "oldName") == null); } - test "snapshot_json: snapshot builds and is valid JSON" { // Explorer uses arena for internal data var arena = std.heap.ArenaAllocator.init(testing.allocator); @@ -148,7 +146,6 @@ test "snapshot_json: snapshot builds and is valid JSON" { try testing.expect(symbol_index.contains("version")); } - test "issue-44: snapshot stale after working tree changes cause stale query results" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -205,7 +202,6 @@ test "issue-44: snapshot stale after working tree changes cause stale query resu try testing.expect(results.len == 1); } - test "issue-46: empty-repo snapshot rejected on load" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -378,7 +374,6 @@ test "snapshot: CONTENT_HASHES records the correct per-file hash (order-aligned) } } - test "issue-220: snapshot fast load restores outlines and lazily rebuilds word index" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -535,7 +530,7 @@ test "snapshot: parallel freshness load re-indexes changed files, restores the r try testing.expect(snapshot_mod.loadSnapshot(io, snap_path, &exp2, &store, arena2.allocator())); try testing.expectEqual(total, exp2.outlines.count()); - var is_changed = [_]bool{false} ** total; + var is_changed: [total]bool = @splat(false); for (changed) |i| is_changed[i] = true; // Changed files carry fresh content (changed branch); the rest keep snapshot @@ -559,7 +554,6 @@ test "snapshot: parallel freshness load re-indexes changed files, restores the r } } - test "snapshot: writer streams uncached file contents for large repos" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -617,7 +611,6 @@ test "snapshot: writer streams uncached file contents for large repos" { try testing.expect(loaded.wordIndexIsComplete()); } - test "issue-220: partial word index state rebuilds before search" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -660,7 +653,6 @@ test "issue-220: partial word index state rebuilds before search" { try testing.expect(exp2.wordIndexNeedsPersist()); } - test "issue-220: word index persistence tracking skips redundant rewrites" { var exp = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer exp.deinit(); @@ -685,7 +677,6 @@ test "issue-220: word index persistence tracking skips redundant rewrites" { try testing.expect(!exp.wordIndexNeedsPersist()); } - test "issue-45: snapshot written in non-git directory cannot be loaded" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -717,7 +708,6 @@ test "issue-45: snapshot written in non-git directory cannot be loaded" { try testing.expect(snap_head == null); } - test "issue-47: concurrent snapshot writes from parallel instances corrupt file" { // BUG: Two codedb instances indexing the same repo write codedb.snapshot // concurrently with no file locking. The second writer can overwrite a @@ -787,7 +777,6 @@ test "issue-47: concurrent snapshot writes from parallel instances corrupt file" try testing.expect(loaded); } - test "issue-42: scan thread is joined before allocator-backed state is freed" { var gpa = std.heap.DebugAllocator(.{}){}; const allocator = gpa.allocator(); @@ -819,7 +808,6 @@ test "issue-42: scan thread is joined before allocator-backed state is freed" { _ = gpa.deinit(); } - test "issue-40: truncated snapshot silently loads partial data" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -860,7 +848,6 @@ test "issue-40: truncated snapshot silently loads partial data" { try testing.expect(!loaded); } - test "issue-41: snapshot not validated against repo identity allows cross-project loading" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -889,7 +876,6 @@ test "issue-41: snapshot not validated against repo identity allows cross-projec try testing.expect(!loaded); } - test "snapshot: symbol detail longer than 4096 bytes survives round-trip" { // Regression for readSectionString rejecting names/details > 4096 bytes. // Before the fix max_len was 4096; any detail longer than that triggered @@ -936,7 +922,6 @@ test "snapshot: symbol detail longer than 4096 bytes survives round-trip" { try testing.expect(results.len >= 1); } - test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { // Regression for the codedb 0.2.56 writer u16 overflow bug: when OUTLINE_STATE // contains a detail that overflows u16 the section cursor de-syncs, making @@ -970,7 +955,7 @@ test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { const ols = sections.get(@intFromEnum(snapshot_mod.SectionId.outline_state)) orelse return; const f = try std.Io.Dir.cwd().openFile(io, snap_path, .{ .mode = .read_write }); defer f.close(io); - try f.writePositionalAll(io, &([_]u8{0xFF} ** 16), ols.offset); + try f.writePositionalAll(io, &@as([16]u8, @splat(0xFF)), ols.offset); } var exp2 = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); @@ -988,7 +973,6 @@ test "snapshot: corrupted OUTLINE_STATE section falls back to CONTENT load" { try testing.expect(results.len >= 1); } - test "issue-379: snapshot loader returns true with zero outlines for empty-explorer snapshot" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -1017,7 +1001,6 @@ test "issue-379: snapshot loader returns true with zero outlines for empty-explo } } - test "issue-528: isSensitivePath parity between snapshot.zig and watcher.zig" { // The secret/credential filter is duplicated (snapshot persistence vs live // indexing). The #528 audit flagged a possible divergence; the two are @@ -1026,20 +1009,20 @@ test "issue-528: isSensitivePath parity between snapshot.zig and watcher.zig" { // one path but not the other. const cases = [_][]const u8{ // secrets — both copies must block - ".env", ".env.local", ".env.production", - ".env.development", ".env.staging", ".env.test", - ".dev.vars", ".npmrc", ".pypirc", - ".netrc", "credentials.json", "service-account.json", - "secrets.json", "secrets.yaml", "secrets.yml", - "id_rsa", "id_ed25519", "server.key", - "cert.pem", "keystore.jks", "identity.pfx", - "bundle.p12", "config/.env.local", "a/b/secrets.yaml", + ".env", ".env.local", ".env.production", + ".env.development", ".env.staging", ".env.test", + ".dev.vars", ".npmrc", ".pypirc", + ".netrc", "credentials.json", "service-account.json", + "secrets.json", "secrets.yaml", "secrets.yml", + "id_rsa", "id_ed25519", "server.key", + "cert.pem", "keystore.jks", "identity.pfx", + "bundle.p12", "config/.env.local", "a/b/secrets.yaml", "deep/nested/.ssh/known_hosts", ".gnupg/secring.gpg", "x/.aws/credentials", // non-secrets — both copies must allow (esp. the .env-prefix edge cases) - ".envoy.json", ".environment", ".envrc", - ".envconfig.yaml", "main.zig", "src/server.zig", - "README.md", "package.json", "id_rsa.pub", - "envvars.ts", "Makefile", "Dockerfile", + ".envoy.json", ".environment", ".envrc", + ".envconfig.yaml", "main.zig", "src/server.zig", + "README.md", "package.json", "id_rsa.pub", + "envvars.ts", "Makefile", "Dockerfile", }; for (cases) |p| { try testing.expectEqual(watcher.isSensitivePath(p), snapshot_mod.isSensitivePath(p)); @@ -1271,7 +1254,6 @@ test "issue-539b: search recall ranks a relevant restored file above quota (inde try testing.expect(found_canonical); } - test "issue-564: snapshot fast-load defers the symbol index until first symbol use" { // Pre-#564 the fast-load eagerly rebuilt the global symbol index for every // restored file (~symbols-per-file map inserts and their heap) even though @@ -1320,7 +1302,7 @@ test "audit: long import specifier round-trips on fast-restore (borrow path pres var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); var exp = Explorer.init(arena.allocator(), Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); - const long_imp = "a" ** 5000; // > 4096 (old read cap), <= 65535 (write cap) + const long_imp: [5000]u8 = @splat('a'); // > 4096 (old read cap), <= 65535 (write cap) const src = "package main\nimport \"" ++ long_imp ++ "\"\nfunc mainFn() {}\n"; try exp.indexFile("probe/main.go", src); const pre = exp.outlines.get("probe/main.go") orelse return error.MissingOutline; diff --git a/src/wasm.zig b/src/wasm.zig index 29e4009f..52ecd334 100644 --- a/src/wasm.zig +++ b/src/wasm.zig @@ -96,8 +96,9 @@ export fn wasm_get_outline( defer outline.deinit(); // Serialize to JSON with proper escaping - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; for (outline.symbols.items, 0..) |sym, i| { if (i > 0) writer.writeByte(',') catch return null; @@ -115,7 +116,7 @@ export fn wasm_get_outline( } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -142,8 +143,9 @@ export fn wasm_search( } // Serialize to JSON - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; for (results, 0..) |r, i| { if (i > 0) writer.writeByte(',') catch return null; @@ -171,7 +173,7 @@ export fn wasm_search( } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -180,8 +182,9 @@ export fn wasm_search( export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { const exp = getExplorer(); - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('[') catch return null; var first = true; var iter = exp.outlines.iterator(); @@ -201,7 +204,7 @@ export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { } writer.writeByte(']') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } @@ -210,8 +213,9 @@ export fn wasm_get_tree(out_len_ptr: *usize) ?[*]u8 { export fn wasm_get_all_outlines(out_len_ptr: *usize) ?[*]u8 { const exp = getExplorer(); - var buf: std.ArrayList(u8) = .empty; - const writer = buf.writer(allocator); + var buf: std.Io.Writer.Allocating = .init(allocator); + defer buf.deinit(); + const writer = &buf.writer; writer.writeByte('{') catch return null; var first = true; var iter = exp.outlines.iterator(); @@ -239,7 +243,7 @@ export fn wasm_get_all_outlines(out_len_ptr: *usize) ?[*]u8 { } writer.writeByte('}') catch return null; - const slice = buf.toOwnedSlice(allocator) catch return null; + const slice = buf.toOwnedSlice() catch return null; out_len_ptr.* = slice.len; return slice.ptr; } diff --git a/src/watcher.zig b/src/watcher.zig index cac08b49..539dc963 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -44,7 +44,7 @@ pub const FsEvent = struct { pub const EventQueue = struct { const CAPACITY = 4096; - events: [CAPACITY]?FsEvent = [_]?FsEvent{null} ** CAPACITY, + events: [CAPACITY]?FsEvent = @splat(null), head: usize = 0, tail: usize = 0, mu: cio.Mutex = .{}, @@ -601,14 +601,12 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat fn parseInitialScanEntry( io: std.Io, - root: []const u8, + dir: std.Io.Dir, entry: InitialScanEntry, content_alloc: std.mem.Allocator, parse_alloc: std.mem.Allocator, ) !?ParsedScanFile { if (shouldSkipFile(entry.path)) return null; - const dir = try std.Io.Dir.cwd().openDir(io, root, .{}); - defer dir.close(io); const content = (try readIndexableFile(io, dir, entry.path, content_alloc, entry.size, true)) orelse return null; errdefer content_alloc.free(content); // Threshold for including a file in the trigram index. Bumped from 64KB to @@ -629,13 +627,13 @@ fn parseInitialScanEntry( fn parseInitialScanWorkerEntry( io: std.Io, results: *WorkerParsedResults, - root: []const u8, + dir: std.Io.Dir, entry: InitialScanEntry, scratch_alloc: std.mem.Allocator, word_shard: ?*WordIndex, ) ?PersistedScanFile { const persist = results.arena.allocator(); - const parsed = parseInitialScanEntry(io, root, entry, persist, scratch_alloc) catch return null; + const parsed = parseInitialScanEntry(io, dir, entry, persist, scratch_alloc) catch return null; if (parsed == null) return null; var file = parsed.?; @@ -661,6 +659,12 @@ fn parseInitialScanWorkerEntry( } fn initialScanWorker(io: std.Io, results: *WorkerParsedResults, root: []const u8, entries: []const InitialScanEntry, word_shard: ?*WordIndex, trigram_shard: ?*TrigramIndex) void { + const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch { + for (0..entries.len) |index| results.publish(io, index, null); + return; + }; + defer dir.close(io); + // Read content directly into the persistent result arena, while allocating // transient parser state in a per-file scratch arena that is reset between // files. The final outline uses packed libc-owned storage so the main thread @@ -676,7 +680,7 @@ fn initialScanWorker(io: std.Io, results: *WorkerParsedResults, root: []const u8 defer scratch.deinit(); for (entries, 0..) |entry, index| { _ = scratch.reset(.retain_capacity); - const file = parseInitialScanWorkerEntry(io, results, root, entry, scratch.allocator(), word_shard); + const file = parseInitialScanWorkerEntry(io, results, dir, entry, scratch.allocator(), word_shard); if (file) |f| { if (trigram_shard) |ts| { if (!f.skip_trigram and f.content.len <= max_trigram_file_bytes) { @@ -724,7 +728,7 @@ pub fn initialScanWithWorkerCount(io: std.Io, store: *Store, explorer: *Explorer { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); - const parsed = try parseInitialScanEntry(io, root, entry, arena.allocator(), arena.allocator()); + const parsed = try parseInitialScanEntry(io, dir, entry, arena.allocator(), arena.allocator()); if (parsed) |file| { try explorer.commitParsedFileOwnedOutline(file.path, file.content, file.outline, true, file.skip_trigram); } @@ -843,42 +847,68 @@ pub fn initialScanWithWorkerCount(io: std.Io, store: *Store, explorer: *Explorer } } +fn extractTrigramMasks( + content: []const u8, + local: *std.AutoHashMap(@import("index.zig").Trigram, @import("index.zig").PostingMask), +) !void { + if (content.len < 3) return; + const index_m = @import("index.zig"); + var c0 = content[0]; + var c1 = content[1]; + var c2 = content[2]; + var n0 = index_m.normalizeChar(c0); + var n1 = index_m.normalizeChar(c1); + var n2 = index_m.normalizeChar(c2); + + for (0..content.len - 2) |i| { + const has_next = i + 3 < content.len; + const c3 = if (has_next) content[i + 3] else 0; + const n3 = if (has_next) index_m.normalizeChar(c3) else 0; + if (!((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and + (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and + (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r'))) + { + const tri = index_m.packTrigram(n0, n1, n2); + const gop = try local.getOrPut(tri); + if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; + gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i & 7); + if (has_next) { + gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(n3 & 7); + } + } + c0 = c1; + c1 = c2; + c2 = c3; + n0 = n1; + n1 = n2; + n2 = n3; + } +} + // Build a private trigram shard from a chunk of InitialScanEntry items — reads // each file, extracts trigrams, and inserts postings directly, all on the worker. // Used by the skip_outlines fast path in initialScanWithTrigrams. fn readAndBuildTrigramShardWorker(io: std.Io, shard: *TrigramIndex, root: []const u8, entries: []const InitialScanEntry, build_error: *?anyerror) void { + const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch |err| { + build_error.* = err; + return; + }; + defer dir.close(io); + const index_m = @import("index.zig"); var local = std.AutoHashMap(index_m.Trigram, index_m.PostingMask).init(std.heap.c_allocator); defer local.deinit(); local.ensureTotalCapacity(4096) catch {}; for (entries) |entry| { if (shouldSkipFile(entry.path)) continue; - const dir = std.Io.Dir.cwd().openDir(io, root, .{}) catch continue; - defer dir.close(io); const content = (readIndexableFile(io, dir, entry.path, std.heap.c_allocator, entry.size, false) catch continue) orelse continue; defer std.heap.c_allocator.free(content); if (content.len > max_trigram_file_bytes) continue; local.clearRetainingCapacity(); - if (content.len >= 3) { - for (0..content.len - 2) |i| { - const c0 = content[i]; - const c1 = content[i + 1]; - const c2 = content[i + 2]; - if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and - (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and - (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; - const tri = index_m.packTrigram(index_m.normalizeChar(c0), index_m.normalizeChar(c1), index_m.normalizeChar(c2)); - const gop = local.getOrPut(tri) catch |err| { - build_error.* = err; - return; - }; - if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; - gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); - if (i + 3 < content.len) { - gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(index_m.normalizeChar(content[i + 3]) % 8); - } - } - } + extractTrigramMasks(content, &local) catch |err| { + build_error.* = err; + return; + }; shard.insertBulkMapNew(entry.path, &local) catch |err| { build_error.* = err; return; @@ -898,26 +928,10 @@ fn cachedTrigramBuildWorker(shard: *TrigramIndex, entries: []const CachedEntry, local.ensureTotalCapacity(4096) catch {}; for (entries) |entry| { local.clearRetainingCapacity(); - if (entry.content.len >= 3) { - for (0..entry.content.len - 2) |i| { - const c0 = entry.content[i]; - const c1 = entry.content[i + 1]; - const c2 = entry.content[i + 2]; - if ((c0 == ' ' or c0 == '\t' or c0 == '\n' or c0 == '\r') and - (c1 == ' ' or c1 == '\t' or c1 == '\n' or c1 == '\r') and - (c2 == ' ' or c2 == '\t' or c2 == '\n' or c2 == '\r')) continue; - const tri = index_m.packTrigram(index_m.normalizeChar(c0), index_m.normalizeChar(c1), index_m.normalizeChar(c2)); - const gop = local.getOrPut(tri) catch |err| { - build_error.* = err; - return; - }; - if (!gop.found_existing) gop.value_ptr.* = index_m.PostingMask{}; - gop.value_ptr.loc_mask |= @as(u8, 1) << @intCast(i % 8); - if (i + 3 < entry.content.len) { - gop.value_ptr.next_mask |= @as(u8, 1) << @intCast(index_m.normalizeChar(entry.content[i + 3]) % 8); - } - } - } + extractTrigramMasks(entry.content, &local) catch |err| { + build_error.* = err; + return; + }; shard.insertBulkMapNew(entry.path, &local) catch |err| { build_error.* = err; return; @@ -1070,7 +1084,7 @@ pub fn initialScanWithTrigrams( for (entries.items) |entry| { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); - const parsed = parseInitialScanEntry(io, root, entry, arena.allocator(), arena.allocator()) catch null; + const parsed = parseInitialScanEntry(io, dir, entry, arena.allocator(), arena.allocator()) catch null; if (parsed) |file| { if (!skip_outlines) { explorer.commitParsedFileOwnedOutline(file.path, file.content, file.outline, true, true) catch continue; diff --git a/vendor/nanoregex/.gitignore b/vendor/nanoregex/.gitignore new file mode 100644 index 00000000..93cfddff --- /dev/null +++ b/vendor/nanoregex/.gitignore @@ -0,0 +1,12 @@ +# Zig build artifacts +.zig-cache/ +zig-cache/ +zig-out/ + +# macOS +.DS_Store + +# Editor +*.swp +.vscode/ +.idea/ diff --git a/vendor/nanoregex/LICENSE b/vendor/nanoregex/LICENSE new file mode 100644 index 00000000..7b09d8bb --- /dev/null +++ b/vendor/nanoregex/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rach Pradhan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/nanoregex/README.md b/vendor/nanoregex/README.md new file mode 100644 index 00000000..14830ed4 --- /dev/null +++ b/vendor/nanoregex/README.md @@ -0,0 +1,136 @@ +# nanoregex + +Small, fast, pure-Zig regex engine. Built to be a drop-in replacement for `zig-regex 0.1.1` with substantially better performance on real workloads. + +3,200 lines of Zig. No FFI. No external dependencies. 27/27 parity fixtures green against Python's `re` module. + +## Benchmarks + +Search across a 142 KB Zig source file, 200 iterations, ReleaseFast. + +| Pattern | Python `re` | **nanoregex** | Winner | +|---|---|---|---| +| pure literal `compileAllocFlags` | 0.067ms | **0.037ms** | nanoregex 1.8× | +| literal-prefix `compileAllocFlags\([a-z]+` | 0.061ms | **0.037ms** | nanoregex 1.65× | +| `fn [A-Za-z]+\(.*alloc` | **0.061ms** | 0.092ms | python 1.5× | +| `\d+` (1307 matches) | 0.991ms | **0.455ms** | nanoregex 2.2× | +| `[a-z]+` (17711 matches) | 1.654ms | **0.594ms** | nanoregex 2.8× | +| alt `foo\|bar\|baz` | 0.731ms | **0.423ms** | nanoregex 1.7× | +| IPv4-ish `\d+\.\d+\.\d+\.\d+` | 0.904ms | **0.416ms** | nanoregex 2.2× | + +8 of 8 head-to-head non-anchored patterns won. Versus `zig-regex 0.1.1` on a pattern that triggers catastrophic backtracking, nanoregex is ~5000× faster (43 seconds → 8 milliseconds). + +## Architecture + +Layered, with five dispatch tiers that compose at compile time: + +``` +parser.zig pattern bytes → AST +ast.zig AST node tagged union, arena-owned +nfa.zig AST → Thompson NFA +exec.zig Pike-VM simulation (always-correct fallback) +dfa.zig Lazy subset-construction DFA (perf path) +minterm.zig Byte-class compression for the DFA's transition table +prefilter.zig Literal-prefix / required-substring extraction +root.zig Public API + dispatch +``` + +`findAll` and `search` route to the cheapest engine that can correctly handle a given pattern: + +1. **Pure-literal pattern** → `std.mem.indexOf` loop (memmem) +2. **Required-literal absent** → return empty (no engine work at all) +3. **Literal-prefix + DFA-eligible** → `indexOfPos` to candidate starts, DFA at each hit +4. **DFA-eligible** → plain lazy DFA +5. **Otherwise** → Pike VM + +DFA-eligible means: no capture groups, no anchors (`^`, `$`, `\b`), no lazy quantifiers, not case-insensitive, and the on-demand DFA stays under the 4096-state budget. Everything that doesn't fit those rules takes the Pike-VM path, which is linear-time and correct on every input. + +Bytes are folded to **minterm classes** before indexing the DFA's transition table. A pattern with `[a-z]+` reduces 256 bytes to 2 classes (in-set, out-of-set), shrinking the per-state row from 1 KB to 8 bytes and letting the whole transition table live in L1 cache. + +## API + +Mirrors `zig-regex 0.1.1` enough that most callers can switch by changing one path in `build.zig`: + +```zig +const nanoregex = @import("nanoregex"); + +var r = try nanoregex.Regex.compile(allocator, "(\\w+)@(\\w+)"); +defer r.deinit(); + +const matches = try r.findAll(allocator, "alice@example bob@host"); +defer { + for (matches) |*m| @constCast(m).deinit(allocator); + allocator.free(matches); +} +for (matches) |m| { + std.debug.print("{d}..{d}\n", .{ m.span.start, m.span.end }); +} +``` + +Methods take `*Regex` (mutable) rather than `*const Regex` because the lazy DFA fills its transition table on the fly. The first `findAll` call on a fresh `Regex` warms the cache; subsequent calls are pure table lookups. + +Compile flags: + +```zig +try nanoregex.Regex.compileWithFlags(alloc, pattern, .{ + .case_insensitive = false, + .multiline = true, // grep-like default — `^`/`$` match line edges + .dot_all = false, +}); +``` + +Backreference expansion in `replaceAll` (`\1`, `\2`, ...): + +```zig +const out = try r.replaceAll(alloc, "alice@example", "\\2/\\1"); +// → "example/alice" +``` + +## Supported syntax (v1) + +- Literals, `.`, character classes `[abc]` / `[^abc]` / `[a-z]` +- Shorthand `\d \D \w \W \s \S` +- Quantifiers `? * + {n} {n,m}` — greedy and lazy (`*?`, `+?`, `??`, `{n,m}?`) +- Groups `(foo)` capturing, `(?:foo)` non-capturing +- Alternation `foo|bar` +- Anchors `^ $ \b \B \A \z` +- Flags: case-insensitive, multiline, dot-all + +**Not yet supported**: backreferences in *patterns* (`\1` inside the regex itself), lookaround `(?=...)`/`(?!...)`, inline flag groups `(?i)...`, named groups `(?P...)`, Unicode property classes. Patterns using these features parse OK if the syntax shape is recognised, but matching may diverge — fall back to a richer engine if you need them. + +## Build + +```bash +zig build install -Doptimize=ReleaseFast +# → zig-out/bin/nanoregex_probe (parity test CLI) +# → zig-out/bin/nanoregex_bench (single-file benchmark) +``` + +## Tests + +Tests are split into narrow per-module steps so the inner loop stays tight: + +```bash +zig build test-ast # 3 tests +zig build test-parser # parser + ast tests +zig build test-nfa # nfa + parser + ast +zig build test-exec # Pike VM tests +zig build test-prefilter # literal-extraction tests +zig build test-minterm # byte-class compression +zig build test-dfa # DFA construction + matching +zig build test-root # public API +zig build parity # Python re parity (requires python3) +zig build test-all # everything, explicit and opt-in +``` + +Add `-Dtest-filter='substring'` to any step to narrow further. + +## Why it exists + +This was extracted from the [zigrepper](https://github.com/justrach/zigrepper) toolchain, where `zig-regex 0.1.1`'s backtracking engine was making `zigrep --regex` take 43 seconds on patterns like `compileAllocFlags\([a-z]+` against a directory tree. After this engine landed, the same query finished in 0.43 seconds end-to-end. + +Inspired by Russ Cox's writing on regex implementation, RE2's lazy DFA, and the [RE# / Resharp blog post](https://iev.ee/blog/resharp-how-we-built-the-fastest-regex-in-fsharp/) which laid out minterm compression and several other optimizations cleanly. + +## License + +MIT diff --git a/vendor/nanoregex/UPSTREAM.md b/vendor/nanoregex/UPSTREAM.md new file mode 100644 index 00000000..b5478ac9 --- /dev/null +++ b/vendor/nanoregex/UPSTREAM.md @@ -0,0 +1,17 @@ +# Vendored source provenance + +This directory is vendored from [`justrach/nanoregex`](https://github.com/justrach/nanoregex) at commit: + +```text +736b46703454d5f37d3e46164fc91354386bb29c +``` + +It was the dependency previously pinned by codedb's `build.zig.zon`. The local changes are limited to Zig `0.17.0-dev.813+2153f8143` compatibility: removed repetition syntax was replaced with typed `@splat`, the package minimum was updated, and the touched build/source files were canonicalized by that Zig snapshot's formatter (format-only whitespace/layout changes). Keep the upstream MIT `LICENSE` file when updating this copy. + +To verify after an update: + +```bash +cd vendor/nanoregex +zig build test-all +zig build parity +``` diff --git a/vendor/nanoregex/build.zig b/vendor/nanoregex/build.zig new file mode 100644 index 00000000..769b5dd6 --- /dev/null +++ b/vendor/nanoregex/build.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // Pass-through filter that the user can layer on top of any named step + // to narrow further: zig build test-dfa -Dtest-filter='alternation' + const user_filter = b.option([]const u8, "test-filter", "Narrow test name filter (substring)"); + + const nanoregex_mod = b.addModule("nanoregex", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + + const probe = b.addExecutable(.{ + .name = "nanoregex_probe", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/probe.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(probe); + + const bench = b.addExecutable(.{ + .name = "nanoregex_bench", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/bench.zig"), + .target = target, + .optimize = .ReleaseFast, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(bench); + + const bench_comptime = b.addExecutable(.{ + .name = "nanoregex_bench_comptime", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/bench_comptime.zig"), + .target = target, + .optimize = .ReleaseFast, + .link_libc = true, + .imports = &.{.{ .name = "nanoregex", .module = nanoregex_mod }}, + }), + }); + b.installArtifact(bench_comptime); + + // ───────────────────────────────────────────────────────────────── + // Per-module test steps. + // + // The user's preferred iteration loop is to run ONE narrow named step + // at a time, so each module has its own step that compiles a small + // test binary scoped to that source file (+ its imports). There is + // NO aggregate `test` step — `test-all` is explicit and opt-in. + // + // To narrow further: zig build test-dfa -Dtest-filter='alternation' + // ───────────────────────────────────────────────────────────────── + + _ = addTestStep(b, "test-ast", "src/ast.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser", "src/parser.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa", "src/nfa.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-exec", "src/exec.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-prefilter", "src/prefilter.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-dfa", "src/dfa.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-minterm", "src/minterm.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-root", "src/root.zig", &.{}, user_filter, target, optimize); + _ = addTestStep(b, "test-comptime", "src/comptime.zig", &.{}, user_filter, target, optimize); + + // ── Pre-baked filtered shortcuts ── + // Each named step runs only tests whose name contains one of the + // listed substrings (Zig's --test-filter is OR-of-substrings). + // Compile is cached and shared with the parent module's step. + + _ = addTestStep(b, "test-parser-core", "src/parser.zig", &.{ "literal", "concat", "alternation" }, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-quant", "src/parser.zig", &.{"quantifier"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-class", "src/parser.zig", &.{"class"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-group", "src/parser.zig", &.{"group"}, user_filter, target, optimize); + _ = addTestStep(b, "test-parser-error", "src/parser.zig", &.{"errors"}, user_filter, target, optimize); + + _ = addTestStep(b, "test-nfa-basic", "src/nfa.zig", &.{ "literal", "concat" }, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-quant", "src/nfa.zig", &.{ "star", "plus", "question", "counted" }, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-alt", "src/nfa.zig", &.{"alt"}, user_filter, target, optimize); + _ = addTestStep(b, "test-nfa-group", "src/nfa.zig", &.{"group"}, user_filter, target, optimize); + + _ = addTestStep(b, "test-exec-basic", "src/exec.zig", &.{ "literal", "no match" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-quant", "src/exec.zig", &.{ "greedy", "lazy", "counted", "optional" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-class", "src/exec.zig", &.{ "class", "digit", "word" }, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-group", "src/exec.zig", &.{"group"}, user_filter, target, optimize); + _ = addTestStep(b, "test-exec-anchor", "src/exec.zig", &.{ "anchor", "boundary" }, user_filter, target, optimize); + + _ = addTestStep(b, "test-dfa-rejects", "src/dfa.zig", &.{"rejects"}, user_filter, target, optimize); + _ = addTestStep(b, "test-dfa-match", "src/dfa.zig", &.{ "literal", "plus", "alt", "class", "wildcard", "longest" }, user_filter, target, optimize); + + _ = addTestStep(b, "test-prefilter-full", "src/prefilter.zig", &.{"full literal"}, user_filter, target, optimize); + _ = addTestStep(b, "test-prefilter-required", "src/prefilter.zig", &.{"required literal"}, user_filter, target, optimize); + + // ───────────────────────────────────────────────────────────────── + // Aggregate sweeps — explicit, opt-in. Run these AFTER you're done + // iterating, not in the inner loop. + // ───────────────────────────────────────────────────────────────── + + const ast_all = addTestStep(b, "_test-ast-all", "src/ast.zig", &.{}, user_filter, target, optimize); + const parser_all = addTestStep(b, "_test-parser-all", "src/parser.zig", &.{}, user_filter, target, optimize); + const nfa_all = addTestStep(b, "_test-nfa-all", "src/nfa.zig", &.{}, user_filter, target, optimize); + const exec_all = addTestStep(b, "_test-exec-all", "src/exec.zig", &.{}, user_filter, target, optimize); + const prefilter_all = addTestStep(b, "_test-prefilter-all", "src/prefilter.zig", &.{}, user_filter, target, optimize); + const dfa_all = addTestStep(b, "_test-dfa-all", "src/dfa.zig", &.{}, user_filter, target, optimize); + const root_all = addTestStep(b, "_test-root-all", "src/root.zig", &.{}, user_filter, target, optimize); + const comptime_all = addTestStep(b, "_test-comptime-all", "src/comptime.zig", &.{}, user_filter, target, optimize); + + const test_all = b.step("test-all", "Run ALL unit tests across every module (slow — use named steps in the inner loop)"); + test_all.dependOn(ast_all); + test_all.dependOn(parser_all); + test_all.dependOn(nfa_all); + test_all.dependOn(exec_all); + test_all.dependOn(prefilter_all); + test_all.dependOn(dfa_all); + test_all.dependOn(root_all); + test_all.dependOn(comptime_all); + + // ── Parity vs Python re (separate; never auto-runs) ── + const parity_cmd = b.addSystemCommand(&.{"bash"}); + parity_cmd.addFileArg(b.path("tests/parity/run.sh")); + parity_cmd.addFileArg(probe.getEmittedBin()); + parity_cmd.addDirectoryArg(b.path("tests/parity/fixtures")); + const parity_step = b.step("parity", "Run Python-re parity tests (separate; opt-in)"); + parity_step.dependOn(&parity_cmd.step); +} + +/// Build one focused test step. `filters` is OR'd substring matching +/// (test runs iff its name contains at least one of the filters, or all +/// pass when the list is empty). The user-level -Dtest-filter is appended +/// so a named step can be narrowed further from the command line. +fn addTestStep( + b: *std.Build, + step_name: []const u8, + root_path: []const u8, + base_filters: []const []const u8, + user_filter: ?[]const u8, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +) *std.Build.Step { + var filter_list = std.ArrayList([]const u8).empty; + filter_list.appendSlice(b.allocator, base_filters) catch @panic("OOM"); + if (user_filter) |f| filter_list.append(b.allocator, f) catch @panic("OOM"); + + const test_mod = b.createModule(.{ + .root_source_file = b.path(root_path), + .target = target, + .optimize = optimize, + }); + const test_exe = b.addTest(.{ + .root_module = test_mod, + .filters = filter_list.toOwnedSlice(b.allocator) catch @panic("OOM"), + }); + const run_step = b.addRunArtifact(test_exe); + const step = b.step(step_name, b.fmt("Run tests in {s}", .{root_path})); + step.dependOn(&run_step.step); + return step; +} diff --git a/vendor/nanoregex/build.zig.zon b/vendor/nanoregex/build.zig.zon new file mode 100644 index 00000000..31960386 --- /dev/null +++ b/vendor/nanoregex/build.zig.zon @@ -0,0 +1,8 @@ +.{ + .name = .nanoregex, + .version = "0.0.1", + .fingerprint = 0xc8e46b5d7121d911, + .minimum_zig_version = "0.17.0-dev.813+2153f8143", + .dependencies = .{}, + .paths = .{""}, +} diff --git a/vendor/nanoregex/src/ast.zig b/vendor/nanoregex/src/ast.zig new file mode 100644 index 00000000..6740c8b1 --- /dev/null +++ b/vendor/nanoregex/src/ast.zig @@ -0,0 +1,186 @@ +//! Regex AST. Built by parser.zig, consumed by nfa.zig (later). +//! +//! All nodes are arena-owned by the parent Regex. The arena is freed in one +//! shot on Regex.deinit, so individual nodes never need their own deinit. +//! Child references use `*const Node` rather than slices to keep the union +//! tag size predictable and to avoid sub-allocations for unary nodes. + +const std = @import("std"); + +pub const Node = union(enum) { + /// A single literal byte. Case-folding (when flags.case_insensitive) is + /// handled at match time so the AST stays case-preserving — useful for + /// reporting in error messages. + literal: u8, + + /// `.` — matches any byte except `\n`, or any byte at all when the + /// pattern was compiled with `flags.dot_all`. The matcher consults the + /// flag; the AST node itself is flag-agnostic. + dot, + + /// `[abc]`, `[^abc]`, `[a-z]`, plus the shorthands `\d \D \w \W \s \S` + /// which the parser desugars into a Class node with the right bitmap. + class: *const Class, + + /// Zero-width anchors: `^ $ \b \B \A \z`. + anchor: Anchor, + + /// A concatenation `abc` — match each sub-node in order. + concat: []const *const Node, + + /// Alternation `a|b|c` — match any one branch (left to right, first + /// wins under leftmost-first semantics, matching Python re). + alt: []const *const Node, + + /// Quantified sub-pattern: `a*`, `a+`, `a?`, `a{n,m}`. + repeat: *const Repeat, + + /// `(foo)` or `(?:foo)`. The group's `index` is 0 for non-capturing, + /// 1..N for capturing groups in left-paren order. Index 0 (the whole + /// match) is implicit at the top level — not a Group node. + group: *const Group, +}; + +pub const Anchor = enum { + /// `^` — start of input, or start of any line in multiline mode. + line_start, + /// `$` — end of input, or end of any line in multiline mode. + line_end, + /// `\b` — boundary between a word char (`[A-Za-z0-9_]`) and not. + word_boundary, + /// `\B` — anywhere `\b` doesn't match. + non_word_boundary, + /// `\A` — start of input (ignores multiline flag). + string_start, + /// `\z` — end of input (ignores multiline flag). + string_end, +}; + +/// Character class represented as a 256-bit bitmap. `bitmap[byte/8] & +/// (1 << (byte%8))` is set iff the byte is included. Negation is folded +/// into the bitmap at parse time so the matcher does a single bit test. +pub const Class = struct { + bitmap: [32]u8, + + pub fn empty() Class { + return .{ .bitmap = @splat(0) }; + } + + pub fn set(self: *Class, byte: u8) void { + self.bitmap[byte / 8] |= @as(u8, 1) << @intCast(byte % 8); + } + + pub fn setRange(self: *Class, lo: u8, hi: u8) void { + var b: usize = lo; + while (b <= hi) : (b += 1) { + self.set(@intCast(b)); + if (b == 0xff) break; + } + } + + pub fn contains(self: *const Class, byte: u8) bool { + return (self.bitmap[byte / 8] >> @intCast(byte % 8)) & 1 != 0; + } + + pub fn negate(self: *Class) void { + for (&self.bitmap) |*b| b.* = ~b.*; + } +}; + +pub const Repeat = struct { + sub: *const Node, + min: u32, + /// `std.math.maxInt(u32)` represents unbounded (`*` and `+`). + max: u32, + /// True for `*`/`+`/`?`/`{n,m}`, false for the lazy `??`/`*?`/`+?`/`{n,m}?` + /// variants. Greedy is the Python re default. + greedy: bool, +}; + +pub const Group = struct { + sub: *const Node, + /// 0 = non-capturing; ≥1 = capture index in left-paren declaration order. + index: u32, + capturing: bool, +}; + +// ── Test helpers ── + +/// Pretty-print an AST for debugging and tests. Indents to make tree shape +/// visible. The format is stable enough to assert against in tests. +/// Pretty-print an AST for debugging and tests. Indents to make tree shape +/// visible. The format is stable enough to assert against in tests. +/// Writes into an ArrayList(u8) rather than a std.io.Writer because Zig 0.16 +/// reworked the writer interface and we don't want to chase the new shape +/// from a leaf debug helper. +pub fn debugWrite(node: *const Node, buf: *std.ArrayList(u8), alloc: std.mem.Allocator, indent: u32) error{OutOfMemory}!void { + var i: u32 = 0; + while (i < indent) : (i += 1) try buf.appendSlice(alloc, " "); + var tmp: [128]u8 = undefined; + switch (node.*) { + .literal => |c| { + const line = std.fmt.bufPrint(&tmp, "literal '{c}'\n", .{c}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .dot => try buf.appendSlice(alloc, "dot\n"), + .anchor => |a| { + const line = std.fmt.bufPrint(&tmp, "anchor {s}\n", .{@tagName(a)}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .class => |c| { + var popcnt: u32 = 0; + for (c.bitmap) |b| popcnt += @popCount(b); + const line = std.fmt.bufPrint(&tmp, "class [{d} bytes]\n", .{popcnt}) catch unreachable; + try buf.appendSlice(alloc, line); + }, + .concat => |children| { + try buf.appendSlice(alloc, "concat\n"); + for (children) |child| try debugWrite(child, buf, alloc, indent + 1); + }, + .alt => |children| { + try buf.appendSlice(alloc, "alt\n"); + for (children) |child| try debugWrite(child, buf, alloc, indent + 1); + }, + .repeat => |r| { + const line = std.fmt.bufPrint(&tmp, "repeat min={d} max={d} greedy={}\n", .{ r.min, r.max, r.greedy }) catch unreachable; + try buf.appendSlice(alloc, line); + try debugWrite(r.sub, buf, alloc, indent + 1); + }, + .group => |g| { + const line = std.fmt.bufPrint(&tmp, "group #{d} cap={}\n", .{ g.index, g.capturing }) catch unreachable; + try buf.appendSlice(alloc, line); + try debugWrite(g.sub, buf, alloc, indent + 1); + }, + } +} + +test "class bitmap set/contains" { + var c = Class.empty(); + c.set('a'); + c.set('z'); + try std.testing.expect(c.contains('a')); + try std.testing.expect(c.contains('z')); + try std.testing.expect(!c.contains('b')); + try std.testing.expect(!c.contains('y')); +} + +test "class range" { + var c = Class.empty(); + c.setRange('a', 'd'); + try std.testing.expect(c.contains('a')); + try std.testing.expect(c.contains('b')); + try std.testing.expect(c.contains('c')); + try std.testing.expect(c.contains('d')); + try std.testing.expect(!c.contains('e')); + try std.testing.expect(!c.contains('`')); +} + +test "class negate" { + var c = Class.empty(); + c.setRange('a', 'z'); + c.negate(); + try std.testing.expect(!c.contains('a')); + try std.testing.expect(!c.contains('z')); + try std.testing.expect(c.contains('A')); + try std.testing.expect(c.contains('0')); +} diff --git a/vendor/nanoregex/src/bench.zig b/vendor/nanoregex/src/bench.zig new file mode 100644 index 00000000..e9715b4f --- /dev/null +++ b/vendor/nanoregex/src/bench.zig @@ -0,0 +1,145 @@ +//! Single-file regex benchmark. +//! +//! Wire shape: nanoregex_bench [iters] +//! +//! Reads `path` into memory once, then runs `r.findAll` against it `iters` +//! times (default 20). Prints mean per-iteration time + match count so the +//! comparison script can diff us against python re and zig-regex on the +//! exact same input. +//! +//! We measure findAll only — not parse/compile — because the latter is a +//! one-shot cost the user pays once per CLI invocation, while findAll +//! dominates real workloads (walks across thousands of files). + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; + +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |a| try args_list.append(alloc, a); + const args = args_list.items; + if (args.len < 3) { + writeAll(2, "usage: nanoregex_bench [iters]\n"); + std.process.exit(2); + } + + const pattern = args[1]; + const path = args[2]; + const iters: usize = if (args.len >= 4) + std.fmt.parseInt(usize, args[3], 10) catch 20 + else + 20; + + // Read the whole file. Using libc fopen+fread to avoid the std.fs API + // churn in 0.16 — this binary is throwaway so the simplest path wins. + const data = readFile(alloc, path) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "read error: {s}\n", .{@errorName(err)}) catch "read error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer alloc.free(data); + + var r = nanoregex.Regex.compile(alloc, pattern) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "parse error: {s}\n", .{@errorName(err)}) catch "parse error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer r.deinit(); + + var total_ns: u128 = 0; + var match_count: usize = 0; + + // One untimed warm-up so JIT-like effects don't bias the first sample. + { + const ms = r.findAll(alloc, data) catch { + writeAll(2, "engine error during warm-up\n"); + std.process.exit(1); + }; + match_count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var iter: usize = 0; + while (iter < iters) : (iter += 1) { + const start_ns = nowNs(); + const ms = r.findAll(alloc, data) catch { + writeAll(2, "engine error in timed loop\n"); + std.process.exit(1); + }; + const end_ns = nowNs(); + total_ns += @intCast(end_ns - start_ns); + match_count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + const mean_ms: f64 = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)) / 1_000_000.0; + + var out_buf: [256]u8 = undefined; + const line = std.fmt.bufPrint(&out_buf, "nanoregex: matches={d} mean={d:.3}ms ({d}KB, {d} iters)\n", .{ + match_count, + mean_ms, + data.len / 1024, + iters, + }) catch return; + writeAll(1, line); +} + +const Timespec = extern struct { tv_sec: i64, tv_nsec: i64 }; +extern "c" fn clock_gettime(clk: c_int, ts: *Timespec) c_int; +const CLOCK_MONOTONIC: c_int = 6; + +fn nowNs() i128 { + var ts: Timespec = .{ .tv_sec = 0, .tv_nsec = 0 }; + _ = clock_gettime(CLOCK_MONOTONIC, &ts); + return @as(i128, ts.tv_sec) * 1_000_000_000 + ts.tv_nsec; +} + +extern "c" fn fopen(path: [*:0]const u8, mode: [*:0]const u8) ?*anyopaque; +extern "c" fn fclose(stream: *anyopaque) c_int; +extern "c" fn fread(ptr: [*]u8, size: usize, n: usize, stream: *anyopaque) usize; +extern "c" fn fseek(stream: *anyopaque, offset: c_long, whence: c_int) c_int; +extern "c" fn ftell(stream: *anyopaque) c_long; +const SEEK_END: c_int = 2; +const SEEK_SET: c_int = 0; + +fn readFile(alloc: std.mem.Allocator, path: []const u8) ![]u8 { + var path_buf: [4096]u8 = undefined; + if (path.len >= path_buf.len) return error.PathTooLong; + @memcpy(path_buf[0..path.len], path); + path_buf[path.len] = 0; + const path_z: [*:0]const u8 = @ptrCast(&path_buf); + + const f = fopen(path_z, "rb") orelse return error.OpenFailed; + defer _ = fclose(f); + + if (fseek(f, 0, SEEK_END) != 0) return error.SeekFailed; + const size_raw = ftell(f); + if (size_raw < 0) return error.SizeFailed; + const size: usize = @intCast(size_raw); + if (fseek(f, 0, SEEK_SET) != 0) return error.SeekFailed; + + const buf = try alloc.alloc(u8, size); + errdefer alloc.free(buf); + const n = fread(buf.ptr, 1, size, f); + if (n != size) return error.ReadShort; + return buf; +} diff --git a/vendor/nanoregex/src/bench_comptime.zig b/vendor/nanoregex/src/bench_comptime.zig new file mode 100644 index 00000000..15b2be1e --- /dev/null +++ b/vendor/nanoregex/src/bench_comptime.zig @@ -0,0 +1,248 @@ +//! bench_comptime.zig — head-to-head: comptime-specialised vs runtime Regex. +//! +//! Wire shape: nanoregex_bench_comptime [iters] +//! +//! Reads `path` once, then runs three fixed patterns through both code paths: +//! +//! 1. "hello" — pure-literal (comptime: needle constant; runtime: memmem) +//! 2. "\\d+" — single-class (comptime: .rodata table; runtime: dfa+table) +//! 3. "[a-z]+" — single-class (comptime: .rodata table; runtime: dfa+table) +//! +//! For each pattern we report: +//! comptime mean, runtime mean, and the ratio (runtime/comptime ≥ 1 = comptime wins). +//! +//! We also accept the bench file path so the same 142 KB fixture used for +//! parity tests can be passed in as the haystack. + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +const Timespec = extern struct { tv_sec: i64, tv_nsec: i64 }; +extern "c" fn clock_gettime(clk: c_int, ts: *Timespec) c_int; +const CLOCK_MONOTONIC: c_int = 6; +fn nowNs() i128 { + var ts: Timespec = .{ .tv_sec = 0, .tv_nsec = 0 }; + _ = clock_gettime(CLOCK_MONOTONIC, &ts); + return @as(i128, ts.tv_sec) * 1_000_000_000 + ts.tv_nsec; +} + +extern "c" fn fopen(path: [*:0]const u8, mode: [*:0]const u8) ?*anyopaque; +extern "c" fn fclose(stream: *anyopaque) c_int; +extern "c" fn fread(ptr: [*]u8, size: usize, n: usize, stream: *anyopaque) usize; +extern "c" fn fseek(stream: *anyopaque, offset: c_long, whence: c_int) c_int; +extern "c" fn ftell(stream: *anyopaque) c_long; +const SEEK_END: c_int = 2; +const SEEK_SET: c_int = 0; + +fn readFile(alloc: std.mem.Allocator, path: []const u8) ![]u8 { + var path_buf: [4096]u8 = undefined; + if (path.len >= path_buf.len) return error.PathTooLong; + @memcpy(path_buf[0..path.len], path); + path_buf[path.len] = 0; + const path_z: [*:0]const u8 = @ptrCast(&path_buf); + const f = fopen(path_z, "rb") orelse return error.OpenFailed; + defer _ = fclose(f); + if (fseek(f, 0, SEEK_END) != 0) return error.SeekFailed; + const size_raw = ftell(f); + if (size_raw < 0) return error.SizeFailed; + const size: usize = @intCast(size_raw); + if (fseek(f, 0, SEEK_SET) != 0) return error.SeekFailed; + const buf = try alloc.alloc(u8, size); + errdefer alloc.free(buf); + const n = fread(buf.ptr, 1, size, f); + if (n != size) return error.ReadShort; + return buf; +} + +/// Run `findAll` `iters` times and return (mean_ns, match_count). +fn benchRuntime(alloc: std.mem.Allocator, pattern: []const u8, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + var r = try nanoregex.Regex.compile(alloc, pattern); + defer r.deinit(); + + // Warm-up. + var count: usize = 0; + { + const ms = try r.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try r.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeLiteral(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("hello"); + + // Warm-up. + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeDigits(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("\\d+"); + + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +fn benchComptimeLower(alloc: std.mem.Allocator, data: []const u8, iters: usize) !struct { mean_ns: f64, count: usize } { + const cx = comptime nanoregex.compileComptime("[a-z]+"); + + var count: usize = 0; + { + const ms = try cx.findAll(alloc, data); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + + var total_ns: u128 = 0; + var i: usize = 0; + while (i < iters) : (i += 1) { + const t0 = nowNs(); + const ms = try cx.findAll(alloc, data); + const t1 = nowNs(); + total_ns += @intCast(t1 - t0); + count = ms.len; + for (ms) |*m| @constCast(m).deinit(alloc); + alloc.free(ms); + } + return .{ + .mean_ns = @as(f64, @floatFromInt(total_ns)) / @as(f64, @floatFromInt(iters)), + .count = count, + }; +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |a| try args_list.append(alloc, a); + const args = args_list.items; + + if (args.len < 2) { + writeAll(2, "usage: nanoregex_bench_comptime [iters]\n"); + std.process.exit(2); + } + + const path = args[1]; + const iters: usize = if (args.len >= 3) + std.fmt.parseInt(usize, args[2], 10) catch 50 + else + 50; + + const data = readFile(alloc, path) catch |err| { + var tmp: [256]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "read error: {s}\n", .{@errorName(err)}) catch "read error\n"; + writeAll(2, msg); + std.process.exit(1); + }; + defer alloc.free(data); + + var hdr: [128]u8 = undefined; + const hdr_line = std.fmt.bufPrint(&hdr, "nanoregex_bench_comptime file={s} size={d}KB iters={d}\n", .{ + path, data.len / 1024, iters, + }) catch "header\n"; + writeAll(1, hdr_line); + writeAll(1, "pattern comptime(µs) runtime(µs) ratio(rt/ct)\n"); + writeAll(1, "──────────────────────────────────────────────────────────\n"); + + // Pattern 1: pure literal "hello" + { + const ct = try benchComptimeLiteral(alloc, data, iters); + const rt = try benchRuntime(alloc, "hello", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"hello\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } + + // Pattern 2: \d+ + { + const ct = try benchComptimeDigits(alloc, data, iters); + const rt = try benchRuntime(alloc, "\\d+", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"\\d+\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } + + // Pattern 3: [a-z]+ + { + const ct = try benchComptimeLower(alloc, data, iters); + const rt = try benchRuntime(alloc, "[a-z]+", data, iters); + const ratio = rt.mean_ns / ct.mean_ns; + var line: [256]u8 = undefined; + const s = std.fmt.bufPrint(&line, "\"[a-z]+\" {d:>10.2} {d:>10.2} {d:.2}x (matches={d})\n", .{ ct.mean_ns / 1000.0, rt.mean_ns / 1000.0, ratio, ct.count }) catch ""; + writeAll(1, s); + } +} diff --git a/vendor/nanoregex/src/comptime.zig b/vendor/nanoregex/src/comptime.zig new file mode 100644 index 00000000..166a3c92 --- /dev/null +++ b/vendor/nanoregex/src/comptime.zig @@ -0,0 +1,546 @@ +//! comptime.zig — comptime-specialised regex matchers. +//! +//! `compileComptime(pattern)` returns a `ComptimeRegex(pattern)` whose +//! internals are fully resolved at compile time for two pattern shapes: +//! +//! 1. Pure literal — no metacharacters → tight `std.mem.indexOf` loop +//! with the needle baked as a `*const [N]u8` constant. +//! +//! 2. Single-class quantifier — `\d+`, `[a-z]+`, `\w{2,}`, etc. → a +//! 256-element `[256]bool` membership table baked into `.rodata` at +//! compile time; the hot loop is a single `table[b]` lookup per byte. +//! +//! Any other pattern shape falls back to the runtime `Regex.compile` path. +//! The fallback is wrapped in the same API surface so callers don't need to +//! branch. +//! +//! # Usage +//! +//! const cx = comptime nanoregex.compileComptime("\\d+"); +//! const matches = try cx.findAll(allocator, haystack); +//! defer { for (matches) |*m| @constCast(m).deinit(allocator); allocator.free(matches); } +//! +//! Note: `cx` is a value type (no heap, no deinit needed). The `findAll` and +//! `search` methods do allocate for the returned Match slices. + +const std = @import("std"); +const root = @import("root.zig"); + +pub const Span = root.Span; +pub const Match = root.Match; + +// ── Pattern classification (all comptime) ────────────────────────────────── + +/// Classify a pattern string into one of three shapes that we can +/// specialize at compile time. +const PatternKind = enum { + /// Pattern has no metacharacters — every byte is literal. + pure_literal, + /// Pattern is exactly `CLASS+` or `CLASS{n,}` where CLASS is one of: + /// `\d`, `\D`, `\w`, `\W`, `\s`, `\S`, or `[...]`. + /// We bake the 256-bool membership table into `.rodata`. + single_class_plus, + /// Everything else — delegate to the runtime engine. + runtime_fallback, +}; + +/// Returns true iff `c` is a regex metacharacter that may alter meaning. +fn isMeta(c: u8) bool { + return switch (c) { + '.', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '^', '$', '\\' => true, + else => false, + }; +} + +/// Classify `pattern` at comptime. +fn classify(comptime pattern: []const u8) PatternKind { + if (pattern.len == 0) return .pure_literal; + + // Check for pure literal: no metacharacters at all. + var has_meta = false; + for (pattern) |c| { + if (isMeta(c)) { + has_meta = true; + break; + } + } + if (!has_meta) return .pure_literal; + + // Check for single-class-plus: CLASS+ or CLASS{n,} + // CLASS is one of the shorthand escapes \d \D \w \W \s \S, or a + // bracket expression [...]. We do a lightweight hand-rolled parse + // here (no allocator, comptime-safe). + return classifyAsSingleClass(pattern); +} + +/// Lightweight comptime check: is `pattern` exactly `CLASS+` or `CLASS{n,}`? +fn classifyAsSingleClass(comptime pattern: []const u8) PatternKind { + if (pattern.len < 2) return .runtime_fallback; + + // Unwrap optional non-capturing group `(?:...)` at the very outside. + const inner: []const u8 = blk: { + if (pattern.len > 4 and + pattern[0] == '(' and pattern[1] == '?' and pattern[2] == ':' and + pattern[pattern.len - 1] == ')') + { + break :blk pattern[3 .. pattern.len - 1]; + } + break :blk pattern; + }; + + // Determine the class span and the quantifier that follows it. + var class_end: usize = 0; + if (inner[0] == '\\') { + // Shorthand escape: \d \D \w \W \s \S + if (inner.len < 2) return .runtime_fallback; + switch (inner[1]) { + 'd', 'D', 'w', 'W', 's', 'S' => class_end = 2, + else => return .runtime_fallback, + } + } else if (inner[0] == '[') { + // Bracket expression — scan forward for the closing `]`. + // Handle `[^...]` negation and `[]...]` (literal `]` first). + var i: usize = 1; + if (i < inner.len and inner[i] == '^') i += 1; + if (i < inner.len and inner[i] == ']') i += 1; // literal `]` at start + while (i < inner.len and inner[i] != ']') i += 1; + if (i >= inner.len) return .runtime_fallback; + class_end = i + 1; // include the `]` + } else { + return .runtime_fallback; + } + + // What follows the class? + const quant = inner[class_end..]; + // Accept: `+`, `{n,}`, `{n,m}` where n >= 1 + if (quant.len == 0) return .runtime_fallback; + if (quant[0] == '+') { + if (quant.len == 1) return .single_class_plus; + return .runtime_fallback; + } + if (quant[0] == '{') { + // Minimal parse: must have a digit, then `,`, optionally digits, then `}`. + var i: usize = 1; + if (i >= quant.len or quant[i] < '0' or quant[i] > '9') return .runtime_fallback; + // Read the min value; reject min == 0. + var min_val: usize = 0; + while (i < quant.len and quant[i] >= '0' and quant[i] <= '9') { + min_val = min_val * 10 + (quant[i] - '0'); + i += 1; + } + if (min_val == 0) return .runtime_fallback; + if (i >= quant.len or quant[i] != ',') return .runtime_fallback; + i += 1; // skip ',' + // Optional upper bound digits. + while (i < quant.len and quant[i] >= '0' and quant[i] <= '9') i += 1; + if (i >= quant.len or quant[i] != '}') return .runtime_fallback; + if (i + 1 == quant.len) return .single_class_plus; + return .runtime_fallback; + } + return .runtime_fallback; +} + +// ── Comptime membership table generation ────────────────────────────────── + +/// Build a `[256]bool` membership table for the CLASS at the front of +/// `pattern` (same syntax as `classifyAsSingleClass` recognises). +/// Called at comptime so the array lives in `.rodata`. +fn buildClassTable(comptime pattern: []const u8) [256]bool { + // Strip outer (?:...) group if present. + const inner: []const u8 = blk: { + if (pattern.len > 4 and + pattern[0] == '(' and pattern[1] == '?' and pattern[2] == ':' and + pattern[pattern.len - 1] == ')') + { + break :blk pattern[3 .. pattern.len - 1]; + } + break :blk pattern; + }; + + var table: [256]bool = @splat(false); + + if (inner[0] == '\\') { + // Shorthand escape. + switch (inner[1]) { + 'd' => { + var b: usize = '0'; + while (b <= '9') : (b += 1) table[b] = true; + }, + 'D' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + if (b < '0' or b > '9') table[b] = true; + } + }, + 'w' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_') table[b] = true; + } + }, + 'W' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if (!((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_')) table[b] = true; + } + }, + 's' => { + for ([_]u8{ ' ', '\t', '\n', '\r', 0x0C, 0x0B }) |c| table[c] = true; + }, + 'S' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if (c != ' ' and c != '\t' and c != '\n' and c != '\r' and + c != 0x0C and c != 0x0B) table[b] = true; + } + }, + else => {}, + } + return table; + } + + // Bracket expression `[...]` or `[^...]`. + std.debug.assert(inner[0] == '['); + var i: usize = 1; + var negate = false; + if (i < inner.len and inner[i] == '^') { + negate = true; + i += 1; + } + + // Handle literal `]` as first char inside brackets. + if (i < inner.len and inner[i] == ']') { + table[']'] = true; + i += 1; + } + + while (i < inner.len and inner[i] != ']') { + if (i + 2 < inner.len and inner[i + 1] == '-' and inner[i + 2] != ']') { + // Range a-z. + var lo: usize = inner[i]; + const hi: usize = inner[i + 2]; + while (lo <= hi) : (lo += 1) table[lo] = true; + i += 3; + } else if (inner[i] == '\\' and i + 1 < inner.len) { + // Escape inside bracket. + switch (inner[i + 1]) { + 'd' => { + var b: usize = '0'; + while (b <= '9') : (b += 1) table[b] = true; + }, + 'w' => { + var b: usize = 0; + while (b < 256) : (b += 1) { + const c: u8 = @intCast(b); + if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or c == '_') table[b] = true; + } + }, + 's' => { + for ([_]u8{ ' ', '\t', '\n', '\r', 0x0C, 0x0B }) |c| table[c] = true; + }, + 'n' => table['\n'] = true, + 't' => table['\t'] = true, + 'r' => table['\r'] = true, + else => table[inner[i + 1]] = true, + } + i += 2; + } else { + table[inner[i]] = true; + i += 1; + } + } + + if (negate) { + for (&table) |*b| b.* = !b.*; + } + return table; +} + +// ── Core scanner functions ───────────────────────────────────────────────── + +/// Find the first match using a compile-time-known literal needle. +fn literalFirst(comptime needle: []const u8, alloc: std.mem.Allocator, haystack: []const u8) !?Match { + if (needle.len == 0) { + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = 0, .end = 0 }; + return .{ .span = .{ .start = 0, .end = 0 }, .captures = caps }; + } + const idx = std.mem.indexOf(u8, haystack, needle) orelse return null; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = idx, .end = idx + needle.len }; + return .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = caps }; +} + +/// Find all matches using a compile-time-known literal needle. +fn literalAll(comptime needle: []const u8, alloc: std.mem.Allocator, haystack: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + if (needle.len == 0) return try results.toOwnedSlice(alloc); + var pos: usize = 0; + while (pos <= haystack.len) { + const idx = std.mem.indexOfPos(u8, haystack, pos, needle) orelse break; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = idx, .end = idx + needle.len }; + try results.append(alloc, .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = caps }); + pos = idx + needle.len; + if (needle.len == 0) break; // guard against empty-needle infinite loop + } + return try results.toOwnedSlice(alloc); +} + +/// Find the first match using a compile-time membership table. +fn classFirst(comptime table: *const [256]bool, alloc: std.mem.Allocator, input: []const u8) !?Match { + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + return .{ .span = .{ .start = start, .end = i }, .captures = caps }; + } + i += 1; + } + return null; +} + +/// Find all matches using a compile-time membership table. +fn classAll(comptime table: *const [256]bool, alloc: std.mem.Allocator, input: []const u8) ![]Match { + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + try out.append(alloc, .{ .span = .{ .start = start, .end = i }, .captures = caps }); + } else { + i += 1; + } + } + return try out.toOwnedSlice(alloc); +} + +// ── Public API ───────────────────────────────────────────────────────────── + +/// A pattern-specialised matcher. The concrete type (and its `search` / +/// `findAll` implementation) are chosen at compile time based on the +/// pattern's shape. Callers never call `deinit` on the `ComptimeRegex` +/// value itself — there's nothing to free. +pub fn ComptimeRegex(comptime pattern: []const u8) type { + const kind = classify(pattern); + return switch (kind) { + .pure_literal => struct { + const needle: []const u8 = pattern; + + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + return literalFirst(needle, alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + return literalAll(needle, alloc, input); + } + }, + .single_class_plus => struct { + const table: [256]bool = buildClassTable(pattern); + + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + return classFirst(&table, alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + return classAll(&table, alloc, input); + } + }, + .runtime_fallback => struct { + // Thin wrapper: we call Regex.compile at first use (lazy) or + // callers can use this struct as a value and call the runtime path. + // + // Because we can't store a Regex by value here without an + // allocator, the runtime fallback re-compiles on each call. This + // is intentional: the comptime path is only meant for patterns + // that ARE pure-literal or single-class. For everything else the + // normal Regex API should be used. The fallback here exists only + // so callers don't need to branch. + pub fn search(self: @This(), alloc: std.mem.Allocator, input: []const u8) !?Match { + _ = self; + var r = try root.Regex.compile(alloc, pattern); + defer r.deinit(); + return r.search(alloc, input); + } + pub fn findAll(self: @This(), alloc: std.mem.Allocator, input: []const u8) ![]Match { + _ = self; + var r = try root.Regex.compile(alloc, pattern); + defer r.deinit(); + return r.findAll(alloc, input); + } + }, + }; +} + +/// Construct a `ComptimeRegex(pattern)` value. Call with a comptime-known +/// string literal; the compiler constant-folds the entire specialisation. +/// +/// const cx = comptime compileComptime("hello"); +/// const m = try cx.findAll(allocator, "say hello world"); +pub fn compileComptime(comptime pattern: []const u8) ComptimeRegex(pattern) { + return .{}; +} + +// ── Tests ────────────────────────────────────────────────────────────────── + +test "comptime: pure-literal findAll matches runtime path" { + const cx = comptime compileComptime("hello"); + const haystack = "say hello world, hello again"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + // Runtime reference + var r = try root.Regex.compile(std.testing.allocator, "hello"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: pure-literal search first match" { + const cx = comptime compileComptime("fox"); + var m = (try cx.search(std.testing.allocator, "the quick brown fox jumps")).?; + defer m.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 16), m.span.start); + try std.testing.expectEqual(@as(usize, 19), m.span.end); +} + +test "comptime: pure-literal no match returns null" { + const cx = comptime compileComptime("xyz"); + const result = try cx.search(std.testing.allocator, "hello world"); + try std.testing.expect(result == null); +} + +test "comptime: \\d+ class findAll matches runtime path" { + const cx = comptime compileComptime("\\d+"); + const haystack = "abc 42 def 1234 xyz"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + var r = try root.Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: [a-z]+ class findAll matches runtime path" { + const cx = comptime compileComptime("[a-z]+"); + const haystack = "Hello World foo BAR baz"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + var r = try root.Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + const rt_ms = try r.findAll(std.testing.allocator, haystack); + defer { + for (rt_ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(rt_ms); + } + try std.testing.expectEqual(rt_ms.len, ms.len); + for (ms, rt_ms) |m, rt| { + try std.testing.expectEqual(rt.span.start, m.span.start); + try std.testing.expectEqual(rt.span.end, m.span.end); + } +} + +test "comptime: [A-Z]+ class (uppercase only)" { + const cx = comptime compileComptime("[A-Z]+"); + const haystack = "Hello WORLD foo BAR"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); + try std.testing.expectEqualStrings("H", haystack[ms[0].span.start..ms[0].span.end]); + try std.testing.expectEqualStrings("WORLD", haystack[ms[1].span.start..ms[1].span.end]); + try std.testing.expectEqualStrings("BAR", haystack[ms[2].span.start..ms[2].span.end]); +} + +test "comptime: \\w+ word class" { + const cx = comptime compileComptime("\\w+"); + const haystack = "foo bar_baz 123"; + const ms = try cx.findAll(std.testing.allocator, haystack); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "comptime: runtime fallback compiles and runs" { + // `(abc)+` is a compound pattern — falls back to runtime. + const cx = comptime compileComptime("(abc)+"); + const ms = try cx.findAll(std.testing.allocator, "abcabc def abc"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + // Should find matches regardless of code path. + try std.testing.expect(ms.len >= 1); +} + +test "comptime: classify pure_literal" { + try std.testing.expectEqual(.pure_literal, comptime classify("hello")); + try std.testing.expectEqual(.pure_literal, comptime classify("compileAllocFlags")); + try std.testing.expectEqual(.pure_literal, comptime classify("")); +} + +test "comptime: classify single_class_plus" { + try std.testing.expectEqual(.single_class_plus, comptime classify("\\d+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("[a-z]+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("\\w+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("\\s+")); + try std.testing.expectEqual(.single_class_plus, comptime classify("[0-9]{3,}")); + try std.testing.expectEqual(.single_class_plus, comptime classify("(?:\\d+)")); +} + +test "comptime: classify runtime_fallback" { + try std.testing.expectEqual(.runtime_fallback, comptime classify("(abc)+")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("a.b")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("foo|bar")); + try std.testing.expectEqual(.runtime_fallback, comptime classify("\\d*")); // min=0 skipped +} diff --git a/vendor/nanoregex/src/dfa.zig b/vendor/nanoregex/src/dfa.zig new file mode 100644 index 00000000..29193a80 --- /dev/null +++ b/vendor/nanoregex/src/dfa.zig @@ -0,0 +1,453 @@ +//! Lazy DFA built by subset construction over the Thompson NFA. +//! +//! Each DFA state is a sorted set of NFA state IDs. Transitions are computed +//! on demand: when (state, byte) is first encountered we run `step`, hash +//! the resulting NFA-state set, look it up (or create a new DFA state for +//! it), and cache the edge. Subsequent bytes through the same transition +//! are a single indexed lookup. +//! +//! Scope for v1: +//! - No capture-group tracking (caller must check `nfa.n_groups == 0`). +//! - No anchors (caller must check the AST for `Anchor` nodes; if present, +//! fall back to the Pike VM). +//! - Bounded state count (MAX_STATES). On overflow we surface an error +//! and the caller falls back to the Pike VM. +//! +//! The forward-only driver below handles unanchored search by trying each +//! starting position. Per-character cost is one table lookup, so this is +//! O(n) for cases where matches don't overlap heavily. + +const std = @import("std"); +const ast = @import("ast.zig"); +const nfa = @import("nfa.zig"); +const minterm = @import("minterm.zig"); + +pub const DfaStateId = u32; +pub const DEAD: DfaStateId = std.math.maxInt(DfaStateId); +const UNCOMPUTED: DfaStateId = std.math.maxInt(DfaStateId) - 1; + +const MAX_STATES: u32 = 4096; + +pub const Error = error{ OutOfMemory, TooManyStates, HasCaptures, HasAnchors, HasLazyQuantifier }; + +/// Knobs the runtime needs the DFA to bake in at construction time — +/// flags whose meaning isn't visible from the NFA alone. (case_insensitive +/// is intentionally absent: v1 falls back to the Pike VM when CI is set.) +pub const BuildOptions = struct { + /// Forwarded from the public Flags.dot_all. When true, `.` matches `\n`. + dot_all: bool = false, +}; + +/// Set of NFA states reached after a particular byte sequence. Sorted, +/// deduplicated; the byte representation is the hash-map key. +const NfaSet = struct { + ids: []const nfa.StateId, + accepts: bool, +}; + +/// Hash-map context that compares NfaSet identity by content of `ids`. +/// We key on the raw byte slice of the sorted ids — same length, same +/// bytes, same set. +const SetMapCtx = struct { + pub fn hash(_: SetMapCtx, key: []const u8) u64 { + return std.hash.Wyhash.hash(0, key); + } + pub fn eql(_: SetMapCtx, a: []const u8, b: []const u8) bool { + if (a.len != b.len) return false; + return std.mem.eql(u8, a, b); + } +}; + +const SetMap = std.HashMap([]const u8, DfaStateId, SetMapCtx, std.hash_map.default_max_load_percentage); + +pub const Dfa = struct { + /// Arena for state sets and transition rows. Lives until deinit. + arena: *std.heap.ArenaAllocator, + parent_alloc: std.mem.Allocator, + nfa_ref: *const nfa.Nfa, + + states: std.ArrayList(NfaSet), + /// Flat 2-D transition table indexed by `state * minterm.n_classes + class_id`. + /// Far smaller than 256-per-row when the pattern's atomic predicates + /// partition the alphabet into a handful of equivalence classes — + /// `[a-z]+` ends up with 2 classes, a 128× shrink. + transitions: []DfaStateId, + /// Resolves byte → class id. Built once from the AST at compile time. + minterm: minterm.Table, + set_to_id: SetMap, + + start: DfaStateId, + /// Whether `.` should match `\n`. Threaded in from the public Flags + /// at compile time so the inner-loop test stays branch-cheap. + dot_all: bool, + + pub fn fromNfa(alloc: std.mem.Allocator, n: *const nfa.Nfa, root: *const ast.Node, opts: BuildOptions) Error!Dfa { + if (n.n_groups != 0) return Error.HasCaptures; + if (containsAnchor(root)) return Error.HasAnchors; + // Lazy quantifiers need leftmost-shortest semantics, which a + // plain subset-construction DFA cannot express — it always picks + // leftmost-longest. Bail and let the Pike VM handle the pattern. + if (containsLazy(root)) return Error.HasLazyQuantifier; + + const arena = try alloc.create(std.heap.ArenaAllocator); + errdefer alloc.destroy(arena); + arena.* = std.heap.ArenaAllocator.init(alloc); + errdefer arena.deinit(); + const aa = arena.allocator(); + + // Compute byte equivalence classes from the pattern so the + // transition table can be indexed by class instead of raw byte. + // Typical pattern → 4-20 classes, so the row shrinks 12-64× and + // fits in L1 instead of L2. + const mt = try minterm.build(aa, root, opts.dot_all); + + const transitions = try aa.alloc(DfaStateId, @as(usize, MAX_STATES) * mt.n_classes); + @memset(transitions, UNCOMPUTED); + + var dfa: Dfa = .{ + .arena = arena, + .parent_alloc = alloc, + .nfa_ref = n, + .states = .empty, + .transitions = transitions, + .minterm = mt, + .set_to_id = SetMap.init(aa), + .start = 0, + .dot_all = opts.dot_all, + }; + + // Seed: the start DFA state is the epsilon-closure of {nfa.start}. + const seed = try epsilonClosure(aa, n, &.{n.start}); + dfa.start = try dfa.internState(seed); + + return dfa; + } + + pub fn deinit(self: *Dfa) void { + self.arena.deinit(); + self.parent_alloc.destroy(self.arena); + self.* = undefined; + } + + /// Insert a state set into the DFA, returning either a fresh id or the + /// existing one. Sorts the input slice in place before hashing. + fn internState(self: *Dfa, ids: []const nfa.StateId) Error!DfaStateId { + // Sort + dedupe — caller is allowed to pass an unsorted set. + const dup = try self.arena.allocator().dupe(nfa.StateId, ids); + std.mem.sort(nfa.StateId, dup, {}, comptime std.sort.asc(nfa.StateId)); + const deduped = uniqueSorted(dup); + + const key_bytes = std.mem.sliceAsBytes(deduped); + if (self.set_to_id.get(key_bytes)) |existing| return existing; + + if (self.states.items.len >= MAX_STATES) return Error.TooManyStates; + + var accepts = false; + for (deduped) |id| if (id == self.nfa_ref.accept) { + accepts = true; + break; + }; + + const id: DfaStateId = @intCast(self.states.items.len); + try self.states.append(self.arena.allocator(), .{ .ids = deduped, .accepts = accepts }); + try self.set_to_id.put(key_bytes, id); + return id; + } + + /// Public wrapper around the inlined hot-path transition. Mostly used + /// by tests; the matching loop in `matchAt` reads `transitions` and + /// `byte_to_class` directly to skip the function-call overhead. + pub fn transition(self: *Dfa, state: DfaStateId, byte: u8) Error!DfaStateId { + const class_id: usize = self.minterm.byte_to_class[byte]; + const idx = @as(usize, state) * self.minterm.n_classes + class_id; + const cached = self.transitions[idx]; + if (cached != UNCOMPUTED) return cached; + return try self.computeAndCacheTransition(state, class_id); + } + + /// Slow path: compute the successor set for `(state, class_id)`, intern + /// it as a new DFA state if needed, and cache the edge. Called from the + /// hot loops only when the transition is missing. + fn computeAndCacheTransition(self: *Dfa, state: DfaStateId, class_id: usize) Error!DfaStateId { + const idx = @as(usize, state) * self.minterm.n_classes + class_id; + const rep_byte = self.minterm.representatives[class_id]; + + const cur = self.states.items[state]; + var next_ids: std.ArrayList(nfa.StateId) = .empty; + defer next_ids.deinit(self.arena.allocator()); + + for (cur.ids) |sid| { + const ns = self.nfa_ref.states[sid]; + const matched = switch (ns.consume) { + .byte => |b| b == rep_byte, + .any => self.dot_all or rep_byte != '\n', + .class => |cls| cls.contains(rep_byte), + .epsilon, .anchor, .group_start, .group_end => false, + }; + if (matched) { + if (ns.out1) |o| try next_ids.append(self.arena.allocator(), o); + } + } + + if (next_ids.items.len == 0) { + self.transitions[idx] = DEAD; + return DEAD; + } + + const closure = try epsilonClosure(self.arena.allocator(), self.nfa_ref, next_ids.items); + const next_id = try self.internState(closure); + self.transitions[idx] = next_id; + return next_id; + } + + /// Anchored match starting at `start` in `input`. Returns the end index + /// of the longest accepted run, or null if no match. + /// + /// The hot loop reads the byte → class table and the transition table + /// directly. The slow-path branch (`UNCOMPUTED`) is hoisted out so the + /// fast path is a tight series of array reads + one compare. After + /// warmup the slow path is essentially never taken, so this trades + /// one predicted-not-taken branch for skipping a function frame and + /// the redundant idx recomputation that the older `transition` call + /// did inside the loop. + pub fn matchAt(self: *Dfa, input: []const u8, start: usize) Error!?usize { + var cur: DfaStateId = self.start; + var longest: ?usize = if (self.states.items[cur].accepts) start else null; + const byte_to_class = &self.minterm.byte_to_class; + const n_classes: usize = self.minterm.n_classes; + const transitions = self.transitions; + + var i: usize = start; + while (i < input.len) : (i += 1) { + const class_id: usize = byte_to_class[input[i]]; + const idx = @as(usize, cur) * n_classes + class_id; + var next = transitions[idx]; + if (next == UNCOMPUTED) { + next = try self.computeAndCacheTransition(cur, class_id); + } + if (next == DEAD) break; + cur = next; + if (self.states.items[cur].accepts) longest = i + 1; + } + return longest; + } + + /// Find every non-overlapping match span in `input`. Tries each + /// starting position; on a hit, skips past the match end. Zero-width + /// matches advance one byte so we don't loop. + pub fn findAll(self: *Dfa, alloc: std.mem.Allocator, input: []const u8) Error![]Span { + var out: std.ArrayList(Span) = .empty; + errdefer out.deinit(alloc); + + var p: usize = 0; + while (p <= input.len) { + const end_opt = try self.matchAt(input, p); + if (end_opt) |end| { + try out.append(alloc, .{ .start = p, .end = end }); + p = if (end > p) end else p + 1; + } else { + p += 1; + } + } + return try out.toOwnedSlice(alloc); + } +}; + +pub const Span = struct { start: usize, end: usize }; + +// ── Helpers ── + +/// Compute the epsilon-closure of `seeds`: every NFA state reachable from +/// the seeds via zero-width transitions (epsilon, group_start, group_end — +/// not anchor, since we caller-fail when anchors are present). +fn epsilonClosure(alloc: std.mem.Allocator, n: *const nfa.Nfa, seeds: []const nfa.StateId) Error![]nfa.StateId { + var stack: std.ArrayList(nfa.StateId) = .empty; + defer stack.deinit(alloc); + var seen = try alloc.alloc(bool, n.states.len); + defer alloc.free(seen); + @memset(seen, false); + + var out: std.ArrayList(nfa.StateId) = .empty; + errdefer out.deinit(alloc); + + for (seeds) |s| { + if (!seen[s]) { + seen[s] = true; + try stack.append(alloc, s); + } + } + + while (stack.pop()) |sid| { + try out.append(alloc, sid); + if (sid == n.accept) continue; + const ns = n.states[sid]; + switch (ns.consume) { + .epsilon, .group_start, .group_end => { + if (ns.out1) |o| if (!seen[o]) { + seen[o] = true; + try stack.append(alloc, o); + }; + if (ns.out2) |o| if (!seen[o]) { + seen[o] = true; + try stack.append(alloc, o); + }; + }, + // Consuming and anchor states don't contribute to the closure — + // they're already terminal for this iteration. + else => {}, + } + } + + return try out.toOwnedSlice(alloc); +} + +fn uniqueSorted(sorted: []nfa.StateId) []nfa.StateId { + if (sorted.len == 0) return sorted; + var w: usize = 1; + var i: usize = 1; + while (i < sorted.len) : (i += 1) { + if (sorted[i] != sorted[w - 1]) { + sorted[w] = sorted[i]; + w += 1; + } + } + return sorted[0..w]; +} + +fn containsAnchor(node: *const ast.Node) bool { + return switch (node.*) { + .anchor => true, + .literal, .dot, .class => false, + .concat => |children| for (children) |c| { + if (containsAnchor(c)) break true; + } else false, + .alt => |children| for (children) |c| { + if (containsAnchor(c)) break true; + } else false, + .repeat => |r| containsAnchor(r.sub), + .group => |g| containsAnchor(g.sub), + }; +} + +/// True iff any quantifier in the AST is lazy (`*?`/`+?`/`??`/`{n,m}?`). +/// The DFA always yields leftmost-longest matches, which contradicts +/// lazy semantics — we caller-fail so the Pike VM runs instead. +fn containsLazy(node: *const ast.Node) bool { + return switch (node.*) { + .literal, .dot, .class, .anchor => false, + .concat => |children| for (children) |c| { + if (containsLazy(c)) break true; + } else false, + .alt => |children| for (children) |c| { + if (containsLazy(c)) break true; + } else false, + .repeat => |r| !r.greedy or containsLazy(r.sub), + .group => |g| containsLazy(g.sub), + }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildDfa(arena: *std.heap.ArenaAllocator, pattern: []const u8) !Dfa { + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + // Heap-allocate the Nfa on the arena so its address is stable for the + // returned Dfa's `nfa_ref`. An earlier version did `&local_const` + // which was a dangling stack pointer the moment buildDfa returned, + // and every test that actually exercised the DFA either crashed or + // returned bogus values from torn-over stack memory. + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + return try Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{}); +} + +test "dfa: literal pattern matches via findAll" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "abc"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "the abc and abc again"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); + try std.testing.expectEqual(@as(usize, 4), spans[0].start); + try std.testing.expectEqual(@as(usize, 7), spans[0].end); +} + +test "dfa: greedy plus consumes longest run" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "\\d+"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "abc 42 def 1234 xyz"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); + // "42" — two digits. + try std.testing.expectEqual(@as(usize, 2), spans[0].end - spans[0].start); + // "1234" — four digits. The earlier expectation of `6` was a typo. + try std.testing.expectEqual(@as(usize, 4), spans[1].end - spans[1].start); +} + +test "dfa: alternation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "cat|dog|bird"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "the cat saw a dog and a bird"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 3), spans.len); +} + +test "dfa: class quantifier" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "[a-z]+"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "Hello World"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 2), spans.len); +} + +test "dfa: rejects capture patterns" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "(abc)"); + const root = try p.parseRoot(); + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + try std.testing.expectError(Error.HasCaptures, Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{})); +} + +test "dfa: rejects anchor patterns" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "^foo"); + const root = try p.parseRoot(); + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + try std.testing.expectError(Error.HasAnchors, Dfa.fromNfa(std.testing.allocator, automaton_ptr, root, .{})); +} + +test "dfa: dot wildcard" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "a.b"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "axb ayb azb"); + defer std.testing.allocator.free(spans); + try std.testing.expectEqual(@as(usize, 3), spans.len); +} + +test "dfa: longest match wins on greedy" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var dfa = try buildDfa(&arena, "a*"); + defer dfa.deinit(); + const spans = try dfa.findAll(std.testing.allocator, "aaa"); + defer std.testing.allocator.free(spans); + // a* should match "aaa" once at position 0, then zero-width at position 3. + try std.testing.expect(spans.len >= 1); + try std.testing.expectEqual(@as(usize, 0), spans[0].start); + try std.testing.expectEqual(@as(usize, 3), spans[0].end); +} diff --git a/vendor/nanoregex/src/exec.zig b/vendor/nanoregex/src/exec.zig new file mode 100644 index 00000000..0a2f6b60 --- /dev/null +++ b/vendor/nanoregex/src/exec.zig @@ -0,0 +1,469 @@ +//! Pike-VM NFA simulator. +//! +//! Classic two-list simulation: at each input position we hold a `clist` of +//! threads parked at consuming states, then step them on input[pos] into a +//! `nlist`. Zero-width states (epsilon, anchor, group_*) are walked inside +//! `addThread` so they never appear in the active lists — only consuming +//! states (byte / any / class) and the accept state do. +//! +//! Leftmost-first semantics: threads are added by DFS following out1 before +//! out2, so the priority order matches the AST's left-to-right reading. +//! Inside one input position the first thread to reach `accept` wins; lower- +//! priority threads at the same position are stopped from advancing. +//! +//! Captures are per-thread arrays of `?Span`. When a thread crosses a +//! group_start / group_end state, we dupe the array first so siblings +//! don't see each other's updates. This is the simplest correct shape; +//! a copy-on-write or generation-tagged store is the obvious v2 win. + +const std = @import("std"); +const ast = @import("ast.zig"); +const nfa = @import("nfa.zig"); + +pub const Flags = struct { + case_insensitive: bool = false, + /// `^` / `$` match line boundaries (default). When false they only match + /// input start / end. + multiline: bool = true, + /// `.` matches `\n`. Default false. + dot_all: bool = false, +}; + +pub const Span = struct { start: usize, end: usize }; + +pub const MatchResult = struct { + span: Span, + /// Slot 0 is the whole match (equal to `span`). Slots 1..n are capture + /// groups by declaration order. A null entry means the group didn't + /// participate in the match. + captures: []const ?Span, + + pub fn deinit(self: *MatchResult, alloc: std.mem.Allocator) void { + alloc.free(self.captures); + self.* = undefined; + } +}; + +const Thread = struct { + pc: nfa.StateId, + captures: []?Span, +}; + +/// Sparse-set thread list. Generation counter avoids per-step clears. +const ThreadList = struct { + threads: std.ArrayList(Thread), + seen_gen: []u32, + cur_gen: u32, + + fn init(alloc: std.mem.Allocator, n_states: usize) !ThreadList { + const seen = try alloc.alloc(u32, n_states); + @memset(seen, 0); + return .{ + .threads = .empty, + .seen_gen = seen, + .cur_gen = 1, + }; + } + + fn deinit(self: *ThreadList, alloc: std.mem.Allocator) void { + self.threads.deinit(alloc); + alloc.free(self.seen_gen); + } + + /// True iff the state was not yet in this generation. Marks it seen. + fn markIfNew(self: *ThreadList, pc: nfa.StateId) bool { + if (self.seen_gen[pc] == self.cur_gen) return false; + self.seen_gen[pc] = self.cur_gen; + return true; + } + + fn clear(self: *ThreadList, alloc: std.mem.Allocator) void { + self.cur_gen += 1; + self.threads.clearRetainingCapacity(); + _ = alloc; + } +}; + +pub const ExecError = error{OutOfMemory}; + +pub const Vm = struct { + alloc: std.mem.Allocator, + automaton: *const nfa.Nfa, + flags: Flags, + /// Total capture-array length: index 0 = whole match, 1..n_groups = explicit. + cap_len: usize, + + pub fn init(alloc: std.mem.Allocator, automaton: *const nfa.Nfa, flags: Flags) Vm { + return .{ + .alloc = alloc, + .automaton = automaton, + .flags = flags, + .cap_len = @as(usize, automaton.n_groups) + 1, + }; + } + + /// Find a single match starting at or after position 0 (whichever + /// position succeeds first, leftmost-first within that). Returns null + /// when nothing in the input matches. + pub fn search(self: *Vm, input: []const u8) ExecError!?MatchResult { + var start: usize = 0; + while (start <= input.len) : (start += 1) { + if (try self.matchAt(input, start)) |m| return m; + } + return null; + } + + /// Find all non-overlapping matches, leftmost-first. Caller owns the + /// returned slice and each MatchResult's captures. + pub fn findAll(self: *Vm, input: []const u8) ExecError![]MatchResult { + var results: std.ArrayList(MatchResult) = .empty; + errdefer { + for (results.items) |*m| m.deinit(self.alloc); + results.deinit(self.alloc); + } + + var pos: usize = 0; + while (pos <= input.len) { + const m_opt = try self.matchAt(input, pos); + if (m_opt) |m| { + try results.append(self.alloc, m); + // Advance past the match. Zero-width match (start == end) + // must still advance one byte or we'd loop forever. + pos = if (m.span.end > pos) m.span.end else pos + 1; + } else { + pos += 1; + } + } + return results.toOwnedSlice(self.alloc); + } + + /// Try to match the pattern against `input` starting exactly at `start`. + /// Returns the longest match the engine finds via leftmost-first + /// exploration, or null. Threads / captures are allocated from + /// `self.alloc` and the returned MatchResult owns its capture slice. + /// Try to match the pattern against `input` starting exactly at `start`. + /// Returns the longest leftmost-first match the engine finds, or null. + /// Captures in the returned MatchResult are owned by `self.alloc`; the + /// per-attempt scratch arena dies at end-of-scope. + fn matchAt(self: *Vm, input: []const u8, start: usize) ExecError!?MatchResult { + var arena = std.heap.ArenaAllocator.init(self.alloc); + defer arena.deinit(); + const aa = arena.allocator(); + + var clist = try ThreadList.init(aa, self.automaton.states.len); + var nlist = try ThreadList.init(aa, self.automaton.states.len); + + const initial_caps = try aa.alloc(?Span, self.cap_len); + @memset(initial_caps, null); + initial_caps[0] = .{ .start = start, .end = start }; + + try self.addThread(&clist, self.automaton.start, start, initial_caps, input, aa); + + var best: ?MatchResult = null; + var pos = start; + while (true) : (pos += 1) { + // Scan clist for accept in priority order. Lower-priority + // threads (after the first accept) are killed for this step; + // higher-priority threads (before it) keep stepping — they + // can still yield a longer leftmost-first match at a later + // position, which beats the recorded one. + var accept_idx: ?usize = null; + for (clist.threads.items, 0..) |t, idx| { + if (t.pc == self.automaton.accept) { + var captures = try self.alloc.alloc(?Span, self.cap_len); + for (t.captures, 0..) |c, i| captures[i] = c; + if (captures[0]) |*c0| c0.end = pos; + // Free the previous best (an older, shorter or + // lower-priority accept) before replacing. + if (best) |*old| old.deinit(self.alloc); + best = .{ + .span = .{ .start = start, .end = pos }, + .captures = captures, + }; + accept_idx = idx; + break; + } + } + + if (pos == input.len) break; + if (clist.threads.items.len == 0) break; + + // Priority cutoff: only threads with index < accept_idx are + // allowed to step. When no accept was found this step, all + // threads step. + const step_limit = accept_idx orelse clist.threads.items.len; + + for (clist.threads.items[0..step_limit]) |t| { + if (t.pc == self.automaton.accept) continue; + const state = self.automaton.states[t.pc]; + switch (state.consume) { + .byte => |b| { + if (self.byteMatches(b, input[pos])) { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + .any => { + if (self.flags.dot_all or input[pos] != '\n') { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + .class => |c| { + if (self.classMatches(c, input[pos])) { + if (state.out1) |o| try self.addThread(&nlist, o, pos + 1, t.captures, input, aa); + } + }, + // Zero-width states never reach an active list — + // addThread walks past them. Reaching here means an + // NFA construction bug. + .epsilon, .anchor, .group_start, .group_end => unreachable, + } + } + + // Nothing advanced this step. If we've recorded a match, ship it. + if (nlist.threads.items.len == 0) break; + + clist.clear(aa); + std.mem.swap(ThreadList, &clist, &nlist); + } + + return best; + } + + /// Walk every zero-width state reachable from `pc` and add any consuming + /// states (or the accept state) into `list`. Captures are duplicated at + /// every group boundary so sibling threads don't observe each other's + /// writes. + fn addThread( + self: *Vm, + list: *ThreadList, + pc: nfa.StateId, + pos: usize, + captures: []?Span, + input: []const u8, + aa: std.mem.Allocator, + ) ExecError!void { + if (!list.markIfNew(pc)) return; + + if (pc == self.automaton.accept) { + try list.threads.append(aa, .{ .pc = pc, .captures = captures }); + return; + } + + const state = self.automaton.states[pc]; + switch (state.consume) { + .epsilon => { + if (state.out1) |o| try self.addThread(list, o, pos, captures, input, aa); + if (state.out2) |o| try self.addThread(list, o, pos, captures, input, aa); + }, + .anchor => |a| { + if (self.anchorMatches(a, input, pos)) { + if (state.out1) |o| try self.addThread(list, o, pos, captures, input, aa); + } + // Anchor fail: thread dies here. + }, + .group_start => |idx| { + const new_caps = try dupeAndSet(aa, captures, idx, .{ .start = pos, .end = pos }); + if (state.out1) |o| try self.addThread(list, o, pos, new_caps, input, aa); + }, + .group_end => |idx| { + const new_caps = try dupeAndSetEnd(aa, captures, idx, pos); + if (state.out1) |o| try self.addThread(list, o, pos, new_caps, input, aa); + }, + .byte, .any, .class => { + try list.threads.append(aa, .{ .pc = pc, .captures = captures }); + }, + } + } + + // ── Predicates ── + + fn byteMatches(self: *const Vm, expected: u8, actual: u8) bool { + if (!self.flags.case_insensitive) return expected == actual; + return toLower(expected) == toLower(actual); + } + + fn classMatches(self: *const Vm, cls: *const ast.Class, actual: u8) bool { + if (cls.contains(actual)) return true; + if (self.flags.case_insensitive) { + const swapped = if (actual >= 'A' and actual <= 'Z') + actual + 32 + else if (actual >= 'a' and actual <= 'z') + actual - 32 + else + actual; + if (swapped != actual and cls.contains(swapped)) return true; + } + return false; + } + + fn anchorMatches(self: *const Vm, a: ast.Anchor, input: []const u8, pos: usize) bool { + return switch (a) { + .string_start => pos == 0, + .string_end => pos == input.len, + .line_start => pos == 0 or (self.flags.multiline and pos > 0 and input[pos - 1] == '\n'), + .line_end => pos == input.len or (self.flags.multiline and pos < input.len and input[pos] == '\n'), + .word_boundary => isAtWordBoundary(input, pos), + .non_word_boundary => !isAtWordBoundary(input, pos), + }; + } +}; + +fn dupeAndSet(alloc: std.mem.Allocator, captures: []?Span, idx: u32, value: Span) ![]?Span { + const out = try alloc.alloc(?Span, captures.len); + @memcpy(out, captures); + if (idx < out.len) out[idx] = value; + return out; +} + +fn dupeAndSetEnd(alloc: std.mem.Allocator, captures: []?Span, idx: u32, end: usize) ![]?Span { + const out = try alloc.alloc(?Span, captures.len); + @memcpy(out, captures); + if (idx < out.len) { + if (out[idx]) |*span| { + span.end = end; + } else { + // group_end without a matching group_start — shouldn't happen + // with our NFA construction, but treat as zero-width if it does. + out[idx] = .{ .start = end, .end = end }; + } + } + return out; +} + +fn toLower(c: u8) u8 { + return if (c >= 'A' and c <= 'Z') c + 32 else c; +} + +fn isWordChar(c: u8) bool { + return (c >= 'a' and c <= 'z') or + (c >= 'A' and c <= 'Z') or + (c >= '0' and c <= '9') or + c == '_'; +} + +fn isAtWordBoundary(input: []const u8, pos: usize) bool { + const left_is_word = pos > 0 and isWordChar(input[pos - 1]); + const right_is_word = pos < input.len and isWordChar(input[pos]); + return left_is_word != right_is_word; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn runFindAll(alloc: std.mem.Allocator, pattern: []const u8, input: []const u8) ![]MatchResult { + return runFindAllFlags(alloc, pattern, input, .{}); +} + +fn runFindAllFlags(alloc: std.mem.Allocator, pattern: []const u8, input: []const u8, flags: Flags) ![]MatchResult { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const automaton = try nfa.build(arena.allocator(), root, p.n_groups); + var vm = Vm.init(alloc, &automaton, flags); + return try vm.findAll(input); +} + +fn freeMatches(alloc: std.mem.Allocator, matches: []MatchResult) void { + for (matches) |*m| m.deinit(alloc); + alloc.free(matches); +} + +test "exec: literal match" { + const ms = try runFindAll(std.testing.allocator, "abc", "the abc and abc again"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 4), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 7), ms[0].span.end); + try std.testing.expectEqual(@as(usize, 12), ms[1].span.start); +} + +test "exec: no match" { + const ms = try runFindAll(std.testing.allocator, "xyz", "the abc"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 0), ms.len); +} + +test "exec: dot star greedy" { + const ms = try runFindAll(std.testing.allocator, "a.*b", "axxxb yyy"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 1), ms.len); + try std.testing.expectEqual(@as(usize, 0), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: dot star lazy" { + const ms = try runFindAll(std.testing.allocator, "a.*?b", "axxxbxxxb"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 1), ms.len); + // Lazy stops at the first 'b'. + try std.testing.expectEqual(@as(usize, 0), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: char class" { + const ms = try runFindAll(std.testing.allocator, "[a-z]+", "Hello World"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 1), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].span.end); +} + +test "exec: alternation prefers leftmost" { + const ms = try runFindAll(std.testing.allocator, "cat|dog|bird", "the dog saw a cat and a bird"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "exec: anchors line start" { + const flags = Flags{ .multiline = true }; + const ms = try runFindAllFlags(std.testing.allocator, "^foo", "foo\nbar foo\nfoo bar", flags); + defer freeMatches(std.testing.allocator, ms); + // multiline: ^foo matches at offset 0 and at offset 8 (after \n). + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "exec: counted quantifier" { + const ms = try runFindAll(std.testing.allocator, "a{2,3}", "a aa aaa aaaa"); + defer freeMatches(std.testing.allocator, ms); + // Single 'a' doesn't match (need >=2). "aa" matches. "aaa" matches. + // "aaaa" matches as "aaa" + "a" (the trailing 'a' alone doesn't qualify), + // so we get exactly: aa, aaa, aaa. + try std.testing.expectEqual(@as(usize, 3), ms.len); + try std.testing.expectEqual(@as(usize, 2), ms[0].span.end - ms[0].span.start); + try std.testing.expectEqual(@as(usize, 3), ms[1].span.end - ms[1].span.start); + try std.testing.expectEqual(@as(usize, 3), ms[2].span.end - ms[2].span.start); +} + +test "exec: capturing group" { + const ms = try runFindAll(std.testing.allocator, "(\\w+)@(\\w+)", "alice@example bob@host"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); + // First match: alice@example, groups: alice, example + try std.testing.expect(ms[0].captures[1] != null); + try std.testing.expectEqual(@as(usize, 0), ms[0].captures[1].?.start); + try std.testing.expectEqual(@as(usize, 5), ms[0].captures[1].?.end); + try std.testing.expectEqual(@as(usize, 6), ms[0].captures[2].?.start); + try std.testing.expectEqual(@as(usize, 13), ms[0].captures[2].?.end); +} + +test "exec: case-insensitive flag" { + const flags = Flags{ .case_insensitive = true }; + const ms = try runFindAllFlags(std.testing.allocator, "hello", "HELLO Hello hello", flags); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "exec: word boundary" { + const ms = try runFindAll(std.testing.allocator, "\\bcat\\b", "the cat sat on a catnap"); + defer freeMatches(std.testing.allocator, ms); + // 'cat' alone matches, 'catnap' doesn't (no boundary after 'cat'). + try std.testing.expectEqual(@as(usize, 1), ms.len); +} + +test "exec: digit shorthand" { + const ms = try runFindAll(std.testing.allocator, "\\d+", "abc 42 def 1234 xyz"); + defer freeMatches(std.testing.allocator, ms); + try std.testing.expectEqual(@as(usize, 2), ms.len); +} diff --git a/vendor/nanoregex/src/minterm.zig b/vendor/nanoregex/src/minterm.zig new file mode 100644 index 00000000..3f0164c2 --- /dev/null +++ b/vendor/nanoregex/src/minterm.zig @@ -0,0 +1,191 @@ +//! Byte-class compression for the DFA's transition table. +//! +//! Two bytes are EQUIVALENT for a given pattern when every atomic predicate +//! in that pattern (literal-byte tests, character classes, `.`) agrees on +//! them. Equivalent bytes drive the DFA to the same next state, so we can +//! collapse them into a single "minterm" class and index transitions by +//! class id instead of raw byte. +//! +//! Concretely: a pattern with `[a-z]+` has two classes — {a..z} and +//! everything else — so the DFA's transition row shrinks from 256 entries +//! to 2. A pattern with several distinct literals plus a class typically +//! has 10-20 classes. Resharp reported a 7× speedup from this alone, +//! mostly because the smaller table fits in L1. +//! +//! Build cost is one O(256 × n_predicates) pass at compile time. We cap +//! n_predicates at 64 (fits a u64 signature); patterns with more atomic +//! predicates than that fall back to an identity (byte == class) table, +//! which makes minterm a no-op and we still match correctly. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const Table = struct { + /// byte → class_id (0..n_classes-1). For n_classes == 256 this is the + /// identity mapping and minterm acts as a no-op. + byte_to_class: [256]u8, + /// Number of distinct classes. 1..256. + n_classes: u16, + /// class_id → arbitrary byte that lives in that class. Used when the + /// DFA's `move` step needs a concrete byte to feed into per-NFA-state + /// predicate tests. + representatives: [256]u8, +}; + +pub const Error = error{OutOfMemory}; + +pub fn build(arena: std.mem.Allocator, root: *const ast.Node, dot_all: bool) Error!Table { + var preds: std.ArrayList(Predicate) = .empty; + defer preds.deinit(arena); + try collectPredicates(root, arena, &preds); + + // Pattern with no consuming atoms — only anchors / empty. Nothing to + // distinguish; collapse the alphabet to a single class. + if (preds.items.len == 0) return singleClass(); + // Too many predicates for our 64-bit signature; bail to identity. The + // matcher still works, the minterm is just a pass-through. + if (preds.items.len > 64) return identity(); + + var sigs: [256]u64 = undefined; + for (0..256) |b| { + var sig: u64 = 0; + for (preds.items, 0..) |pred, i| { + if (matches(pred, @intCast(b), dot_all)) { + sig |= @as(u64, 1) << @intCast(i); + } + } + sigs[b] = sig; + } + + var sig_to_class = std.AutoHashMap(u64, u16).init(arena); + defer sig_to_class.deinit(); + + var byte_to_class: [256]u8 = undefined; + var representatives: [256]u8 = undefined; + var n_classes: u16 = 0; + + for (0..256) |b| { + const sig = sigs[b]; + if (sig_to_class.get(sig)) |existing| { + byte_to_class[b] = @intCast(existing); + } else { + const c = n_classes; + try sig_to_class.put(sig, c); + representatives[c] = @intCast(b); + byte_to_class[b] = @intCast(c); + n_classes += 1; + } + } + + return .{ + .byte_to_class = byte_to_class, + .n_classes = n_classes, + .representatives = representatives, + }; +} + +// ── Internals ── + +const Predicate = union(enum) { + byte: u8, + any, + class: *const ast.Class, +}; + +fn collectPredicates(node: *const ast.Node, arena: std.mem.Allocator, out: *std.ArrayList(Predicate)) Error!void { + switch (node.*) { + .literal => |c| try out.append(arena, .{ .byte = c }), + .dot => try out.append(arena, .any), + .class => |cls| try out.append(arena, .{ .class = cls }), + .anchor => {}, // zero-width — doesn't partition bytes + .concat => |children| for (children) |c| try collectPredicates(c, arena, out), + .alt => |children| for (children) |c| try collectPredicates(c, arena, out), + .repeat => |r| try collectPredicates(r.sub, arena, out), + .group => |g| try collectPredicates(g.sub, arena, out), + } +} + +fn matches(p: Predicate, b: u8, dot_all: bool) bool { + return switch (p) { + .byte => |v| v == b, + .any => dot_all or b != '\n', + .class => |cls| cls.contains(b), + }; +} + +fn singleClass() Table { + var bts: [256]u8 = undefined; + @memset(&bts, 0); + var reps: [256]u8 = undefined; + @memset(&reps, 0); + return .{ .byte_to_class = bts, .n_classes = 1, .representatives = reps }; +} + +fn identity() Table { + var bts: [256]u8 = undefined; + var reps: [256]u8 = undefined; + for (0..256) |i| { + bts[i] = @intCast(i); + reps[i] = @intCast(i); + } + return .{ .byte_to_class = bts, .n_classes = 256, .representatives = reps }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildFor(pattern: []const u8, dot_all: bool) !Table { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return try build(std.testing.allocator, root, dot_all); +} + +test "minterm: single class for a-z+" { + const t = try buildFor("[a-z]+", false); + // {a..z} ⇒ one class, everything else ⇒ another. 2 classes. + try std.testing.expectEqual(@as(u16, 2), t.n_classes); + try std.testing.expectEqual(t.byte_to_class['a'], t.byte_to_class['z']); + try std.testing.expect(t.byte_to_class['a'] != t.byte_to_class['A']); +} + +test "minterm: pure-literal pattern splits each distinct byte" { + const t = try buildFor("abc", false); + // 'a', 'b', 'c' each get a class; everything else is one more. 4. + try std.testing.expectEqual(@as(u16, 4), t.n_classes); + try std.testing.expect(t.byte_to_class['a'] != t.byte_to_class['b']); + try std.testing.expect(t.byte_to_class['b'] != t.byte_to_class['c']); +} + +test "minterm: dot collapses everything except newline" { + const t = try buildFor("a.b", false); + // Classes: 'a', 'b', '\n' (because . doesn't match it), and "rest". + try std.testing.expectEqual(@as(u16, 4), t.n_classes); + try std.testing.expect(t.byte_to_class['\n'] != t.byte_to_class['x']); +} + +test "minterm: dot-all collapses newline with the rest" { + const t = try buildFor("a.b", true); + // With dot_all the . matches \n too. So \n joins the "rest" class. + // Classes: 'a', 'b', "rest" (incl '\n'). 3. + try std.testing.expectEqual(@as(u16, 3), t.n_classes); + try std.testing.expectEqual(t.byte_to_class['\n'], t.byte_to_class['x']); +} + +test "minterm: anchors don't partition bytes" { + const t = try buildFor("^abc$", false); + // ^ and $ are zero-width, they don't appear in the predicate list. + // Classes are the same as for "abc": 'a', 'b', 'c', "rest" = 4. + try std.testing.expectEqual(@as(u16, 4), t.n_classes); +} + +test "minterm: representatives are valid bytes in their class" { + const t = try buildFor("[a-z]+", false); + var c: u16 = 0; + while (c < t.n_classes) : (c += 1) { + const rep = t.representatives[c]; + try std.testing.expectEqual(@as(u16, c), @as(u16, t.byte_to_class[rep])); + } +} diff --git a/vendor/nanoregex/src/nfa.zig b/vendor/nanoregex/src/nfa.zig new file mode 100644 index 00000000..a23369e4 --- /dev/null +++ b/vendor/nanoregex/src/nfa.zig @@ -0,0 +1,449 @@ +//! Thompson NFA construction. +//! +//! Each AST node compiles to a Frag: a single entry state plus a list of +//! "dangling" out-edges that the parent patches when concatenating. The +//! final NFA has one start state and one accept state, both well-defined +//! indices into `states`. The graph is arena-allocated alongside the AST. +//! +//! Greedy vs lazy quantifiers differ only in the *order* of out-edges on +//! the fork state — out1 is preferred by the matcher, so greedy puts the +//! sub-fragment on out1 (consume more) and lazy puts the continuation on +//! out1 (consume less). The matcher honours the ordering during simulation. +//! +//! Counted quantifiers `{m,n}` are unfolded inline — m mandatory copies +//! followed by (n-m) optional copies, capped at 1024 unfolds to keep +//! compile time bounded. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const StateId = u32; + +pub const Consume = union(enum) { + /// No input consumed; no observation. Used at forks and merges. + epsilon, + /// Consume one byte; succeeds iff `byte == value`. + byte: u8, + /// Consume one byte; succeeds iff byte matches dot. The matcher consults + /// its `dot_all` flag to decide whether `\n` is included. + any, + /// Consume one byte; succeeds iff `byte ∈ class.bitmap`. + class: *const ast.Class, + /// Zero-width anchor — succeeds iff the current position satisfies it. + anchor: ast.Anchor, + /// No input consumed. Side effect: mark capture group N start at the + /// current position. The matcher updates its capture array. + group_start: u32, + /// No input consumed. Side effect: mark capture group N end. + group_end: u32, +}; + +pub const State = struct { + consume: Consume, + /// Primary out edge. For consuming states (byte/any/class), out1 is + /// followed only when the byte matches. For zero-width states + /// (epsilon/anchor/group_*), out1 is followed unconditionally. + out1: ?StateId, + /// Secondary out — populated only when this is an alt or repeat fork. + /// Both out edges are then epsilon-like. + out2: ?StateId, +}; + +pub const Nfa = struct { + states: []State, + start: StateId, + /// The accept state is an epsilon state with no out edges. Matcher + /// detects acceptance by id, not by Consume tag. + accept: StateId, + n_groups: u32, +}; + +pub const BuildError = error{ OutOfMemory, QuantifierTooLarge }; + +/// Compile an AST into an NFA. Allocates state buffer + dangling-edge +/// scratch from `arena`. The returned slice is also arena-owned. +pub fn build(arena: std.mem.Allocator, root: *const ast.Node, n_groups: u32) BuildError!Nfa { + var b: Builder = .{ .arena = arena, .states = .empty }; + var frag = try b.compile(root); + defer frag.dangles.deinit(arena); + + // All dangling out-edges become inputs to a single accept state. + const accept = try b.newState(.epsilon, null, null); + for (frag.dangles.items) |p| b.patch(p, accept); + + return .{ + .states = try arena.dupe(State, b.states.items), + .start = frag.start, + .accept = accept, + .n_groups = n_groups, + }; +} + +// ── Internal construction state ── + +const PatchSlot = enum { out1, out2 }; + +const PatchPoint = struct { + state: StateId, + slot: PatchSlot, +}; + +const Frag = struct { + start: StateId, + /// Out-edges that haven't been wired to anything yet. The parent + /// node patches them when concatenating with the next fragment. + dangles: std.ArrayList(PatchPoint), +}; + +const Builder = struct { + arena: std.mem.Allocator, + states: std.ArrayList(State), + + fn newState(self: *Builder, consume: Consume, out1: ?StateId, out2: ?StateId) BuildError!StateId { + const id: StateId = @intCast(self.states.items.len); + try self.states.append(self.arena, .{ .consume = consume, .out1 = out1, .out2 = out2 }); + return id; + } + + fn patch(self: *Builder, point: PatchPoint, target: StateId) void { + switch (point.slot) { + .out1 => self.states.items[point.state].out1 = target, + .out2 => self.states.items[point.state].out2 = target, + } + } + + fn singleton(self: *Builder, consume: Consume) BuildError!Frag { + const s = try self.newState(consume, null, null); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ .state = s, .slot = .out1 }); + return .{ .start = s, .dangles = dangles }; + } + + fn epsilonFrag(self: *Builder) BuildError!Frag { + return self.singleton(.epsilon); + } + + fn compile(self: *Builder, node: *const ast.Node) BuildError!Frag { + return switch (node.*) { + .literal => |c| try self.singleton(.{ .byte = c }), + .dot => try self.singleton(.any), + .class => |cls| try self.singleton(.{ .class = cls }), + .anchor => |a| try self.singleton(.{ .anchor = a }), + .concat => |children| try self.compileConcat(children), + .alt => |children| try self.compileAlt(children), + .repeat => |r| try self.compileRepeat(r), + .group => |g| try self.compileGroup(g), + }; + } + + fn compileConcat(self: *Builder, children: []const *const ast.Node) BuildError!Frag { + if (children.len == 0) return try self.epsilonFrag(); + var acc = try self.compile(children[0]); + var i: usize = 1; + while (i < children.len) : (i += 1) { + const next = try self.compile(children[i]); + for (acc.dangles.items) |p| self.patch(p, next.start); + acc.dangles.deinit(self.arena); + acc.dangles = next.dangles; + } + return acc; + } + + fn compileAlt(self: *Builder, children: []const *const ast.Node) BuildError!Frag { + if (children.len == 1) return self.compile(children[0]); + + // Build right-to-left so the leftmost branch is preferred at the + // top-level fork (matches Python re's leftmost-first semantics). + var i: usize = children.len; + i -= 1; + var rest = try self.compile(children[i]); + while (i > 0) { + i -= 1; + var branch = try self.compile(children[i]); + const fork = try self.newState(.epsilon, branch.start, rest.start); + // Merge danglers from both sides. + for (rest.dangles.items) |p| try branch.dangles.append(self.arena, p); + rest.dangles.deinit(self.arena); + rest = .{ .start = fork, .dangles = branch.dangles }; + } + return rest; + } + + fn compileRepeat(self: *Builder, r: *const ast.Repeat) BuildError!Frag { + // Fast paths for the common shapes — *, +, ?. + if (r.min == 0 and r.max == std.math.maxInt(u32)) return try self.compileStar(r.sub, r.greedy); + if (r.min == 1 and r.max == std.math.maxInt(u32)) return try self.compilePlus(r.sub, r.greedy); + if (r.min == 0 and r.max == 1) return try self.compileQuestion(r.sub, r.greedy); + return try self.compileCounted(r); + } + + fn compileStar(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + for (sub.dangles.items) |p| self.patch(p, fork); + sub.dangles.deinit(self.arena); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = fork, .dangles = dangles }; + } + + fn compilePlus(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + for (sub.dangles.items) |p| self.patch(p, fork); + sub.dangles.deinit(self.arena); + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = sub.start, .dangles = dangles }; + } + + fn compileQuestion(self: *Builder, sub_node: *const ast.Node, greedy: bool) BuildError!Frag { + var sub = try self.compile(sub_node); + const fork = if (greedy) + try self.newState(.epsilon, sub.start, null) + else + try self.newState(.epsilon, null, sub.start); + try sub.dangles.append(self.arena, .{ + .state = fork, + .slot = if (greedy) .out2 else .out1, + }); + return .{ .start = fork, .dangles = sub.dangles }; + } + + fn compileCounted(self: *Builder, r: *const ast.Repeat) BuildError!Frag { + // Bound unfold size — a pattern like `a{100000}` is almost certainly + // pathological; refuse so compile-time stays sane. + const max_unfold: u32 = 1024; + if (r.min > max_unfold) return BuildError.QuantifierTooLarge; + if (r.max != std.math.maxInt(u32) and r.max > max_unfold) return BuildError.QuantifierTooLarge; + + var head: ?Frag = null; + var idx: u32 = 0; + while (idx < r.min) : (idx += 1) { + const next = try self.compile(r.sub); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, next.start); + h.dangles.deinit(self.arena); + h.dangles = next.dangles; + } else { + head = next; + } + } + + if (r.max == std.math.maxInt(u32)) { + // {m,∞} → m mandatory copies followed by a star tail. + const tail = try self.compileStar(r.sub, r.greedy); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, tail.start); + h.dangles.deinit(self.arena); + h.dangles = tail.dangles; + return h.*; + } + return tail; + } + + // {m,n} → m mandatory + (n-m) optional copies. + const optionals = r.max - r.min; + var i: u32 = 0; + while (i < optionals) : (i += 1) { + const opt = try self.compileQuestion(r.sub, r.greedy); + if (head) |*h| { + for (h.dangles.items) |p| self.patch(p, opt.start); + h.dangles.deinit(self.arena); + h.dangles = opt.dangles; + } else { + head = opt; + } + } + + if (head) |h| return h; + // Pure {0,0} — match the empty string. + return try self.epsilonFrag(); + } + + fn compileGroup(self: *Builder, g: *const ast.Group) BuildError!Frag { + var sub = try self.compile(g.sub); + if (!g.capturing) return sub; + + // Wrap sub with group_start ... sub ... group_end. + const start = try self.newState(.{ .group_start = g.index }, sub.start, null); + const end = try self.newState(.{ .group_end = g.index }, null, null); + for (sub.dangles.items) |p| self.patch(p, end); + sub.dangles.deinit(self.arena); + + var dangles: std.ArrayList(PatchPoint) = .empty; + try dangles.append(self.arena, .{ .state = end, .slot = .out1 }); + return .{ .start = start, .dangles = dangles }; + } +}; + +// ── Validation helpers (also used by tests) ── + +/// Walk every state and assert that out-edge ids point at real states. +/// Catches construction bugs that would otherwise show up as wrong matches. +pub fn validate(nfa: Nfa) !void { + if (nfa.start >= nfa.states.len) return error.InvalidStartState; + if (nfa.accept >= nfa.states.len) return error.InvalidAcceptState; + for (nfa.states, 0..) |s, i| { + if (s.out1) |o| if (o >= nfa.states.len) { + std.debug.print("state {d} out1={d} out-of-bounds\n", .{ i, o }); + return error.InvalidOutEdge; + }; + if (s.out2) |o| if (o >= nfa.states.len) { + std.debug.print("state {d} out2={d} out-of-bounds\n", .{ i, o }); + return error.InvalidOutEdge; + }; + } +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn buildFrom(arena: *std.heap.ArenaAllocator, pattern: []const u8) !Nfa { + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return try build(arena.allocator(), root, p.n_groups); +} + +test "nfa: literal compiles to one byte state + accept" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a"); + try validate(nfa); + try std.testing.expectEqual(@as(usize, 2), nfa.states.len); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: concat 'ab' chains two byte states" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "ab"); + try validate(nfa); + try std.testing.expectEqual(@as(usize, 3), nfa.states.len); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); + const second = nfa.states[nfa.start].out1.?; + try std.testing.expectEqual(Consume{ .byte = 'b' }, nfa.states[second].consume); +} + +test "nfa: alt 'a|b' builds a fork" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a|b"); + try validate(nfa); + // Fork (epsilon) + a + b + accept = 4. + try std.testing.expectEqual(@as(usize, 4), nfa.states.len); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + try std.testing.expect(nfa.states[nfa.start].out1 != null); + try std.testing.expect(nfa.states[nfa.start].out2 != null); +} + +test "nfa: star 'a*' builds fork pointing at sub + continuation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a*"); + try validate(nfa); + // a + fork + accept = 3. + try std.testing.expectEqual(@as(usize, 3), nfa.states.len); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + // Greedy: out1 should be the sub. + const sub_id = nfa.states[nfa.start].out1.?; + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[sub_id].consume); +} + +test "nfa: lazy star 'a*?' reverses fork ordering" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a*?"); + try validate(nfa); + // Lazy: out1 is the continuation, out2 is the sub. + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); + const sub_id = nfa.states[nfa.start].out2.?; + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[sub_id].consume); +} + +test "nfa: plus 'a+' starts at sub, not fork" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a+"); + try validate(nfa); + // 'a' must be matched at least once — start is the byte state. + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: question 'a?' makes sub optional" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a?"); + try validate(nfa); + try std.testing.expectEqual(Consume.epsilon, nfa.states[nfa.start].consume); +} + +test "nfa: capturing group wraps sub with group_start / group_end" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "(ab)"); + try validate(nfa); + try std.testing.expectEqual(@as(u32, 1), nfa.n_groups); + switch (nfa.states[nfa.start].consume) { + .group_start => |idx| try std.testing.expectEqual(@as(u32, 1), idx), + else => return error.ExpectedGroupStart, + } +} + +test "nfa: non-capturing group has no group_start/end" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "(?:ab)"); + try validate(nfa); + try std.testing.expectEqual(@as(u32, 0), nfa.n_groups); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: counted {2,3}" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a{2,3}"); + try validate(nfa); + // 2 mandatory + 1 optional + accept = at least 4 states. + try std.testing.expect(nfa.states.len >= 4); + try std.testing.expectEqual(Consume{ .byte = 'a' }, nfa.states[nfa.start].consume); +} + +test "nfa: oversized quantifier errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), "a{2000}"); + const root = try p.parseRoot(); + try std.testing.expectError(BuildError.QuantifierTooLarge, build(arena.allocator(), root, p.n_groups)); +} + +test "nfa: anchor compiles to anchor state" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "^abc"); + try validate(nfa); + switch (nfa.states[nfa.start].consume) { + .anchor => |a| try std.testing.expectEqual(ast.Anchor.line_start, a), + else => return error.ExpectedAnchor, + } +} + +test "nfa: validate catches no-bug case" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const nfa = try buildFrom(&arena, "a(b|c)*d"); + try validate(nfa); +} diff --git a/vendor/nanoregex/src/parser.zig b/vendor/nanoregex/src/parser.zig new file mode 100644 index 00000000..9f1f943b --- /dev/null +++ b/vendor/nanoregex/src/parser.zig @@ -0,0 +1,495 @@ +//! Recursive-descent regex parser. Pattern bytes → AST. +//! +//! Grammar (v1, mirrors the Python re subset we ship as feature parity): +//! +//! regex ::= alt +//! alt ::= concat ('|' concat)* +//! concat ::= atom* +//! atom ::= primary quantifier? +//! primary ::= literal | dot | class | anchor | group +//! group ::= '(' ('?:')? regex ')' +//! class ::= '[' '^'? class_item+ ']' +//! class_item ::= char | char '-' char | escape +//! quantifier ::= ( '?' | '*' | '+' | '{' n (',' m?)? '}' ) '?'? +//! +//! Deferred to v2: backreferences, lookarounds, named groups, inline flags. +//! +//! All AST nodes are allocated in the caller-provided allocator (expected to +//! be a Regex-owned arena). Parse errors return a typed error; the partial +//! AST is freed when the arena drops. + +const std = @import("std"); +const ast = @import("ast.zig"); + +pub const ParseError = error{ + UnexpectedEnd, + UnbalancedParen, + UnbalancedBracket, + InvalidEscape, + InvalidQuantifier, + NothingToRepeat, + InvalidCharRange, + OutOfMemory, +}; + +pub const Parser = struct { + alloc: std.mem.Allocator, + src: []const u8, + pos: usize = 0, + /// Count of capturing groups seen so far. Incremented on `(` (but not + /// on `(?:`). Used to assign group indices in declaration order. + n_groups: u32 = 0, + + pub fn init(alloc: std.mem.Allocator, pattern: []const u8) Parser { + return .{ .alloc = alloc, .src = pattern }; + } + + pub fn parseRoot(self: *Parser) ParseError!*const ast.Node { + const root = try self.parseAlt(); + if (self.pos < self.src.len) { + // A stray closing `)` would land here. + return ParseError.UnbalancedParen; + } + return root; + } + + // ── alt = concat ('|' concat)* ── + + fn parseAlt(self: *Parser) ParseError!*const ast.Node { + var branches: std.ArrayList(*const ast.Node) = .empty; + defer branches.deinit(self.alloc); + + try branches.append(self.alloc, try self.parseConcat()); + while (self.peek() == '|') { + self.pos += 1; + try branches.append(self.alloc, try self.parseConcat()); + } + if (branches.items.len == 1) return branches.items[0]; + const slice = try self.alloc.dupe(*const ast.Node, branches.items); + return try self.node(.{ .alt = slice }); + } + + // ── concat = atom* ── + + fn parseConcat(self: *Parser) ParseError!*const ast.Node { + var pieces: std.ArrayList(*const ast.Node) = .empty; + defer pieces.deinit(self.alloc); + + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == '|' or c == ')') break; + try pieces.append(self.alloc, try self.parseAtom()); + } + if (pieces.items.len == 0) { + // Empty concat matches the empty string. Represent as an empty + // concat node — the matcher treats it as zero-width success. + const slice = try self.alloc.dupe(*const ast.Node, &.{}); + return try self.node(.{ .concat = slice }); + } + if (pieces.items.len == 1) return pieces.items[0]; + const slice = try self.alloc.dupe(*const ast.Node, pieces.items); + return try self.node(.{ .concat = slice }); + } + + // ── atom = primary quantifier? ── + + fn parseAtom(self: *Parser) ParseError!*const ast.Node { + const primary = try self.parsePrimary(); + return try self.maybeQuantify(primary); + } + + fn parsePrimary(self: *Parser) ParseError!*const ast.Node { + if (self.pos >= self.src.len) return ParseError.UnexpectedEnd; + const c = self.src[self.pos]; + switch (c) { + '.' => { + self.pos += 1; + return try self.node(.dot); + }, + '^' => { + self.pos += 1; + return try self.node(.{ .anchor = .line_start }); + }, + '$' => { + self.pos += 1; + return try self.node(.{ .anchor = .line_end }); + }, + '(' => return try self.parseGroup(), + '[' => return try self.parseClass(), + '\\' => return try self.parseEscape(), + '*', '+', '?', '{' => return ParseError.NothingToRepeat, + ')', '|' => return ParseError.UnexpectedEnd, + else => { + self.pos += 1; + return try self.node(.{ .literal = c }); + }, + } + } + + // ── group = '(' ('?:')? regex ')' ── + + fn parseGroup(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '('); + self.pos += 1; + + var capturing = true; + if (self.pos + 1 < self.src.len and self.src[self.pos] == '?' and self.src[self.pos + 1] == ':') { + capturing = false; + self.pos += 2; + } + + // Reserve the capture index BEFORE recursing so nested groups get + // higher indices, matching Python re's left-paren declaration order. + var index: u32 = 0; + if (capturing) { + self.n_groups += 1; + index = self.n_groups; + } + + const sub = try self.parseAlt(); + + if (self.peek() != ')') return ParseError.UnbalancedParen; + self.pos += 1; + + const g = try self.alloc.create(ast.Group); + g.* = .{ .sub = sub, .index = index, .capturing = capturing }; + return try self.node(.{ .group = g }); + } + + // ── class = '[' '^'? items ']' ── + + fn parseClass(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '['); + self.pos += 1; + + const cls = try self.alloc.create(ast.Class); + cls.* = ast.Class.empty(); + + var negate = false; + if (self.peek() == '^') { + negate = true; + self.pos += 1; + } + + // A `]` as the very first char inside the class is treated as a + // literal `]`, matching Python re's behaviour. Otherwise `]` ends. + var first = true; + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == ']' and !first) break; + first = false; + + const lo = try self.parseClassChar(); + // Range `a-z` only if the `-` is followed by a non-`]` char. + if (self.pos + 1 < self.src.len and self.src[self.pos] == '-' and self.src[self.pos + 1] != ']') { + self.pos += 1; // consume '-' + const hi = try self.parseClassChar(); + if (hi < lo) return ParseError.InvalidCharRange; + cls.setRange(lo, hi); + } else { + cls.set(lo); + } + } + if (self.peek() != ']') return ParseError.UnbalancedBracket; + self.pos += 1; + + if (negate) cls.negate(); + return try self.node(.{ .class = cls }); + } + + fn parseClassChar(self: *Parser) ParseError!u8 { + if (self.pos >= self.src.len) return ParseError.UnbalancedBracket; + const c = self.src[self.pos]; + if (c == '\\') { + self.pos += 1; + if (self.pos >= self.src.len) return ParseError.InvalidEscape; + const e = self.src[self.pos]; + self.pos += 1; + return switch (e) { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '0' => 0, + else => e, + }; + } + self.pos += 1; + return c; + } + + // ── escape = '\' (shorthand | metaliteral) ── + + fn parseEscape(self: *Parser) ParseError!*const ast.Node { + std.debug.assert(self.src[self.pos] == '\\'); + self.pos += 1; + if (self.pos >= self.src.len) return ParseError.InvalidEscape; + const e = self.src[self.pos]; + self.pos += 1; + return switch (e) { + 'd' => try self.shorthandClass(digitClass()), + 'D' => try self.shorthandClass(negated(digitClass())), + 'w' => try self.shorthandClass(wordClass()), + 'W' => try self.shorthandClass(negated(wordClass())), + 's' => try self.shorthandClass(spaceClass()), + 'S' => try self.shorthandClass(negated(spaceClass())), + 'b' => try self.node(.{ .anchor = .word_boundary }), + 'B' => try self.node(.{ .anchor = .non_word_boundary }), + 'A' => try self.node(.{ .anchor = .string_start }), + 'z' => try self.node(.{ .anchor = .string_end }), + 'n' => try self.node(.{ .literal = '\n' }), + 't' => try self.node(.{ .literal = '\t' }), + 'r' => try self.node(.{ .literal = '\r' }), + '0' => try self.node(.{ .literal = 0 }), + else => try self.node(.{ .literal = e }), + }; + } + + fn shorthandClass(self: *Parser, cls_value: ast.Class) ParseError!*const ast.Node { + const cls = try self.alloc.create(ast.Class); + cls.* = cls_value; + return try self.node(.{ .class = cls }); + } + + fn digitClass() ast.Class { + var c = ast.Class.empty(); + c.setRange('0', '9'); + return c; + } + + fn wordClass() ast.Class { + var c = ast.Class.empty(); + c.setRange('a', 'z'); + c.setRange('A', 'Z'); + c.setRange('0', '9'); + c.set('_'); + return c; + } + + fn spaceClass() ast.Class { + var c = ast.Class.empty(); + c.set(' '); + c.set('\t'); + c.set('\n'); + c.set('\r'); + c.set(0x0b); // \v + c.set(0x0c); // \f + return c; + } + + fn negated(cls_in: ast.Class) ast.Class { + var c = cls_in; + c.negate(); + return c; + } + + // ── quantifier ── + + fn maybeQuantify(self: *Parser, primary: *const ast.Node) ParseError!*const ast.Node { + if (self.pos >= self.src.len) return primary; + const c = self.src[self.pos]; + var min: u32 = 0; + var max: u32 = 0; + switch (c) { + '?' => { + self.pos += 1; + min = 0; + max = 1; + }, + '*' => { + self.pos += 1; + min = 0; + max = std.math.maxInt(u32); + }, + '+' => { + self.pos += 1; + min = 1; + max = std.math.maxInt(u32); + }, + '{' => { + const parsed = try self.parseCountedQuantifier(); + min = parsed.min; + max = parsed.max; + }, + else => return primary, + } + var greedy = true; + if (self.peek() == '?') { + greedy = false; + self.pos += 1; + } + const r = try self.alloc.create(ast.Repeat); + r.* = .{ .sub = primary, .min = min, .max = max, .greedy = greedy }; + return try self.node(.{ .repeat = r }); + } + + fn parseCountedQuantifier(self: *Parser) ParseError!struct { min: u32, max: u32 } { + std.debug.assert(self.src[self.pos] == '{'); + self.pos += 1; + const lo = try self.readNumber(); + var hi = lo; + if (self.peek() == ',') { + self.pos += 1; + if (self.peek() == '}') { + hi = std.math.maxInt(u32); + } else { + hi = try self.readNumber(); + } + } + if (self.peek() != '}') return ParseError.InvalidQuantifier; + self.pos += 1; + if (hi < lo) return ParseError.InvalidQuantifier; + return .{ .min = lo, .max = hi }; + } + + fn readNumber(self: *Parser) ParseError!u32 { + const start = self.pos; + while (self.pos < self.src.len and self.src[self.pos] >= '0' and self.src[self.pos] <= '9') { + self.pos += 1; + } + if (self.pos == start) return ParseError.InvalidQuantifier; + return std.fmt.parseInt(u32, self.src[start..self.pos], 10) catch ParseError.InvalidQuantifier; + } + + // ── Helpers ── + + fn peek(self: *const Parser) ?u8 { + if (self.pos >= self.src.len) return null; + return self.src[self.pos]; + } + + fn node(self: *Parser, value: ast.Node) ParseError!*const ast.Node { + const n = try self.alloc.create(ast.Node); + n.* = value; + return n; + } +}; + +// ── Tests ── + +fn parseToString(alloc: std.mem.Allocator, pattern: []const u8) ![]u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(alloc); + try ast.debugWrite(root, &buf, alloc, 0); + return alloc.dupe(u8, buf.items); +} + +test "parse single literal" { + const out = try parseToString(std.testing.allocator, "a"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("literal 'a'\n", out); +} + +test "parse concat of literals" { + const out = try parseToString(std.testing.allocator, "abc"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse alternation" { + const out = try parseToString(std.testing.allocator, "a|b"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\alt + \\ literal 'a' + \\ literal 'b' + \\ + , out); +} + +test "parse star quantifier" { + const out = try parseToString(std.testing.allocator, "a*"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=0 max=4294967295 greedy=true + \\ literal 'a' + \\ + , out); +} + +test "parse lazy quantifier" { + const out = try parseToString(std.testing.allocator, "a+?"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=1 max=4294967295 greedy=false + \\ literal 'a' + \\ + , out); +} + +test "parse counted quantifier" { + const out = try parseToString(std.testing.allocator, "a{2,5}"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\repeat min=2 max=5 greedy=true + \\ literal 'a' + \\ + , out); +} + +test "parse char class with range" { + const out = try parseToString(std.testing.allocator, "[a-z]"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("class [26 bytes]\n", out); +} + +test "parse capturing group" { + const out = try parseToString(std.testing.allocator, "(abc)"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\group #1 cap=true + \\ concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse non-capturing group" { + const out = try parseToString(std.testing.allocator, "(?:abc)"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings( + \\group #0 cap=false + \\ concat + \\ literal 'a' + \\ literal 'b' + \\ literal 'c' + \\ + , out); +} + +test "parse shorthand class \\d" { + const out = try parseToString(std.testing.allocator, "\\d"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("class [10 bytes]\n", out); +} + +test "parse anchor word_boundary" { + const out = try parseToString(std.testing.allocator, "\\b"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("anchor word_boundary\n", out); +} + +test "parse unbalanced paren errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), "(abc"); + try std.testing.expectError(ParseError.UnbalancedParen, p.parseRoot()); +} + +test "parse nothing-to-repeat errors" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = Parser.init(arena.allocator(), "*abc"); + try std.testing.expectError(ParseError.NothingToRepeat, p.parseRoot()); +} diff --git a/vendor/nanoregex/src/prefilter.zig b/vendor/nanoregex/src/prefilter.zig new file mode 100644 index 00000000..ec044744 --- /dev/null +++ b/vendor/nanoregex/src/prefilter.zig @@ -0,0 +1,517 @@ +//! Compile-time pattern analysis for fast-path optimisations. +//! +//! Two analyses, both consumed by `findAll`: +//! +//! 1. `extractFullLiteral` — when the AST is purely literal (literal bytes, +//! optional non-capturing concat/group wrapping, no metacharacters, no +//! capture groups), returns the literal byte sequence. Callers can then +//! bypass the NFA entirely and use `std.mem.indexOf` in a loop — a +//! 50-100x win on patterns like `compileAllocFlags` that came in as +//! "regex" but have no actual regex content. +//! +//! 2. `extractRequiredLiteral` — for genuinely-regex patterns, finds the +//! longest contiguous run of bytes that MUST appear in any successful +//! match. Callers use it as a pre-filter: if the haystack doesn't +//! contain the substring, no match exists and we skip the engine. When +//! the haystack does contain it, we still gate engine work to windows +//! around hits via `findOccurrences`. +//! +//! Both analyses are conservative: when in doubt, they return null/empty so +//! the matcher falls back to the full engine. The unit tests pin down what +//! we extract for each shape. + +const std = @import("std"); +const ast = @import("ast.zig"); + +/// If the AST is purely literal — only `.literal` / `.concat` / non-capturing +/// `.group` nodes — flatten it to a single byte slice. Returns null when any +/// regex feature (dot, class, anchor, alt, repeat, capturing group) is +/// present. The returned slice is arena-allocated. +pub fn extractFullLiteral(arena: std.mem.Allocator, root: *const ast.Node) !?[]const u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(arena); + if (!try collectLiteral(root, arena, &buf)) { + buf.deinit(arena); + return null; + } + if (buf.items.len == 0) { + buf.deinit(arena); + return null; + } + return try buf.toOwnedSlice(arena); +} + +fn collectLiteral(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) !bool { + return switch (node.*) { + .literal => |c| { + try buf.append(arena, c); + return true; + }, + .concat => |children| { + for (children) |child| { + if (!try collectLiteral(child, arena, buf)) return false; + } + return true; + }, + .group => |g| { + // A capturing group changes externally-observable behaviour + // (callers may want span info), so bail out and let the engine + // handle it. Non-capturing groups are pure parens; transparent. + if (g.capturing) return false; + return collectLiteral(g.sub, arena, buf); + }, + .dot, .class, .anchor, .alt, .repeat => false, + }; +} + +/// Find the longest run of unconditionally-required literal bytes inside +/// the pattern. "Required" means every successful match must contain these +/// bytes contiguously. Returns null when no run of length ≥ `min_len` can +/// be extracted; below that threshold the prefilter overhead beats the +/// win. +pub fn extractRequiredLiteral(arena: std.mem.Allocator, root: *const ast.Node, min_len: usize) !?[]const u8 { + var best: std.ArrayList(u8) = .empty; + errdefer best.deinit(arena); + var current: std.ArrayList(u8) = .empty; + defer current.deinit(arena); + + try walkRequired(root, arena, ¤t, &best); + // Final flush in case the longest run ends at the AST tail. + if (current.items.len > best.items.len) { + best.deinit(arena); + best = current; + current = .empty; + } + + if (best.items.len < min_len) { + best.deinit(arena); + return null; + } + return try best.toOwnedSlice(arena); +} + +fn walkRequired( + node: *const ast.Node, + arena: std.mem.Allocator, + current: *std.ArrayList(u8), + best: *std.ArrayList(u8), +) error{OutOfMemory}!void { + switch (node.*) { + .literal => |c| try current.append(arena, c), + .concat => |children| for (children) |child| try walkRequired(child, arena, current, best), + .group => |g| try walkRequired(g.sub, arena, current, best), + .repeat => |r| { + // The sub-pattern is required iff min ≥ 1. When it is, treat the + // first mandatory copy as additional bytes in the current run. + // Past that, the rest can repeat-with-variation so we flush + // and start fresh. + if (r.min >= 1) { + try walkRequired(r.sub, arena, current, best); + } + try flush(current, best, arena); + }, + .dot, .class, .anchor => try flush(current, best, arena), + .alt => { + // We could pick the longest common prefix across branches but + // for v1 we bail conservatively — the run ends here. + try flush(current, best, arena); + }, + } +} + +fn flush(current: *std.ArrayList(u8), best: *std.ArrayList(u8), arena: std.mem.Allocator) !void { + if (current.items.len > best.items.len) { + best.deinit(arena); + best.* = current.*; + current.* = .empty; + } else { + current.clearRetainingCapacity(); + } +} + +/// Extract the contiguous literal byte sequence at the very START of the +/// pattern, if any. Differs from `extractRequiredLiteral` in two ways: +/// - We anchor at the pattern start instead of picking the longest run. +/// - Callers can use the result as a STARTING-POSITION hint: every match +/// must begin where this byte sequence occurs in the haystack. +/// +/// Bails on alternation at the top level (different branches → different +/// possible prefixes; we'd need their common prefix). Returns null when +/// the prefix is shorter than `min_len`, the threshold below which the +/// indexOf-and-resume overhead beats the engine. +pub fn extractLiteralPrefix(arena: std.mem.Allocator, root: *const ast.Node, min_len: usize) !?[]const u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(arena); + _ = collectPrefix(root, arena, &buf) catch |err| switch (err) { + error.OutOfMemory => return err, + }; + if (buf.items.len < min_len) { + buf.deinit(arena); + return null; + } + return try buf.toOwnedSlice(arena); +} + +/// Walk the AST left-to-right adding ONLY contiguous required literal bytes at +/// the start. Returns `false` when the prefix must stop here — at a non-literal +/// node, or after a quantifier. A quantifier ends the contiguous prefix even +/// when it contributes a mandatory copy: bytes past that copy land at a variable +/// offset, so nothing after the repeat may join the prefix. Without this, `hel+o` +/// yielded the prefix `helo`, which wrongly demands the `o` sit immediately after +/// a single `l` — making `search` miss `helllo` entirely. +fn collectPrefix(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) error{OutOfMemory}!bool { + switch (node.*) { + .literal => |c| { + try buf.append(arena, c); + return true; + }, + .concat => |children| { + for (children) |child| { + const cont = try collectPrefix(child, arena, buf); + if (!cont) return false; + } + return true; + }, + .group => |g| return try collectPrefix(g.sub, arena, buf), + .repeat => |r| { + // min ≥ 1 contributes one mandatory copy of the sub-pattern's + // prefix; min == 0 contributes nothing. Either way the contiguous + // prefix ends here — the repeat may match more bytes, pushing + // anything that follows to a variable offset. + if (r.min >= 1) _ = try collectPrefix(r.sub, arena, buf); + return false; + }, + // Class, dot, anchor, and alt all end the prefix. + .class, .dot, .anchor, .alt => return false, + } +} + +/// If the entire AST is a single `class+` or `class{n,}` quantifier (e.g. +/// `\d+`, `[a-z]+`, `\w{2,}`), return the underlying class so callers can +/// dispatch to a specialised SIMD-only scanner: "first byte in class → +/// first byte not in class → emit; repeat." Avoids the DFA entirely and +/// runs at memmem speed. +/// +/// Returns null for any compound shape (concat, alt, group with non-trivial +/// structure, anchors, etc.) — those need the real engine. +pub fn extractSingleClassRepeat(root: *const ast.Node) ?*const ast.Class { + // Allow `(?:class+)` and `(class+)` wrapping but only if the group is + // non-capturing (we don't track group spans on this path). + var n = root; + while (true) { + switch (n.*) { + .group => |g| if (!g.capturing) { + n = g.sub; + continue; + } else return null, + else => break, + } + } + switch (n.*) { + .repeat => |r| { + if (r.min < 1) return null; // min=0 has a zero-width quirk we skip + if (r.max != std.math.maxInt(u32)) return null; + switch (r.sub.*) { + .class => |cls| return cls, + else => return null, + } + }, + else => return null, + } +} + +/// If the entire AST is a pure literal alternation — `foo|bar|baz`, with +/// each branch a concat of `.literal` nodes only — return the branch byte +/// strings. Callers dispatch to a multi-literal scanner that finds the +/// earliest match across all needles via per-needle `indexOfPos`. This is +/// the same niche `aho-corasick`'s Teddy fills for ripgrep. +/// +/// Returns null for any branch that isn't purely literal, or for a single +/// branch (that'd already be caught by `extractFullLiteral`). Caller owns +/// the returned slice via the passed arena. +pub fn extractLiteralAlternation(arena: std.mem.Allocator, root: *const ast.Node) !?[]const []const u8 { + // Unwrap one layer of non-capturing group at the root (very common + // shape from `(?:foo|bar)`). + var n = root; + if (n.* == .group) { + const g = n.group; + if (g.capturing) return null; + n = g.sub; + } + if (n.* != .alt) return null; + const branches = n.alt; + if (branches.len < 2) return null; + + var out: std.ArrayList([]const u8) = .empty; + errdefer { + for (out.items) |b| arena.free(b); + out.deinit(arena); + } + for (branches) |branch| { + var bytes: std.ArrayList(u8) = .empty; + errdefer bytes.deinit(arena); + if (!try collectBranchLiteral(branch, arena, &bytes)) { + bytes.deinit(arena); + for (out.items) |b| arena.free(b); + out.deinit(arena); + return null; + } + if (bytes.items.len == 0) { + // Empty branch — would match at every position, not worth this fast path. + bytes.deinit(arena); + for (out.items) |b| arena.free(b); + out.deinit(arena); + return null; + } + try out.append(arena, try bytes.toOwnedSlice(arena)); + } + return try out.toOwnedSlice(arena); +} + +fn collectBranchLiteral(node: *const ast.Node, arena: std.mem.Allocator, buf: *std.ArrayList(u8)) !bool { + return switch (node.*) { + .literal => |c| blk: { + try buf.append(arena, c); + break :blk true; + }, + .concat => |children| blk: { + for (children) |child| { + if (!try collectBranchLiteral(child, arena, buf)) break :blk false; + } + break :blk true; + }, + .group => |g| if (g.capturing) false else collectBranchLiteral(g.sub, arena, buf), + else => false, + }; +} + +// ── Tests ── + +const parser = @import("parser.zig"); + +fn fullFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractFullLiteral(arena.allocator(), root); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn requiredFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractRequiredLiteral(arena.allocator(), root, 3); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn prefixFor(alloc: std.mem.Allocator, pattern: []const u8) !?[]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lit = try extractLiteralPrefix(arena.allocator(), root, 3); + if (lit) |bytes| return try alloc.dupe(u8, bytes); + return null; +} + +fn singleClassFor(pattern: []const u8) !bool { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + return extractSingleClassRepeat(root) != null; +} + +test "full literal: plain identifier" { + const got = (try fullFor(std.testing.allocator, "compileAllocFlags")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags", got); +} + +test "full literal: non-capturing group is transparent" { + const got = (try fullFor(std.testing.allocator, "(?:abc)def")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("abcdef", got); +} + +test "full literal: capturing group blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "(abc)")) == null); +} + +test "full literal: dot blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "a.b")) == null); +} + +test "full literal: class blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "a[bc]")) == null); +} + +test "full literal: alternation blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "abc|def")) == null); +} + +test "full literal: repeat blocks" { + try std.testing.expect((try fullFor(std.testing.allocator, "ab+")) == null); +} + +test "required literal: extracts prefix before class" { + const got = (try requiredFor(std.testing.allocator, "compileAllocFlags\\([a-z]+")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags(", got); +} + +test "required literal: picks longest run" { + const got = (try requiredFor(std.testing.allocator, "[a-z]hello\\d+worlds\\s+end")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("worlds", got); +} + +test "required literal: too short returns null" { + try std.testing.expect((try requiredFor(std.testing.allocator, "a.b")) == null); +} + +test "required literal: alternation bails" { + try std.testing.expect((try requiredFor(std.testing.allocator, "foo|bar")) == null); +} + +test "required literal: min=0 quantifier doesn't contribute" { + // 'a*bar' — 'a' is optional, so the required run is 'bar'. + const got = (try requiredFor(std.testing.allocator, "a*barbaz")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("barbaz", got); +} + +test "required literal: min=1 quantifier contributes single copy" { + // 'a+bar' — 'a' is required (at least once), then 'bar'. + // After walking the repeat, we flush — so 'a' alone is the run, but + // 'bar' is longer. Best = 'bar'. + const got = (try requiredFor(std.testing.allocator, "a+barbaz")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("barbaz", got); +} + +test "literal prefix: simple identifier" { + const got = (try prefixFor(std.testing.allocator, "compileAllocFlags\\([a-z]+")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("compileAllocFlags(", got); +} + +test "literal prefix: class at start bails" { + try std.testing.expect((try prefixFor(std.testing.allocator, "[a-z]+hello")) == null); +} + +test "literal prefix: alternation at top bails" { + try std.testing.expect((try prefixFor(std.testing.allocator, "foo|bar")) == null); +} + +test "literal prefix: under threshold returns null" { + try std.testing.expect((try prefixFor(std.testing.allocator, "ab.c")) == null); +} + +test "literal prefix: stops after + quantifier (regression)" { + // `hel+o` — the contiguous start prefix is `hel`, never `helo`: the `o` + // sits at a variable offset after one-or-more `l`. Demanding `helo` as a + // starting-position hint made `search` miss `helllo` outright. + const got = (try prefixFor(std.testing.allocator, "hel+o")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("hel", got); +} + +test "literal prefix: stops after {n,m} quantifier (regression)" { + const got = (try prefixFor(std.testing.allocator, "hel{1,3}o")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("hel", got); +} + +test "literal prefix: stops at optional byte" { + // `foox?bar` — 'foo' is mandatory, then 'x?' is optional, so prefix is 'foo'. + const got = (try prefixFor(std.testing.allocator, "foox?bar")).?; + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("foo", got); +} + +test "single-class-repeat: \\d+" { + try std.testing.expect(try singleClassFor("\\d+")); +} + +test "single-class-repeat: [a-z]+" { + try std.testing.expect(try singleClassFor("[a-z]+")); +} + +test "single-class-repeat: counted lower bound" { + try std.testing.expect(try singleClassFor("[0-9]{3,}")); +} + +test "single-class-repeat: non-capturing group is transparent" { + try std.testing.expect(try singleClassFor("(?:\\w+)")); +} + +test "single-class-repeat: capturing group blocks" { + try std.testing.expect(!(try singleClassFor("(\\w+)"))); +} + +test "single-class-repeat: concat blocks" { + try std.testing.expect(!(try singleClassFor("\\d+abc"))); +} + +test "single-class-repeat: alternation blocks" { + try std.testing.expect(!(try singleClassFor("\\d+|abc"))); +} + +test "single-class-repeat: star (min=0) skipped" { + // min=0 has a zero-width-match quirk — not yet handled by the fast path. + try std.testing.expect(!(try singleClassFor("\\d*"))); +} + +fn altLiteralFor(alloc: std.mem.Allocator, pattern: []const u8) !?[][]const u8 { + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + const lits = try extractLiteralAlternation(arena.allocator(), root); + if (lits) |bs| { + var owned = try alloc.alloc([]const u8, bs.len); + for (bs, 0..) |b, i| owned[i] = try alloc.dupe(u8, b); + return owned; + } + return null; +} + +test "literal alternation: three branches" { + const got = (try altLiteralFor(std.testing.allocator, "foo|bar|baz")).?; + defer { + for (got) |b| std.testing.allocator.free(b); + std.testing.allocator.free(got); + } + try std.testing.expectEqual(@as(usize, 3), got.len); + try std.testing.expectEqualStrings("foo", got[0]); + try std.testing.expectEqualStrings("bar", got[1]); + try std.testing.expectEqualStrings("baz", got[2]); +} + +test "literal alternation: non-capturing group wraps" { + const got = (try altLiteralFor(std.testing.allocator, "(?:cat|dog|bird)")).?; + defer { + for (got) |b| std.testing.allocator.free(b); + std.testing.allocator.free(got); + } + try std.testing.expectEqual(@as(usize, 3), got.len); +} + +test "literal alternation: bails on non-literal branch" { + try std.testing.expect((try altLiteralFor(std.testing.allocator, "foo|b.r")) == null); +} + +test "literal alternation: bails on capturing group" { + try std.testing.expect((try altLiteralFor(std.testing.allocator, "(foo|bar)")) == null); +} + +test "literal alternation: single branch is null" { + // Only one branch — handled by extractFullLiteral, not this path. + try std.testing.expect((try altLiteralFor(std.testing.allocator, "foo")) == null); +} diff --git a/vendor/nanoregex/src/probe.zig b/vendor/nanoregex/src/probe.zig new file mode 100644 index 00000000..73cbe201 --- /dev/null +++ b/vendor/nanoregex/src/probe.zig @@ -0,0 +1,87 @@ +//! Parity-test probe CLI. +//! +//! Wire shape: nanoregex_probe [flags] +//! Flags string (3rd argv) is any concatenation of `i`/`m`/`s` matching +//! Python re's IGNORECASE / MULTILINE / DOTALL — same encoding used by +//! tests/parity/run.sh. +//! +//! Output: one match per line as `start..end\tmatched_bytes`. Bytes are +//! emitted raw (no escaping) so the harness can diff byte-for-byte against +//! `re.finditer` output formatted the same way. +//! +//! Stdio is via extern libc `write`: Zig 0.16 removed the synchronous +//! stdlib wrappers we used to rely on, and nanoregex links libc anyway. + +const std = @import("std"); +const nanoregex = @import("nanoregex"); + +extern "c" fn write(fd: c_int, ptr: [*]const u8, len: usize) isize; + +fn writeAll(fd: c_int, data: []const u8) void { + var rem = data; + while (rem.len > 0) { + const n = write(fd, rem.ptr, rem.len); + if (n <= 0) return; + rem = rem[@intCast(n)..]; + } +} + +fn parseFlags(s: []const u8) nanoregex.Flags { + var f: nanoregex.Flags = .{}; + for (s) |c| switch (c) { + 'i' => f.case_insensitive = true, + 'm' => f.multiline = true, + 's' => f.dot_all = true, + else => {}, + }; + return f; +} + +pub fn main(init: std.process.Init) !void { + const alloc = init.gpa; + + var args_list: std.ArrayList([]const u8) = .empty; + defer args_list.deinit(alloc); + var args_iter = init.minimal.args.iterate(); + while (args_iter.next()) |arg| try args_list.append(alloc, arg); + const args = args_list.items; + + if (args.len < 2) { + writeAll(2, "usage: nanoregex_probe [] []\n"); + std.process.exit(2); + } + const pattern = args[1]; + const haystack: []const u8 = if (args.len >= 3) args[2] else ""; + const flags = parseFlags(if (args.len >= 4) args[3] else ""); + + var r = nanoregex.Regex.compileWithFlags(alloc, pattern, flags) catch |err| { + var tmp: [128]u8 = undefined; + const msg = std.fmt.bufPrint(&tmp, "PARSE_ERROR: {s}\n", .{@errorName(err)}) catch "PARSE_ERROR\n"; + writeAll(1, msg); + // Exit 0 — the harness checks output content, not exit code, so + // PARSE_ERROR on both sides should match cleanly. + std.process.exit(0); + }; + defer r.deinit(); + + const matches = r.findAll(alloc, haystack) catch { + writeAll(2, "ENGINE_ERROR\n"); + std.process.exit(1); + }; + defer { + for (matches) |*m| @constCast(m).deinit(alloc); + alloc.free(matches); + } + + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(alloc); + + var line_buf: [256]u8 = undefined; + for (matches) |m| { + const header = std.fmt.bufPrint(&line_buf, "{d}..{d}\t", .{ m.span.start, m.span.end }) catch continue; + try buf.appendSlice(alloc, header); + try buf.appendSlice(alloc, haystack[m.span.start..m.span.end]); + try buf.append(alloc, '\n'); + } + writeAll(1, buf.items); +} diff --git a/vendor/nanoregex/src/root.zig b/vendor/nanoregex/src/root.zig new file mode 100644 index 00000000..089c79f8 --- /dev/null +++ b/vendor/nanoregex/src/root.zig @@ -0,0 +1,676 @@ +//! nanoregex — pure-Zig regex engine with Python-re-compatible semantics. +//! +//! Layered design: +//! 1. parser.zig — pattern bytes → AST +//! 2. ast.zig — AST node tagged union, arena-owned +//! 3. nfa.zig — AST → Thompson NFA +//! 4. exec.zig — Pike-VM NFA simulation (always-correct fallback) +//! 5. prefilter.zig — literal/required-substring extraction (fast path) +//! 6. dfa.zig — Lazy subset-construction DFA (perf path) +//! +//! Dispatch policy in findAll/search: +//! 1. Pure-literal AST + no captures + case-sensitive → memmem loop +//! 2. Required literal absent in haystack → early-return empty +//! 3. DFA eligible (no captures, no anchors, no case-insensitive) → DFA +//! 4. Otherwise → Pike VM +//! +//! The DFA is built eagerly at compile time (when eligible) so the hot loop +//! is a single transition-table lookup per byte. Regex deinit cleans the +//! DFA's arena. + +const std = @import("std"); +pub const ast = @import("ast.zig"); +pub const parser = @import("parser.zig"); +pub const nfa = @import("nfa.zig"); +pub const exec = @import("exec.zig"); +pub const prefilter = @import("prefilter.zig"); +pub const dfa = @import("dfa.zig"); +pub const minterm = @import("minterm.zig"); +const comptime_mod = @import("comptime.zig"); + +/// comptime-specialised matcher factory. For patterns known at compile time, +/// returns a zero-overhead specialised scanner. See `src/comptime.zig`. +pub const compileComptime = comptime_mod.compileComptime; +pub fn ComptimeRegex(comptime pattern: []const u8) type { + return comptime_mod.ComptimeRegex(pattern); +} + +pub const Flags = exec.Flags; +pub const Span = exec.Span; +pub const Match = exec.MatchResult; + +pub const Regex = struct { + /// Backing arena for the AST + NFA + prefilter slices. Lives until deinit(). + arena: *std.heap.ArenaAllocator, + parent_alloc: std.mem.Allocator, + root: *const ast.Node, + /// Heap-allocated on `arena` so the address is stable for `dfa.nfa_ref` + /// and for `exec.Vm.init`. Storing the Nfa by-value here used to give + /// us a dangling pointer when the Regex was returned by value. + automaton: *nfa.Nfa, + flags: Flags, + n_groups: u32, + + /// Non-null iff the pattern is purely literal AND has zero capture + /// groups. Callers bypass the engine entirely and use SIMD `indexOf`. + pure_literal: ?[]const u8, + /// Non-null iff a contiguous substring is required to appear in every + /// match. Used as a coarse pre-filter. + required_literal: ?[]const u8, + /// Non-null iff every match's start position is at an occurrence of + /// this byte sequence. Used to skip directly to candidate start + /// positions via SIMD-accelerated indexOf, then run the DFA only at + /// those hits. + literal_prefix: ?[]const u8, + + /// Non-null iff the entire pattern is a literal alternation (`foo|bar|baz`). + /// At match time we run one `indexOfPos` scan per literal across the + /// haystack, then merge the results — far cheaper than the DFA for + /// patterns made of a few literal alternatives. Owned by `arena`. + literal_alternation: ?[]const []const u8, + + /// Non-null iff the whole pattern is a `class+` / `class{n,}` repeat — + /// the matcher then bypasses the DFA entirely and uses a precomputed + /// 256-byte membership table to extract every contiguous run of + /// class-matching bytes. One load + one branch per input byte; no + /// library call per match. `\d+` and `[a-z]+` are the canonical + /// cases. Lifetime tied to `arena`. + single_class_table: ?*const [256]bool, + + /// Built eagerly when the pattern is DFA-eligible (no captures, no + /// anchors, no case-insensitive). Null otherwise. Mutable because the + /// DFA fills its transition table lazily during matching. + dfa_engine: ?dfa.Dfa, + + pub fn compile(alloc: std.mem.Allocator, pattern: []const u8) !Regex { + return compileWithFlags(alloc, pattern, .{}); + } + + pub fn compileWithFlags(alloc: std.mem.Allocator, pattern: []const u8, flags: Flags) !Regex { + const arena = try alloc.create(std.heap.ArenaAllocator); + errdefer alloc.destroy(arena); + arena.* = std.heap.ArenaAllocator.init(alloc); + errdefer arena.deinit(); + + var p = parser.Parser.init(arena.allocator(), pattern); + const root = try p.parseRoot(); + + // Heap-allocate the Nfa on the arena. We point at it from both the + // Regex itself and the Dfa; storing by value would invalidate the + // address once compileWithFlags returns by value. + const automaton_ptr = try arena.allocator().create(nfa.Nfa); + automaton_ptr.* = try nfa.build(arena.allocator(), root, p.n_groups); + + const pure_lit: ?[]const u8 = if (flags.case_insensitive) + null + else if (p.n_groups != 0) + null + else + try prefilter.extractFullLiteral(arena.allocator(), root); + + const req_lit: ?[]const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractRequiredLiteral(arena.allocator(), root, 3); + + const lit_prefix: ?[]const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractLiteralPrefix(arena.allocator(), root, 3); + + const lit_alt: ?[]const []const u8 = if (flags.case_insensitive) + null + else + try prefilter.extractLiteralAlternation(arena.allocator(), root); + + // Single-class-repeat fast path: precompute a 256-byte membership + // table so the runtime scanner is a tight `if (table[byte])` loop. + const single_class: ?*const [256]bool = blk: { + if (flags.case_insensitive) break :blk null; + const cls = prefilter.extractSingleClassRepeat(root) orelse break :blk null; + const table = try arena.allocator().create([256]bool); + var b: u16 = 0; + while (b < 256) : (b += 1) { + table[b] = cls.contains(@intCast(b)); + } + break :blk table; + }; + + // Try to build a DFA. Falls back to null (=> Pike VM at runtime) + // when the pattern has captures, anchors, or grows the state + // table past the budget. Case-insensitive also skips DFA for v1 — + // adding case-folding to the bitmap test is straightforward but + // not done yet. + var dfa_engine: ?dfa.Dfa = null; + if (!flags.case_insensitive) { + if (dfa.Dfa.fromNfa(alloc, automaton_ptr, root, .{ .dot_all = flags.dot_all })) |built| { + dfa_engine = built; + } else |_| { + // Any DFA build error (HasCaptures, HasAnchors, TooManyStates, + // OOM) falls back silently. The Pike VM handles the same + // patterns correctly, just slower. + dfa_engine = null; + } + } + + return .{ + .arena = arena, + .parent_alloc = alloc, + .root = root, + .automaton = automaton_ptr, + .flags = flags, + .n_groups = p.n_groups, + .pure_literal = pure_lit, + .required_literal = req_lit, + .literal_prefix = lit_prefix, + .literal_alternation = lit_alt, + .single_class_table = single_class, + .dfa_engine = dfa_engine, + }; + } + + pub fn deinit(self: *Regex) void { + if (self.dfa_engine) |*d| d.deinit(); + self.arena.deinit(); + self.parent_alloc.destroy(self.arena); + self.* = undefined; + } + + /// First leftmost match, or null. Caller owns the returned Match. + pub fn search(self: *Regex, alloc: std.mem.Allocator, input: []const u8) !?Match { + if (self.required_literal) |lit| { + if (std.mem.indexOf(u8, input, lit) == null) return null; + } + if (self.pure_literal) |lit| return try literalFirst(alloc, lit, input); + if (self.literal_alternation) |lits| return try multiLiteralFirst(alloc, lits, input); + if (self.single_class_table) |tbl| return try singleClassFirst(alloc, tbl, input); + if (self.literal_prefix) |prefix| if (self.dfa_engine) |*d| + return try dfaFirstWithPrefix(alloc, d, input, prefix); + if (self.dfa_engine) |*d| return try dfaFirst(alloc, d, input); + + var vm = exec.Vm.init(alloc, self.automaton, self.flags); + return try vm.search(input); + } + + /// All non-overlapping matches, leftmost-first. + pub fn findAll(self: *Regex, alloc: std.mem.Allocator, input: []const u8) ![]Match { + if (self.required_literal) |lit| { + if (std.mem.indexOf(u8, input, lit) == null) { + return try alloc.alloc(Match, 0); + } + } + if (self.pure_literal) |lit| return try literalAll(alloc, lit, input); + if (self.literal_alternation) |lits| return try multiLiteralAll(alloc, lits, input); + if (self.single_class_table) |tbl| return try singleClassAll(alloc, tbl, input); + if (self.literal_prefix) |prefix| if (self.dfa_engine) |*d| + return try dfaAllWithPrefix(alloc, d, input, prefix); + if (self.dfa_engine) |*d| return try dfaAll(alloc, d, input); + + var vm = exec.Vm.init(alloc, self.automaton, self.flags); + return try vm.findAll(input); + } + + /// Replace every non-overlapping match. Backreferences (`\N`) honoured. + pub fn replaceAll(self: *Regex, alloc: std.mem.Allocator, input: []const u8, replacement: []const u8) ![]u8 { + const matches = try self.findAll(alloc, input); + defer { + for (matches) |*m| @constCast(m).deinit(alloc); + alloc.free(matches); + } + + var out: std.ArrayList(u8) = .empty; + defer out.deinit(alloc); + + var cursor: usize = 0; + for (matches) |m| { + try out.appendSlice(alloc, input[cursor..m.span.start]); + try appendReplacement(alloc, &out, replacement, m, input); + cursor = m.span.end; + } + try out.appendSlice(alloc, input[cursor..]); + return try out.toOwnedSlice(alloc); + } +}; + +// ── Literal fast paths ── + +fn literalFirst(alloc: std.mem.Allocator, needle: []const u8, haystack: []const u8) !?Match { + if (needle.len == 0) { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = 0, .end = 0 }; + return .{ .span = .{ .start = 0, .end = 0 }, .captures = captures }; + } + const idx = std.mem.indexOf(u8, haystack, needle) orelse return null; + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = idx, .end = idx + needle.len }; + return .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = captures }; +} + +fn literalAll(alloc: std.mem.Allocator, needle: []const u8, haystack: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + if (needle.len == 0) return try results.toOwnedSlice(alloc); + + var pos: usize = 0; + while (pos <= haystack.len) { + const idx = std.mem.indexOfPos(u8, haystack, pos, needle) orelse break; + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = idx, .end = idx + needle.len }; + try results.append(alloc, .{ .span = .{ .start = idx, .end = idx + needle.len }, .captures = captures }); + pos = idx + needle.len; + } + return try results.toOwnedSlice(alloc); +} + +// ── DFA wrappers ── +// +// Adapter from `dfa.Dfa`'s span-only output to the public `Match` shape +// (which carries a captures slice). DFA mode has no captures so we emit a +// 1-element captures array containing just the whole-match span. + +fn dfaFirst(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8) !?Match { + var p: usize = 0; + while (p <= input.len) : (p += 1) { + const end_opt = try d.matchAt(input, p); + if (end_opt) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = p, .end = end }; + return .{ .span = .{ .start = p, .end = end }, .captures = captures }; + } + } + return null; +} + +fn dfaAll(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8) ![]Match { + const spans = try d.findAll(alloc, input); + defer alloc.free(spans); + + var results = try alloc.alloc(Match, spans.len); + var built: usize = 0; + errdefer { + for (results[0..built]) |*m| m.deinit(alloc); + alloc.free(results); + } + for (spans) |span| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = span.start, .end = span.end }; + results[built] = .{ .span = .{ .start = span.start, .end = span.end }, .captures = captures }; + built += 1; + } + return results; +} + +/// Like `dfaFirst` but uses `prefix` to skip directly to candidate match +/// starts via `std.mem.indexOfPos`. Far fewer engine invocations for +/// sparse literal-prefixed patterns. +fn dfaFirstWithPrefix(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8, prefix: []const u8) !?Match { + var pos: usize = 0; + while (true) { + const hit = std.mem.indexOfPos(u8, input, pos, prefix) orelse return null; + if (try d.matchAt(input, hit)) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = hit, .end = end }; + return .{ .span = .{ .start = hit, .end = end }, .captures = captures }; + } + // DFA didn't accept at this hit (the prefix matched but the rest + // of the pattern didn't). Advance one byte past this hit and + // resume the indexOf scan. + pos = hit + 1; + } +} + +fn dfaAllWithPrefix(alloc: std.mem.Allocator, d: *dfa.Dfa, input: []const u8, prefix: []const u8) ![]Match { + var results: std.ArrayList(Match) = .empty; + errdefer { + for (results.items) |*m| @constCast(m).deinit(alloc); + results.deinit(alloc); + } + + var pos: usize = 0; + while (true) { + const hit = std.mem.indexOfPos(u8, input, pos, prefix) orelse break; + if (try d.matchAt(input, hit)) |end| { + const captures = try alloc.alloc(?Span, 1); + captures[0] = .{ .start = hit, .end = end }; + try results.append(alloc, .{ + .span = .{ .start = hit, .end = end }, + .captures = captures, + }); + // Skip past the match end. Zero-width match falls back to + // hit+1 so we don't infinite-loop. + pos = if (end > hit) end else hit + 1; + } else { + pos = hit + 1; + } + } + return try results.toOwnedSlice(alloc); +} + +fn appendReplacement( + alloc: std.mem.Allocator, + out: *std.ArrayList(u8), + replacement: []const u8, + m: Match, + input: []const u8, +) !void { + var i: usize = 0; + while (i < replacement.len) { + const c = replacement[i]; + if (c == '\\' and i + 1 < replacement.len) { + const n = replacement[i + 1]; + switch (n) { + '0'...'9' => { + const idx: usize = n - '0'; + if (idx < m.captures.len) { + if (m.captures[idx]) |span| try out.appendSlice(alloc, input[span.start..span.end]); + } + i += 2; + continue; + }, + 'n' => { + try out.append(alloc, '\n'); + i += 2; + continue; + }, + 't' => { + try out.append(alloc, '\t'); + i += 2; + continue; + }, + 'r' => { + try out.append(alloc, '\r'); + i += 2; + continue; + }, + '\\' => { + try out.append(alloc, '\\'); + i += 2; + continue; + }, + else => { + try out.append(alloc, '\\'); + i += 1; + continue; + }, + } + } + try out.append(alloc, c); + i += 1; + } +} + +// ── Single-class-repeat fast paths ── +// +// For `class+` / `class{n,}` patterns the whole match is "consecutive +// bytes in `class`". indexOfAnyPos finds the first matching byte (start); +// indexOfNonePos finds the first non-matching byte (end). Both are +// SIMD-accelerated in Zig stdlib. Two SIMD scans per match — vs the +// DFA's per-byte transition lookup. + +fn singleClassFirst(alloc: std.mem.Allocator, table: *const [256]bool, input: []const u8) !?Match { + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + return .{ .span = .{ .start = start, .end = i }, .captures = caps }; + } + i += 1; + } + return null; +} + +fn singleClassAll(alloc: std.mem.Allocator, table: *const [256]bool, input: []const u8) ![]Match { + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var i: usize = 0; + while (i < input.len) { + if (table[input[i]]) { + const start = i; + while (i < input.len and table[input[i]]) i += 1; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = start, .end = i }; + try out.append(alloc, .{ .span = .{ .start = start, .end = i }, .captures = caps }); + } else { + i += 1; + } + } + return try out.toOwnedSlice(alloc); +} + +test "single-class fast path: \\d+ matches digits only" { + var r = try Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + try std.testing.expect(r.single_class_table != null); + const ms = try r.findAll(std.testing.allocator, "abc 42 def 1234 xyz"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); + try std.testing.expectEqual(@as(usize, 4), ms[0].span.start); + try std.testing.expectEqual(@as(usize, 6), ms[0].span.end); +} + +test "single-class fast path: [a-z]+ ignores uppercase" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + try std.testing.expect(r.single_class_table != null); + const ms = try r.findAll(std.testing.allocator, "Hello World"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +// ── Multi-literal alternation fast paths ── +// +// For `lit1|lit2|...|litN` patterns we scan the haystack once per literal +// using std.mem.indexOfPos (memchr-backed when the literal is one byte, +// SIMD-accelerated memmem otherwise). On each step we pick the earliest +// candidate across all literals and emit it. The amortised cost is +// O(needles * input.len) but each scan is on Zig stdlib's tight loop — +// faster than a DFA whose hot path costs ~5 ns/byte. ripgrep accomplishes +// the same with Teddy (a SIMD-pshufb-based multi-string algorithm); this +// is the scalar equivalent. + +const LiteralHit = struct { + start: usize, + len: usize, + /// Index of the matching literal in the alternation. Lower wins ties + /// (leftmost-first alternation semantics, matching Python re). + lit_idx: u32, +}; + +fn hitLessThan(_: void, a: LiteralHit, b: LiteralHit) bool { + if (a.start != b.start) return a.start < b.start; + return a.lit_idx < b.lit_idx; +} + +/// Collect every occurrence of every literal across the input in one +/// pass per literal. Each pass is `std.mem.indexOfPos` (memchr/memmem in +/// Zig stdlib). For N literals on H bytes that's O(N · H), then a sort +/// of M total hits at O(M log M). Avoids the O(N · H · M) explosion the +/// naïve "earliest hit per step" scanner showed on dense workloads. +fn collectAllLiteralHits(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) ![]LiteralHit { + var hits: std.ArrayList(LiteralHit) = .empty; + errdefer hits.deinit(alloc); + for (literals, 0..) |lit, idx| { + if (lit.len == 0) continue; + var pos: usize = 0; + while (std.mem.indexOfPos(u8, input, pos, lit)) |found| { + try hits.append(alloc, .{ + .start = found, + .len = lit.len, + .lit_idx = @intCast(idx), + }); + pos = found + 1; + } + } + const slice = try hits.toOwnedSlice(alloc); + std.mem.sort(LiteralHit, slice, {}, hitLessThan); + return slice; +} + +fn multiLiteralFirst(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) !?Match { + // search() only needs the first hit — we can stop as soon as we find + // the leftmost across all literals. Cheaper than the full collect. + var best: ?LiteralHit = null; + for (literals, 0..) |lit, idx| { + if (lit.len == 0) continue; + const hit = std.mem.indexOf(u8, input, lit) orelse continue; + if (best) |b| { + if (hit < b.start or (hit == b.start and idx < b.lit_idx)) { + best = LiteralHit{ .start = hit, .len = lit.len, .lit_idx = @intCast(idx) }; + } + } else { + best = LiteralHit{ .start = hit, .len = lit.len, .lit_idx = @intCast(idx) }; + } + } + const h = best orelse return null; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = h.start, .end = h.start + h.len }; + return .{ .span = .{ .start = h.start, .end = h.start + h.len }, .captures = caps }; +} + +fn multiLiteralAll(alloc: std.mem.Allocator, literals: []const []const u8, input: []const u8) ![]Match { + const all_hits = try collectAllLiteralHits(alloc, literals, input); + defer alloc.free(all_hits); + + var out: std.ArrayList(Match) = .empty; + errdefer { + for (out.items) |*m| @constCast(m).deinit(alloc); + out.deinit(alloc); + } + var cursor: usize = 0; + for (all_hits) |h| { + // Skip hits that overlap with an already-emitted match. With the + // hits sorted by (start, lit_idx), this also collapses the case + // where two literals would have matched at the same position — + // leftmost-first wins. + if (h.start < cursor) continue; + const caps = try alloc.alloc(?Span, 1); + caps[0] = .{ .start = h.start, .end = h.start + h.len }; + try out.append(alloc, .{ + .span = .{ .start = h.start, .end = h.start + h.len }, + .captures = caps, + }); + cursor = h.start + h.len; + } + return try out.toOwnedSlice(alloc); +} + +test "multi-literal alternation findAll" { + var r = try Regex.compile(std.testing.allocator, "cat|dog|bird"); + defer r.deinit(); + try std.testing.expect(r.literal_alternation != null); + const ms = try r.findAll(std.testing.allocator, "the cat saw a dog and a bird"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 3), ms.len); +} + +test "module imports compile" { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(ast); + std.testing.refAllDecls(parser); + std.testing.refAllDecls(nfa); + std.testing.refAllDecls(exec); + std.testing.refAllDecls(prefilter); + std.testing.refAllDecls(dfa); +} + +test "Regex.search basic" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + var m = (try r.search(std.testing.allocator, "Hello World")).?; + defer m.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 1), m.span.start); + try std.testing.expectEqual(@as(usize, 5), m.span.end); +} + +test "Regex.findAll" { + var r = try Regex.compile(std.testing.allocator, "\\d+"); + defer r.deinit(); + const ms = try r.findAll(std.testing.allocator, "abc 42 xyz 1234"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "Regex pure literal fast path" { + var r = try Regex.compile(std.testing.allocator, "compileAllocFlags"); + defer r.deinit(); + try std.testing.expect(r.pure_literal != null); + try std.testing.expectEqualStrings("compileAllocFlags", r.pure_literal.?); + const ms = try r.findAll(std.testing.allocator, "abc compileAllocFlags xyz compileAllocFlags"); + defer { + for (ms) |*m| @constCast(m).deinit(std.testing.allocator); + std.testing.allocator.free(ms); + } + try std.testing.expectEqual(@as(usize, 2), ms.len); +} + +test "Regex DFA engine is built when eligible" { + var r = try Regex.compile(std.testing.allocator, "[a-z]+"); + defer r.deinit(); + // [a-z]+ has no captures, no anchors → DFA should be built. + try std.testing.expect(r.dfa_engine != null); +} + +test "Regex falls back to Pike VM with captures" { + var r = try Regex.compile(std.testing.allocator, "(abc)"); + defer r.deinit(); + try std.testing.expect(r.dfa_engine == null); +} + +test "Regex falls back to Pike VM with anchors" { + var r = try Regex.compile(std.testing.allocator, "^foo"); + defer r.deinit(); + try std.testing.expect(r.dfa_engine == null); +} + +test "Regex required-literal pre-filter skips haystack with no candidates" { + var r = try Regex.compile(std.testing.allocator, "hello\\d+"); + defer r.deinit(); + try std.testing.expectEqualStrings("hello", r.required_literal.?); + const ms = try r.findAll(std.testing.allocator, "no candidates anywhere here"); + defer std.testing.allocator.free(ms); + try std.testing.expectEqual(@as(usize, 0), ms.len); +} + +test "Regex search: literal after + quantifier matches a variable repeat (regression)" { + // The literal-prefix fast path used to extract `helo` for `hel+o`, then + // require `helo` as a contiguous start hint — so `helllo` (he + lll + o) + // was missed entirely. The prefix must be `hel`. See prefilter.collectPrefix. + var r = try Regex.compile(std.testing.allocator, "hel+o"); + defer r.deinit(); + if (try r.search(std.testing.allocator, "helllo")) |m| { + @constCast(&m).deinit(std.testing.allocator); + } else { + return error.TestUnexpectedResult; + } + // Negative: zero `l` must still not satisfy `l+`. + try std.testing.expect((try r.search(std.testing.allocator, "heo")) == null); +} + +test "Regex.replaceAll with backreference" { + var r = try Regex.compile(std.testing.allocator, "(\\w+)@(\\w+)"); + defer r.deinit(); + const out = try r.replaceAll(std.testing.allocator, "alice@example bob@host", "\\2/\\1"); + defer std.testing.allocator.free(out); + try std.testing.expectEqualStrings("example/alice host/bob", out); +} diff --git a/vendor/nanoregex/tests/parity/fixtures/001_literal.txt b/vendor/nanoregex/tests/parity/fixtures/001_literal.txt new file mode 100644 index 00000000..24a9ec9e --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/001_literal.txt @@ -0,0 +1,3 @@ +abc + +the quick abc jumps over abc lazy abc diff --git a/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt b/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt new file mode 100644 index 00000000..d195989f --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/002_dot_star.txt @@ -0,0 +1,3 @@ +a.*b + +start aXYZb middle a___b end diff --git a/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt b/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt new file mode 100644 index 00000000..e2ff7827 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/003_char_class.txt @@ -0,0 +1,3 @@ +[a-z]+ + +Hello World 42 diff --git a/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt b/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt new file mode 100644 index 00000000..bb24a3bf --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/004_anchors.txt @@ -0,0 +1,5 @@ +^foo +m +foo +bar foo +foo bar diff --git a/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt b/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt new file mode 100644 index 00000000..764da967 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/005_alternation.txt @@ -0,0 +1,3 @@ +cat|dog|bird + +the cat saw a dog and a bird diff --git a/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt b/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt new file mode 100644 index 00000000..9487302b --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/006_shorthand_digit.txt @@ -0,0 +1,3 @@ +\d+ + +version 1.2.345 build 678 diff --git a/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt b/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt new file mode 100644 index 00000000..f9dd7040 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/007_group_capture.txt @@ -0,0 +1,3 @@ +(\w+)@(\w+) + +alice@example bob@host diff --git a/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt b/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt new file mode 100644 index 00000000..a6140ede --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/008_lazy_star.txt @@ -0,0 +1,3 @@ +a.*?b + +start aXXXb middle aYYYb end diff --git a/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt b/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt new file mode 100644 index 00000000..d421e363 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/009_lazy_plus.txt @@ -0,0 +1,3 @@ +<.+?> + + diff --git a/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt b/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt new file mode 100644 index 00000000..58c877e7 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/010_nested_groups.txt @@ -0,0 +1,3 @@ +((a)(b))+ + +ababab xyz abab diff --git a/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt b/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt new file mode 100644 index 00000000..7c850910 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/011_escape_dot.txt @@ -0,0 +1,3 @@ +\. + +foo.bar baz.qux .end diff --git a/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt b/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt new file mode 100644 index 00000000..4b919c6c --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/012_word_boundary.txt @@ -0,0 +1,3 @@ +\bcat\b + +the cat sat on a catnap but a cat is a cat diff --git a/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt b/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt new file mode 100644 index 00000000..0e4a6f58 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/013_dollar_multiline.txt @@ -0,0 +1,6 @@ +foo$ +m +foo +bar foo +foo bar +foo diff --git a/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt b/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt new file mode 100644 index 00000000..3e78a7ff --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/014_case_insensitive.txt @@ -0,0 +1,3 @@ +hello +i +HELLO Hello hello hElLo diff --git a/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt b/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt new file mode 100644 index 00000000..94be27f4 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/015_dot_all.txt @@ -0,0 +1,6 @@ +a.b +s +aXb +a +b a +b diff --git a/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt b/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt new file mode 100644 index 00000000..3c4f1201 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/016_counted_range.txt @@ -0,0 +1,3 @@ +a{2,4} + +a aa aaa aaaa aaaaa aaaaaa diff --git a/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt b/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt new file mode 100644 index 00000000..b4c3eb6c --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/017_alternation_anchors.txt @@ -0,0 +1,6 @@ +^cat|dog$ +m +cat sees dog +dog sees cat +catnap +bulldog diff --git a/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt b/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt new file mode 100644 index 00000000..ef772ce5 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/018_catastrophic_backtrack.txt @@ -0,0 +1,3 @@ +(a+)+b + +aaaaaaaaaab end diff --git a/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt b/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt new file mode 100644 index 00000000..4bfefaec --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/019_char_class_negation.txt @@ -0,0 +1,3 @@ +[^aeiou]+ + +hello world diff --git a/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt b/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt new file mode 100644 index 00000000..4b71115e --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/020_non_capturing_group.txt @@ -0,0 +1,3 @@ +(?:foo|bar)+ + +foo barbar foobarfoo xxx baz diff --git a/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt b/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt new file mode 100644 index 00000000..98b3b539 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/021_optional_quantifier.txt @@ -0,0 +1,3 @@ +colou?r + +color and colour are both valid diff --git a/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt b/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt new file mode 100644 index 00000000..ed8924a6 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/022_min_zero_quantifier.txt @@ -0,0 +1,3 @@ +x{0,3}y + +y xy xxy xxxy xxxxy diff --git a/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt b/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt new file mode 100644 index 00000000..da21f6ba --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/023_email_like.txt @@ -0,0 +1,3 @@ +[a-zA-Z0-9_.]+@[a-zA-Z0-9.]+ + +contact me at alice@example.com or bob.test@host.io diff --git a/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt b/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt new file mode 100644 index 00000000..118ddb0f --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/024_string_anchor.txt @@ -0,0 +1,4 @@ +\Aabc +m +abc +abc again diff --git a/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt b/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt new file mode 100644 index 00000000..75621efa --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/025_function_def_pattern.txt @@ -0,0 +1,5 @@ +fn [A-Za-z_][A-Za-z0-9_]*\( + +pub fn main() void +fn helper(x: i32) i32 +const fn_ptr = foo diff --git a/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt b/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt new file mode 100644 index 00000000..91855391 --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/026_unbounded_min.txt @@ -0,0 +1,3 @@ +a{3,} + +a aa aaa aaaa aaaaaaaa diff --git a/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt b/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt new file mode 100644 index 00000000..ba281a7a --- /dev/null +++ b/vendor/nanoregex/tests/parity/fixtures/027_zero_width_loop.txt @@ -0,0 +1,3 @@ +a* + +bbb diff --git a/vendor/nanoregex/tests/parity/run.sh b/vendor/nanoregex/tests/parity/run.sh new file mode 100755 index 00000000..e308f410 --- /dev/null +++ b/vendor/nanoregex/tests/parity/run.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# parity test harness: zigregex vs python re +# +# Each fixture in $FIXTURES is a 3+ line text file: +# line 1: regex pattern (raw, no quoting) +# line 2: flags (comma-separated subset of: i, m, s — empty line if none) +# line 3+: haystack (joined with \n if multiline) +# +# For each fixture we run python3's re.findall and the zigregex_probe +# binary on the same input. They must produce byte-identical output. +# +# Until exec.zig lands, the matcher is a panic — we tolerate "matcher not +# yet implemented" on the zig side and only verify the python reference +# itself runs. Flip $REQUIRE_MATCH to 1 once the matcher is wired. + +set -uo pipefail + +PROBE="${1:-}" +FIXTURES="${2:-tests/parity/fixtures}" +REQUIRE_MATCH="${REQUIRE_MATCH:-1}" + +if [ -z "$PROBE" ] || [ ! -x "$PROBE" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +if [ ! -d "$FIXTURES" ]; then + echo "fixtures dir not found: $FIXTURES" >&2 + exit 2 +fi + +pass=0 +fail=0 +skip=0 + +for f in "$FIXTURES"/*.txt; do + [ -e "$f" ] || continue + name=$(basename "$f" .txt) + + pattern=$(sed -n '1p' "$f") + flags=$(sed -n '2p' "$f") + haystack=$(sed -n '3,$p' "$f") + + py_out=$( + PATTERN="$pattern" FLAGS="$flags" HAYSTACK="$haystack" \ + python3 - <<'PY' +import os, re, sys +pattern = os.environ["PATTERN"] +flags_str = os.environ.get("FLAGS", "") +haystack = os.environ["HAYSTACK"] +flag_bits = 0 +if "i" in flags_str: flag_bits |= re.IGNORECASE +if "m" in flags_str: flag_bits |= re.MULTILINE +if "s" in flags_str: flag_bits |= re.DOTALL +try: + pat = re.compile(pattern, flag_bits) +except re.error as e: + print(f"PARSE_ERROR: {e}") + sys.exit(0) +for m in pat.finditer(haystack): + print(f"{m.start()}..{m.end()}\t{m.group(0)}") +PY + ) + + if [ "$REQUIRE_MATCH" != "1" ]; then + # Phase 1: just verify the python reference computes a result. + # We don't yet compare against zig output because the matcher + # panics. Once exec.zig lands, set REQUIRE_MATCH=1. + skip=$((skip + 1)) + echo "SKIP $name (python ref: $(echo "$py_out" | wc -l | tr -d ' ') lines)" + continue + fi + + zig_out=$("$PROBE" "$pattern" "$haystack" "$flags" 2>&1) + if [ "$py_out" = "$zig_out" ]; then + pass=$((pass + 1)) + echo "PASS $name" + else + fail=$((fail + 1)) + echo "FAIL $name" + echo " pattern: $pattern" + echo " py: $(echo "$py_out" | head -5)" + echo " zig: $(echo "$zig_out" | head -5)" + fi +done + +echo "" +echo "parity: $pass passed, $fail failed, $skip skipped" +exit $(( fail > 0 ? 1 : 0 )) From 4ad082f3ca9e2696d39225235f53b96f5d9e859e Mon Sep 17 00:00:00 2001 From: Codegraff Date: Sat, 11 Jul 2026 12:14:22 +0800 Subject: [PATCH 2/2] fix(watcher): serialize snapshot commits after stat fan Keep parallel workers limited to statting disjoint entry slices, then record snapshots in deterministic discovery order after every join. This removes shared worker error state and avoids concurrent sequence assignment. Add sequential-vs-parallel snapshot sequence and size parity coverage. --- src/test_index.zig | 24 +++++++++++++++++++++ src/watcher.zig | 52 +++++++++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/test_index.zig b/src/test_index.zig index 6537ac3f..5cf0aa87 100644 --- a/src/test_index.zig +++ b/src/test_index.zig @@ -586,6 +586,19 @@ test "watcher: parallel initial scan matches sequential results" { var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); + const previous_load_workers = if (cio.posixGetenv("CODEDB_LOAD_WORKERS")) |value| + try testing.allocator.dupe(u8, value) + else + null; + defer { + if (previous_load_workers) |value| { + cio.posixSetenv("CODEDB_LOAD_WORKERS", value); + testing.allocator.free(value); + } else { + cio.posixUnsetenv("CODEDB_LOAD_WORKERS"); + } + } + try tmp_dir.dir.createDirPath(io, "src/nested"); try tmp_dir.dir.writeFile(io, .{ .sub_path = "src/main.zig", .data = "const std = @import(\"std\");\npub fn alpha() void {}\n// TODO: keep me\n" }); try tmp_dir.dir.writeFile(io, .{ .sub_path = "src/nested/util.py", .data = "def beta():\n return 42\n# TODO later\n" }); @@ -600,6 +613,7 @@ test "watcher: parallel initial scan matches sequential results" { var explorer_seq = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_seq.deinit(); explorer_seq.setRoot(io, root); + cio.posixSetenv("CODEDB_LOAD_WORKERS", "1"); try watcher.initialScanWithWorkerCount(io, &store_seq, &explorer_seq, root, testing.allocator, false, 1); var store_par = Store.init(testing.allocator); @@ -607,6 +621,7 @@ test "watcher: parallel initial scan matches sequential results" { var explorer_par = Explorer.init(testing.allocator, Explorer.DEFAULT_CONTENT_CACHE_CAPACITY); defer explorer_par.deinit(); explorer_par.setRoot(io, root); + cio.posixSetenv("CODEDB_LOAD_WORKERS", "4"); try watcher.initialScanWithWorkerCount(io, &store_par, &explorer_par, root, testing.allocator, false, 4); const tree_seq = try explorer_seq.getTree(testing.allocator, false); @@ -622,6 +637,15 @@ test "watcher: parallel initial scan matches sequential results" { try testing.expectEqual(seq_hits.len, par_hits.len); try testing.expectEqual(explorer_seq.outlines.count(), explorer_par.outlines.count()); + + // Parallel stat workers only fill disjoint entry slots; snapshots are + // committed serially afterward, preserving discovery-order sequence IDs. + for ([_][]const u8{ "src/main.zig", "src/nested/util.py", "README.md" }) |path| { + const seq_version = store_seq.getLatest(path) orelse return error.TestUnexpectedResult; + const par_version = store_par.getLatest(path) orelse return error.TestUnexpectedResult; + try testing.expectEqual(seq_version.seq, par_version.seq); + try testing.expectEqual(seq_version.size, par_version.size); + } } test "watcher: rolling trigram shards match canonical masks exactly" { diff --git a/src/watcher.zig b/src/watcher.zig index 539dc963..d570fb9c 100644 --- a/src/watcher.zig +++ b/src/watcher.zig @@ -85,6 +85,7 @@ const FileMap = std.StringHashMap(FileState); const InitialScanEntry = struct { path: []u8, size: u64, + stat_succeeded: bool, skip_trigram: bool, }; @@ -507,20 +508,15 @@ pub fn trigramFileCap() usize { return std.fmt.parseInt(usize, raw, 10) catch 15_000; } -// statScanForInitial: helper matching loadSnapshotFast's freshnessScan pattern. -// Paths collected serially first (in caller), then stats fanned via chunks to -// workers; side-effect is stat + recordSnapshot + storing the size back into the -// entry. Per-stat errors are swallowed like the ref; record failures set shared -// err_out and stop the worker. -fn statScanForInitial(io: std.Io, store: *Store, dir: std.Io.Dir, recs: []InitialScanEntry, err_out: *?anyerror) void { +// Paths are collected serially first, then stat calls fan out across disjoint +// chunks. Workers only publish size/stat_succeeded into their own entries; the +// caller records snapshots serially in discovery order after every join. That +// keeps sequence assignment deterministic and avoids shared worker error state. +fn statScanForInitial(io: std.Io, dir: std.Io.Dir, recs: []InitialScanEntry) void { for (recs) |*record| { - if (err_out.* != null) return; const ds = dir.statFile(io, record.path, .{}) catch continue; record.size = ds.size; - _ = store.recordSnapshot(record.path, ds.size, 0) catch |e| { - err_out.* = e; - return; - }; + record.stat_succeeded = true; } } @@ -540,14 +536,14 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat try entries.append(allocator, .{ .path = try allocator.dupe(u8, entry.path), .size = 0, + .stat_succeeded = false, .skip_trigram = false, // set after parallel stat fan }); } - // Stats fanned out to workers (exact pattern from loadSnapshotFast: Pass A serial collect, - // Pass B chunk+spawn to statScanForInitial on disjoint slices; swallows per-stat errors, - // propagates record error, uses env/256/4 idiom, static chunks, join). + // Stats fan out across disjoint slices; per-stat errors are swallowed. After + // joining, snapshots are recorded serially so sequence order remains stable + // and record errors propagate without cross-thread error sharing. if (entries.items.len > 0) { - var build_err: ?anyerror = null; const want_workers = blk: { if (cio.posixGetenv("CODEDB_LOAD_WORKERS")) |raw| { const parsed = std.fmt.parseInt(usize, raw, 10) catch 0; @@ -558,10 +554,17 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat break :blk @min(@as(usize, @intCast(cpu_count)), 4); }; const n_workers = @max(@as(usize, 1), @min(want_workers, entries.items.len)); - if (n_workers <= 1) { - statScanForInitial(io, store, dir, entries.items, &build_err); - } else if (allocator.alloc(std.Thread, n_workers)) |threads| { + stat_fan: { + if (n_workers <= 1) { + statScanForInitial(io, dir, entries.items); + break :stat_fan; + } + const threads = allocator.alloc(std.Thread, n_workers) catch { + statScanForInitial(io, dir, entries.items); + break :stat_fan; + }; defer allocator.free(threads); + const chunk = entries.items.len / n_workers; const rem = entries.items.len % n_workers; var off: usize = 0; @@ -573,22 +576,23 @@ fn collectInitialScanEntries(io: std.Io, store: *Store, dir: std.Io.Dir, allocat off += chunk + extra; const recs = entries.items[start..off]; if (spawn_failed) { - statScanForInitial(io, store, dir, recs, &build_err); + statScanForInitial(io, dir, recs); continue; } - if (std.Thread.spawn(.{}, statScanForInitial, .{ io, store, dir, recs, &build_err })) |t| { + if (std.Thread.spawn(.{}, statScanForInitial, .{ io, dir, recs })) |t| { threads[spawned] = t; spawned += 1; } else |_| { - statScanForInitial(io, store, dir, recs, &build_err); + statScanForInitial(io, dir, recs); spawn_failed = true; } } for (threads[0..spawned]) |t| t.join(); - } else |_| { - statScanForInitial(io, store, dir, entries.items, &build_err); } - if (build_err) |e| return e; + for (entries.items) |entry| { + if (!entry.stat_succeeded) continue; + _ = try store.recordSnapshot(entry.path, entry.size, 0); + } } // Set skip_trigram using serial count (preserves original cap semantics on discovery order). var file_count: usize = 0;