diff --git a/COOKBOOK.md b/COOKBOOK.md index 65a730d..55f0824 100644 --- a/COOKBOOK.md +++ b/COOKBOOK.md @@ -185,6 +185,9 @@ hprscript -p 'https?://[a-zA-Z0-9.\-]*[\x{0400}-\x{04FF}][a-zA-Z0-9.\-]*' suspec ### Editing files 33. [Guarded code edits (`hprscript edit`)](#33-guarded-code-edits-hprscript-edit) +### AI-agent workflows +34. [Context retrieval & ranking for LLM agents](#34-context-retrieval--ranking-for-llm-agents) + --- # Logs & Observability @@ -3387,6 +3390,110 @@ undo. A rerun of the same command is safe — already-applied sites report --- +## 34. Context retrieval & ranking for LLM agents + +hprscript's edge over embedding-based RAG: relevance, ranking, and +context-budget-fit computed deterministically and freshly from the live +tree — no index to go stale, no setup, fully explainable. These recipes are +the ones an agent reaches for when the question isn't "does X exist" but +"where should I look, and what's the smallest slice of code that answers +the question." + +### 34.1 "Where's the code for X?" in one call — hotspot ranking + +**Problem:** N `grep` hits across a dozen files, no sense of which one is actually *the* place to look. `-hotspots` ranks every matching file by a rarity/coverage/proximity score (files matching more of the queried terms, more densely, in fewer overall files, rank higher) and reports each file's single densest match window. + +**Input:** Source tree. + +```bash +hprscript -p 'RetryPolicy' -p 'backoff' -hotspots 5 -llm -glob '**/*.go' +# → src/retry/policy.go:12-64 score=1.4 patterns=p0,p1 +# → src/client/http.go:88-88 score=0.3 patterns=p0 +``` + +### 34.2 Pack the best context into a token budget + +**Problem:** You need "everything relevant to X" but have a hard context budget. `-budget N` ranks every matching file (same formula as `-hotspots`) and renders them score-descending in scope-aware chunks until `N` bytes are spent — degrading a file that doesn't fit to a one-line summary, then to a named "dropped" entry, rather than truncating mid-file or silently omitting anything. + +**Input:** Source tree. + +```bash +hprscript -p 'AuthMiddleware' -p 'validateToken' -budget 6000 -glob '**/*.go' +# → src/auth/middleware.go +# → 12-40 func AuthMiddleware +# → ... +# → src/auth/legacy.go:9-9 score=0.2 patterns=p1 (compact — full render didn't fit the budget) +# → --- budget: 2 file(s) in full, 1 compact, 0 dropped --- +``` + +### 34.3 Compact excerpts instead of whole functions + +**Problem:** A block-extracted function body can be hundreds of lines when only two of them matter. `-elide` prints the signature and matched lines with a little context, folding everything else into `… (+N lines)` — the shape a human skimming the function would produce by eye, and far cheaper than `-block-open`/`-block-close`'s full body for a large function. + +**Input:** Source tree. + +```bash +hprscript -p 'RetryPolicy' -elide -scope auto -glob '**/*.go' +# → src/retry/policy.go +# → 12-64 func NewRetryPolicy +# → func NewRetryPolicy(opts ...Option) *RetryPolicy { +# → … (+48 lines) +# → return p +# → } +``` + +### 34.4 Find identifiers regardless of naming convention + +**Problem:** The symbol you're hunting could be `parseConfig`, `parse_config`, or `ConfigParser` depending on who wrote it and when — a regex has to enumerate every casing variant by hand. `-ident 'term1 term2'` splits every identifier on camelCase/snake_case/acronym/digit boundaries and matches when all given terms appear as subtokens, in any order, any casing. + +**Input:** Source tree. + +```bash +hprscript -ident 'parse config' -glob '**/*.go' +# → matches parseConfig, parse_config, ConfigParser, PARSE_CONFIG, ... +``` + +Each `-ident` group is a first-class pattern (`ident0`, `ident1`, …) — usable in `-name`, `-near`/`-far`, and `-file-where` exactly like a `-p` pattern. + +### 34.5 Don't re-pay tokens for unchanged context across agent turns + +**Problem:** An agent iterating — search, edit, search again — re-reads the same unchanged function every round. `-seen ` hashes each rendered chunk's raw source and, on a later run against the same state file, collapses anything unchanged to a one-line pointer instead of the full body. + +**Input:** Source tree, across repeated invocations in the same session. + +```bash +hprscript -p 'AuthMiddleware' -elide -seen .hpr-seen -glob '**/*.go' +# first call: full chunks, .hpr-seen written +# every later call this session, for functions you haven't touched: +# → 12-40 func AuthMiddleware (unchanged, already shown) +``` + +Works with `-budget` too. A chunk `-budget` measures to decide whether it fits, then discards in favor of a compact summary or a drop, is never recorded as "shown" — a later run with more budget still renders it in full. + +### 34.6 Filter by how actively a file is being worked on + +**Problem:** "Files with lots of TODOs" is noisy across a whole repo; "files with lots of TODOs that are also under active development" is the actual review queue. `churn(days)` extends `-file-where` with a git-commit-count condition alongside the existing pattern-presence predicate; `count(pat)` raises the bar from "matched once" to "matched at least N times." + +**Input:** A git repository. + +```bash +hprscript -p TODO -name t -file-where 'count(t) >= 3 AND churn(30) > 2' -llm -glob '**/*.go' +``` + +`churn(30)` runs one `git log --since=30.days.ago` call regardless of how many files match — never one subprocess per file. `lang == go` (only `==`/`!=`) is also available, for globs that sweep in more than one language. + +### 34.7 Ranked file listing instead of walk order + +**Problem:** `-f`/`-c` stream in filesystem order, which has no relationship to relevance. `-order-by score` sorts the file list by the same ranking formula `-hotspots` uses; `count` sorts by total matches; `path` sorts lexicographically. + +**Input:** Source tree. + +```bash +hprscript -p TODO -c -order-by score -glob '**/*.go' +``` + +--- + ## See also - **[HPRSCRIPT.md](HPRSCRIPT.md)** — full reference for every flag, the script-mode JSON DSL, regex syntax, exit codes, and the agent-focused cookbook. diff --git a/HPRSCRIPT.md b/HPRSCRIPT.md index fedc54c..88aeadb 100644 --- a/HPRSCRIPT.md +++ b/HPRSCRIPT.md @@ -83,9 +83,11 @@ Positional file/dir args after `-s`/`-script` (or after a positional script file | `-p ` | Case-sensitive search pattern (repeatable for multi-pattern, all match in one pass) | | `-pi ` | Case-insensitive search pattern (HS `CASELESS`; folds Unicode by default; repeatable, mixable with `-p`) | | `-F ` / `-Fi ` | Fixed-string pattern (case-sensitive / -insensitive) — matched literally, no regex interpretation. Repeatable, mixable with `-p`/`-pi`. | -| `-name ` | Name the preceding `-p`/`-pi`/`-F`/`-Fi`: the id (`[A-Za-z_]\w*`) replaces the auto `p` in `pat`, `$PAT_ID`, `-llm` tags, relations, and `-file-where`. | +| `-name ` | Name the preceding `-p`/`-pi`/`-F`/`-Fi`/`-ident`: the id (`[A-Za-z_]\w*`) replaces the auto `p`/`ident` in `pat`, `$PAT_ID`, `-llm` tags, relations, and `-file-where`. | | `-patterns-from ` | Load additional patterns from a JSONL rule file — one `{"id","regexp"\|"literal","case_insensitive","word_boundary","utf8","ref"}` object per line, `#` comments allowed. Repeatable. See [Fixed strings & pattern files](#fixed-strings--pattern-files--f--fi--patterns-from). | -| `-file-where ` | Per-file predicate over pattern ids (`'err AND NOT recovery'`). See [Per-file conditions](#per-file-conditions--file-where). | +| `-ident ''` | Match identifiers whose subtokens include ALL given space-separated terms, regardless of casing/separator (`parseConfig` ~ `parse_config`). Repeatable = OR. See [Identifier matching](#identifier-matching--ident). | +| `-file-where ` | Per-file predicate: pattern ids, plus `count(pat) > n` / `churn(days) > n` / `lang == name` conditions (`'err AND NOT recovery'`, `'churn(30) > 2'`). See [Per-file conditions](#per-file-conditions--file-where). | +| `-order-by ` | Sort `-f`/`-c` output by `score`/`count`/`path` instead of walk order. See [Sorting file-grouped output](#sorting-file-grouped-output--order-by). | | `-records line` | With `-absent`: record-level absence — one JSON record per non-empty line lacking each pattern. See [Record-level absence](#record-level-absence--records-line). | | `-glob ` | Scan glob (e.g. `"**/*.go"`; repeatable). Brace alternation is supported (`"src/**/*.{ts,tsx}"`), and absolute bases work too (`"/var/log/**/*.log"`). | | `-exclude ` | Exclude rule (repeatable). Three forms: glob (`"*.log"`), bare directory name (`"vendor"` skips any `vendor/` dir), path prefix with `/` (`"src/generated/"`). | @@ -136,6 +138,7 @@ Positional file/dir args after `-s`/`-script` (or after a positional script file | `-format ` | Custom one-line template | | `-absent` | Files where the pattern is **not** found (like `grep -L`) | | `-llm` | Token-efficient plain text grouped by file (LLM-friendly). See [LLM output mode](#llm-output-mode). | +| `-elide` | Scope-aware chunks with unmatched interior lines folded away. See [Elided scope output](#elided-scope-output--elide). | `-format` template tokens (substituted per match): @@ -384,6 +387,39 @@ attributed to the entry (`pat`) that produced it. --- +## Identifier matching (`-ident`) + +Regex search misses the #1 source of missed hits in code: naming-convention drift. Searching for `parseConfig` doesn't find `parse_config` or `ConfigParser` — same concept, different casing. `-ident 'term1 term2 …'` matches identifiers by their **subtokens** instead of literal text: it splits every identifier in the scanned files on `_`, camelCase boundaries, acronym runs (`HTTPServer` → `HTTP`, `Server`), and letter/digit transitions (`utf8` → `utf`, `8`), then checks whether all of the given terms appear (case-insensitively) as one of those subtokens. + +```bash +hprscript -ident 'parse config' -glob '**/*.go' +# → matches parseConfig, parse_config, ConfigParser, PARSE_CONFIG, ... +``` + +Terms inside one `-ident` invocation **AND** together (the identifier must contain all of them); repeat `-ident` for separate groups that **OR** together: + +```bash +# Either "parse config" or a bare "validate" hit. +hprscript -ident 'parse config' -ident validate -glob '**/*.go' +``` + +Each `-ident` group becomes a synthetic pattern — auto-numbered `ident0`, `ident1`, … (independently of `-p`'s `p0`/`p1`/… numbering, so adding an `-ident` group never renumbers existing `-p` patterns) — and is a first-class pattern everywhere one is accepted: `-name`, `-near`/`-far`, `-file-where`, `-in-scope`, every output mode. A matched identifier's **whole span** is the match (`$MATCH` is `parseConfig`, not just `parse`). + +```bash +# "config" mentions that are NOT near a "validate" call. +hprscript -ident config -name cfg -p 'validate\(' -name v -far cfg:v:10 -llm src/*.go +``` + +**Constraints.** + +- ASCII identifiers only (`[A-Za-z_][A-Za-z0-9_]*`) — Unicode identifiers (e.g. non-Latin Go identifiers) aren't scanned. +- Hand-scanned, not compiled into the Vectorscan database — the term set is dynamic per invocation and doesn't fit the compiled-pattern-database model. Cost scales with identifier count × `-ident` groups, fine for normal source files. +- A term matches a subtoken **starting boundary**, not requiring the match to end at one — `utf8` matches `UTF8Decoder` (spans the internal digit split), but `arse` does not match `parseConfig` (no boundary mid-word). +- No hyphen support: kebab-case names (`kebab-case-name`) scan as separate single-word identifiers split at each hyphen, not one joined identifier — a query spanning multiple kebab segments won't match. Config-file/CSS-style kebab-case is a possible future extension. +- `-extract` cannot follow `-ident` (no capture groups on identifier matches). Search-mode only — not available in `edit` mode or script mode (script-mode DSL support may come later). + +--- + ## Block extraction (CLI) `-block-open` and `-block-close` pair every match with its **balanced delimiter block**. The scanner searches forward from **match-start**, finds the first opening delimiter — one contained in the match itself counts, so anchors like `^@article\{` or a PEM `-----BEGIN` header pair with their own block — then tracks nesting until depth returns to zero. This is how you grep for a function signature and pull back the function body in one go. @@ -1833,6 +1869,57 @@ that fail it emit nothing (in any output mode) and don't count toward not with `-absent` — express absence inside the predicate instead (`-file-where 'NOT x' -f` is exactly `grep -L`). This replaces the script-mode "has A but not B" variable boilerplate for the common cases. +Works in `edit` mode too — targeting reuses this exact predicate. + +### Metadata conditions: `count()`, `churn()`, `lang` + +Beyond bare pattern-presence, three condition forms extend the same +`AND`/`OR`/`NOT` grammar with comparisons (`>`, `<`, `>=`, `<=`, `==`, `!=`): + +| Condition | Meaning | +|---|---| +| `count(pat) >= 3` | The pattern matched at least 3 times in this file (not just "at least once") | +| `churn(30) > 2` | More than 2 commits touched this file in the last 30 days (`git log --since=30.days.ago`) | +| `lang == go` | The file's auto-detected language (same guess `-scope auto` uses) equals `go` | + +```bash +# Files where errors cluster (≥3 hits) in code that's actively being worked on. +hprscript -p ERROR -name err -file-where 'count(err) >= 3 AND churn(30) > 2' -llm --glob '**/*.go' + +# Only Go files, regardless of what else -glob happened to sweep in. +hprscript -p TODO -file-where 'lang == go' -f -glob '**/*' +``` + +`churn(N)` runs **one** `git log` call per distinct `N` referenced in the +predicate (not one per file, not one per match) and requires the scan +target to be a git repository — a git failure (not a repo, `git` missing) +is a hard error (exit 2). `count(...)`/`churn(...)` need a git repo/pattern +respectively; `lang` only supports `==`/`!=` (ordering a language name is +meaningless). A file with no commits in the churn window is treated as +`churn(N) == 0`, not an error. + +--- + +## Sorting file-grouped output (`-order-by`) + +`-f`/`-c` stream in walk order — usually filesystem order, not anything meaningful. `-order-by ` buffers the file list and sorts it first: + +| Field | Order | +|---|---| +| `score` | Descending, same rarity/coverage/proximity formula as [`-hotspots`](#hotspot-ranking--hotspots-n) | +| `count` | Descending, total matches in the file | +| `path` | Ascending, lexicographic | + +```bash +# Which files have the pattern most concentrated, not just present? +hprscript -p 'TODO' -c -order-by score -glob '**/*.go' +``` + +Requires `-f` or `-c` (other output modes stream per-match, not per-file, so +"sort the file list" doesn't apply the same way); mutually exclusive with +`-sample`/`-hotspots`/`-budget`, which already define their own file +ordering. `-c`'s existing behavior of listing every scanned file — even a +`:0` — is preserved; only the order changes. --- @@ -1882,12 +1969,63 @@ near-duplicates. - Memory cap: at most `max(100×N, 10000)` matches buffered across all files. Files exceeding the cap silently truncate. - Mutually exclusive with output modes that don't emit per-match payloads - (`-f`, `-c`, `-absent`) — combining them errors out. + (`-f`, `-c`, `-absent`) or that render a whole file at once (`-elide`) — + combining them errors out. +- Mutually exclusive with [`-hotspots`](#hotspot-ranking--hotspots-n) — both + buffer the scan to pick a subset, with different selection strategies. - CLI-only in v1. Script-mode sampling is planned; for now, run a CLI sample and pipe results into a follow-up script if needed. --- +## Hotspot ranking (`-hotspots N`) + +"Where is the code for X" is usually answered by which *file* matches the most, rarest, most-co-located patterns — not by any single match. `-hotspots N` buffers the whole scan and answers exactly that, reusing script mode's [rank](#match-ranking-rank) formula (coverage × weighted hits ÷ size + a proximity bonus for co-located matches) instead of a second implementation. Quick-search patterns get an implicit weight of `1.0` — `-hotspots` doesn't yet expose `rank_surprise`/`rank_rich_clusters`; use script mode's `rank` when you need those. + +Each result is also annotated with its **best window**: the file's single densest match cluster (most distinct pattern ids, ties broken by point count) as a 1-based line range — a "look here first" pointer. + +```bash +hprscript -p 'ScopeIndex' -p 'find_innermost' -hotspots 5 src/*.cpp src/*.hpp +# → {"type":"hotspot","file":"src/scope.hpp","score":0.94,"line_start":45,"line_end":57,"patterns":["p0","p1"]} +# → {"type":"hotspot","file":"src/scope.cpp","score":0.88,"line_start":173,"line_end":173,"patterns":["p0","p1"]} +# → ... +``` + +Composes with the other output modes: default output is one JSONL hotspot record per file (`file`, `score`, `line_start`/`line_end` — the best window — `patterns`); `-llm` prints a flat `file:Ls-Le score=.. patterns=a,b` line per file; `-elide` renders each hotspot file's full match set through [elided scope output](#elided-scope-output--elide) instead of a bare pointer — the closest thing to a single RAG-style retrieval call this tool has. + +**Constraints.** + +- Mutually exclusive with `-sample` (both buffer the scan to pick a subset, with different selection strategies). +- Output is JSONL (default), `-llm`, or `-elide` only — `-f`/`-c`/`-o`/`-format`/`-absent` don't apply. +- `-limit`/`-max-output-bytes` don't bound hotspot output — `N` itself is the cap. `-summary`'s `emitted` count reflects hotspot rows, not raw matches (`matches` still reflects everything seen). +- Full file content is only buffered when `-elide` is active (JSONL/`-llm` rows need nothing but the accumulated scores); under `-elide` the same `max(100×N, 10000)`-match memory cap as `-sample` applies, and files past the cap are silently dropped from `-elide` rendering (their JSONL/`-llm` row logic doesn't apply here since eliding is the active mode). + +--- + +## Budget-packed context (`-budget N`) + +The feature that makes one hprscript call function like a single RAG retrieval: rank every matching file (same formula as [`-hotspots`](#hotspot-ranking--hotspots-n)/script mode's `rank`), then render score-descending in [`-elide`](#elided-scope-output--elide)'s shape until `N` bytes are spent. A file that doesn't fit in full degrades to a one-line summary (`file:Ls-Le score=.. patterns=a,b`); once even that doesn't fit, it's dropped and named in a trailing footer — nothing disappears silently. + +```bash +hprscript -p 'ScopeIndex' -p 'find_innermost' -budget 4000 src/*.cpp src/*.hpp +# → src/scope.hpp +# → 45-57 func ... +# → ... +# → src/scope.cpp:173-173 score=0.88 patterns=p0,p1 (compact — full render didn't fit the budget) +# → --- budget: 4 file(s) in full, 1 compact, 2 dropped: foo.cpp, bar.cpp --- +``` + +Bytes, not tokens — consistent with the `-max-*-bytes` family; there's no tokenizer dependency, so treat `N` as a rough proxy (roughly 3-4 bytes per token for typical source text). + +**Constraints.** + +- Defines its own output shape: mutually exclusive with every other output-mode flag (`-j`/`-f`/`-c`/`-o`/`-format`/`-absent`/`-llm`/`-elide`) and with `-sample`/`-hotspots`. Implies `-scope auto` when no `-scope` config is given, same as `-elide`. +- `-limit`/`-max-output-bytes` don't apply — `-budget`'s own byte accounting is the cap. +- Ranks and buffers **every** file with ≥1 match (not a fixed top-N like `-hotspots`) since it doesn't know ahead of time how many will fit; bounded by a fixed 20000-match memory cap, past which files are silently added to the "dropped" footer rather than considered at all. +- The footer names up to 10 dropped files, then collapses the rest to `(+N more)`. + +--- + ## Byte budgets Cap text-field sizes and total output to keep agent context windows from @@ -2009,7 +2147,92 @@ hprscript -p 'TODO' -llm -limit 1 -glob '**/*.go' - **Quick human eyeballing.** The grouped layout reads like `grep -n` output rather than JSON Lines — easier to skim. - **You don't need byte offsets / capture groups.** Pattern IDs still show up (as `[]` tags when multiple patterns are active), but if a downstream tool needs `from`/`to` offsets or the `extracted` map, stick with `-j`. -`-llm` is mutually exclusive with the other output modes (`-j`, `-f`, `-c`, `-o`, `-format`, `-absent`). +`-llm` is mutually exclusive with the other output modes (`-j`, `-f`, `-c`, `-o`, `-format`, `-absent`, `-elide`). + +--- + +## Elided scope output (`-elide`) + +`-llm` shows matched lines with a fixed `-A`/`-B` window; anything wider means either narrow context (you miss the surrounding logic) or the whole block (you pay for lines with nothing relevant in them). `-elide` picks a middle ground: it renders each matched function's signature and matched lines in full, then folds everything else in the scope into a single `… (+N lines)` marker — the shape a human skimming the function would produce by eye. + +Implies `-scope auto` when no `-scope`/`-scope-pattern` config is given (same convention as `-in-scope`). Matches outside any detected scope fall back to a plain ` : ` line, same as `-llm`'s line branch. + +Per scope, the shape is: + +``` + - + + + … (+N lines) + +``` + +Small gaps (2 lines or fewer) are shown in full rather than elided — `… (+1 lines)` saves nothing and reads worse than the line itself. + +### Example + +```bash +hprscript -p 'cli\.limit' -elide -scope cpp src/runner.cpp +# → +# src/runner.cpp +# 59-510 func run_search +# int run_search(const Cli &cli) { +# … (+59 lines) +# oo.global_limit = cli.limit; +# … (+390 lines) +# } +``` + +### Constraints + +- Renders a whole file's kept matches in one call rather than streaming per match, so it's mutually exclusive with `-sample` (see [Sample mode](#sample-mode--sample-n)). +- `-m` caps how many of a file's matches participate in the render; there's no mid-file stopping point the way per-match modes have one. +- No per-match pattern-id tagging inside a rendered window (unlike `-llm`'s `[]` tags) — use `-llm` or the default JSON mode when you need to attribute a specific line to a specific pattern. + +--- + +## Cross-invocation dedup (`-seen`) + +Agents iterate: search, refine, search again — and re-pay tokens for the same unchanged function every round. `-seen ` fixes that for [`-elide`](#elided-scope-output--elide) and [`-budget`](#budget-packed-context--budget-n): before rendering a matched scope in full, its raw source bytes are hashed and checked against `path`; an unchanged chunk collapses to one line instead of its full body, and the file is rewritten at the end of every run with what was actually shown. + +```bash +hprscript -p 'ScopeIndex' -elide -seen .hpr-seen src/scope.hpp +# first run — full chunks, .hpr-seen written +# → src/scope.hpp +# → 45-57 func ... +# → ... + +hprscript -p 'ScopeIndex' -elide -seen .hpr-seen src/scope.hpp +# nothing changed since — every chunk collapses +# → src/scope.hpp +# → 45-57 func ... (unchanged, already shown) + +# edit scope.hpp, then re-run — only the touched function expands +``` + +State format is plain text, one line per remembered chunk: +`\t\t\t`. It's the file's line range +*and* content that must match for a collapse — a function that grew or +shrank (even with unchanged logic) gets its new range, misses the lookup, +and renders in full again automatically. + +**Constraints.** + +- Requires `-elide` or `-budget` — there's no "chunk" concept to collapse + in any other output mode. +- Only enclosing-scope chunks collapse; matches outside any detected scope + (the orphan/context-block fallback) always render in full — they're + typically a line or two already, not worth the bookkeeping. +- `-budget` measures a file's full render to decide whether it fits the + byte budget before committing to it; a measurement that's ultimately + degraded to a compact summary or dropped is **not** recorded as shown, so + a later run with more budget still renders it in full rather than + wrongly treating it as already seen. +- The hash (FNV-1a, non-cryptographic) is a change-detection checksum, not + a security boundary — don't rely on it for anything adversarial. +- A missing or corrupt state file is treated as empty (first run), not an + error; a write failure at the end warns on stderr but doesn't change the + run's exit code — the actual search output already printed successfully. --- @@ -2218,8 +2441,9 @@ without parsing stderr. record carries `symlink_target`. - **Binary files** (NUL in the first 512 bytes) are skipped like search mode; `-diagnostics` surfaces them. -- **Not available in edit mode**: `-o`/`-f`/`-c`/`-llm`/`-absent`/`-format`, - `-A`/`-B`/`-C`, `-sample`, `-records`, `-s`/`-script`, and stdin as a scan +- **Not available in edit mode**: `-o`/`-f`/`-c`/`-llm`/`-elide`/`-absent`/ + `-format`, `-A`/`-B`/`-C`, `-sample`, `-hotspots`, `-budget`, `-ident`, + `-order-by`, `-seen`, `-records`, `-s`/`-script`, and stdin as a scan target — every input must be an explicit file, glob, list, or git selection. diff --git a/skills/hprscript/SKILL.md b/skills/hprscript/SKILL.md index 499957b..1793514 100644 --- a/skills/hprscript/SKILL.md +++ b/skills/hprscript/SKILL.md @@ -1,6 +1,6 @@ --- name: hprscript -description: Full-power multi-pattern content search, code intelligence, and guarded file editing via the `hprscript` CLI (Vectorscan/Hyperscan). One pass matches ALL patterns at once, so multi-pattern search costs the same as single-pattern. Covers keyword/regex search, finding files that contain or lack a pattern, per-file counts, extracting function bodies and balanced delimiter blocks, annotating each match with its enclosing function, capture-group extraction, proximity filters (X near or without Y), representative-sample usages, ranking files by relevance to a query, and aggregation/grouping/cross-file symbol resolution via a JSON script DSL. The `edit` subcommand does precise search-targeted file modification (replace/insert/delete, whole functions by name via -in-scope/scope spans) with dry-run diffs, -expect count guards, and atomic writes. -in-scope/-lines restrict search or edits to a named function/class or line range; -list-scopes outlines a file's functions. TRIGGER whenever you would grep/search/find/locate/count across files, ask where something is defined, what calls it, show me usages, or which files contain X — or would otherwise reach for the Grep tool, `grep`, or `rg`; ALSO trigger for mechanical multi-site edits, renames, or function-body swaps where you would otherwise reach for `sed -i` or repeated Edit-tool calls. Invoke via the Bash tool; the `hprscript` binary is on PATH. +description: Full-power multi-pattern content search, code intelligence, context-ranking, and guarded file editing via the `hprscript` CLI (Vectorscan/Hyperscan). One pass matches ALL patterns at once, so multi-pattern search costs the same as single-pattern. Covers keyword/regex search, finding files that contain or lack a pattern, per-file counts, extracting function bodies and balanced delimiter blocks, annotating each match with its enclosing function, capture-group extraction, proximity filters (X near or without Y), representative-sample usages, identifier search across naming conventions (parseConfig ~ parse_config), ranking files by relevance to a query (rarity/coverage/proximity, no script needed), packing the best-ranked context into a byte budget, scope-aware compact excerpts instead of whole function bodies, cross-invocation dedup so repeated agent queries don't re-pay tokens for unchanged code, filtering files by git commit-churn or match density, and aggregation/grouping/cross-file symbol resolution via a JSON script DSL. The `edit` subcommand does precise search-targeted file modification (replace/insert/delete, whole functions by name via -in-scope/scope spans) with dry-run diffs, -expect count guards, and atomic writes. -in-scope/-lines restrict search or edits to a named function/class or line range; -list-scopes outlines a file's functions. TRIGGER whenever you would grep/search/find/locate/count across files, ask where something is defined, what calls it, show me usages, which files contain X, where should I look first, or need to fit relevant code into a limited context window — or would otherwise reach for the Grep tool, `grep`, `rg`, or a RAG/embedding search; ALSO trigger for mechanical multi-site edits, renames, or function-body swaps where you would otherwise reach for `sed -i` or repeated Edit-tool calls. Invoke via the Bash tool; the `hprscript` binary is on PATH. --- # hprscript — full-power multi-pattern search (CLI) @@ -30,9 +30,15 @@ Always prefer `hprscript` over the Grep tool, `grep`, or `rg` — for the whole | X **with** Y nearby / X **without** Y nearby | `-near` / `-far` | | X with/without Y in the **same function** | `-same-scope` / `-not-same-scope` (+ `-scope`) | | Files where A but not B (no script) | `-file-where 'a AND NOT b'` | +| Files by commit churn / match density (no script) | `-file-where 'churn(30) > 2'` / `'count(pat) >= 3'` | | **Records** (lines) missing a field | `-absent -records line` | | Representative usages, dedup near-identical lines | `-sample N` | -| **Rank files by relevance** to a set of signals | `-s '{"rank":true,...}'` | +| Identifier by meaning, any naming convention | `-ident 'parse config'` (matches `parseConfig`/`parse_config`/`ConfigParser`) | +| **"Where's the code for X?"** — rank files, no script | `-hotspots N` (or `-s '{"rank":true,...}'` for surprise/rich-cluster weighting) | +| **Fit relevant context into a byte budget** | `-budget N` — ranks + renders full→compact→dropped until spent | +| Compact excerpt instead of the whole function body | `-elide` (signature + matched lines; rest folded as `… (+N lines)`) | +| Skip re-showing unchanged code across repeated calls | `-elide`/`-budget -seen ` | +| Sort `-f`/`-c` by relevance instead of walk order | `-order-by score\|count\|path` | | Counts, sums, manifests, grouping, cross-file resolve | `-s` script DSL | Cheapness ladder when you don't need the match text: `-f` / `-absent` / `-c` ≪ `-o` / `-llm` ≪ default JSON. Use `-limit N` aggressively when you only need to know *whether* something exists — scanning stops early. @@ -47,6 +53,7 @@ Cheapness ladder when you don't need the match text: `-f` / `-absent` / `-c` ≪ - `-F ` / `-Fi ` — fixed string, matched literally (`-F 'foo[0].bar()'` — no escaping). - `-name ` — name the preceding pattern; the id replaces `p0`/`p1` in `pat`, `[tags]`, relations, `-file-where`. - `-patterns-from ` — JSONL rule pack: `{"id","regexp"|"literal","case_insensitive",…}` per line, `#` comments. The right tool for IOC lists — no alternation-building in shell. +- `-ident ''` — match identifiers by subtoken, any casing/separator: `-ident 'parse config'` finds `parseConfig`/`parse_config`/`ConfigParser`/`PARSE_CONFIG`. Space-separated terms inside one `-ident` AND together; repeat the flag to OR groups. Each group is a first-class pattern (`ident0`, `ident1`, …) usable in `-name`/relations/`-file-where`. Search-mode only, not `edit`. - `-w` — whole-word (`\b(?:…)\b`). Or write `\b…\b` inline for per-alternative control. **Split alternations into separate `-p` patterns when you want to know *which* branch matched *where*.** Every match is tagged with the id of the pattern that produced it — `pat` in `-j`, a `[p0]`/`[p1]` prefix in `-llm`, `$PAT_ID` in `-format`/scripts. With one alternation (`-p 'alpha|beta|gamma'`) every hit collapses to `p0` — you lose attribution. Split it (`-p alpha -p beta -p gamma`) and each hit reports `p0`/`p1`/`p2`, so you see exactly which term fired on each line. Since adding patterns is free, **default to splitting**; keep an alternation only when the branches are one concept you don't need to tell apart, or must share a single `-near`/`-far` operand. In `-s` scripts give each pattern a meaningful `"id"` (`"auth"`, `"db"`) and it shows up verbatim as `$PAT_ID` instead of `p3`. @@ -78,8 +85,9 @@ Cheapness ladder when you don't need the match text: `-f` / `-absent` / `-c` ≪ | `-o` | Matched text only (`grep -o`) | | `-format ''` | Custom line with `$FILE $LINE $COL $MATCH $CONTEXT $FROM $TO $PAT_ID` (+ `$BLOCK*`, `$ENCLOSING_*`, `$EXTRACT_*` when active) | | `-llm` | Token-efficient text grouped by file; auto-adapts to `-block`/`-scope`; prints a `limit reached` footer when truncated | +| `-elide` | Scope-aware chunks: signature + matched lines, everything else folded as `… (+N lines)` — cheaper than `-llm` for a match buried in a large function | -`-llm` is the best choice when **you** are about to read the matches — it strips JSON noise and dedupes file headers (~30–50% fewer tokens than `-j`). Switch back to `-j` only when you need offsets, `pat`, or `extracted`. +`-llm` is the best choice when **you** are about to read the matches — it strips JSON noise and dedupes file headers (~30–50% fewer tokens than `-j`). Switch back to `-j` only when you need offsets, `pat`, or `extracted`. Reach for `-elide` instead of `-llm`/`-block-open` when the match sits inside a big function and you only want the relevant slice, not the whole body. --- @@ -157,6 +165,38 @@ hprscript -p '\bLock\(\)' -name lk -p '\bUnlock\(\)' -name ul \ hprscript -p 'httpClient' -sample 10 -glob '**/*.go' ``` +### Hotspot ranking — "where's the code for X?" without a script +`-hotspots N` buffers the whole scan and ranks files by a rarity/coverage/proximity score (files matching more of the queried terms, more densely, in fewer overall files, rank higher) — the same formula script mode's `rank` uses, but no JSON needed. Each result carries its densest match window. + +```bash +hprscript -p 'RetryPolicy' -p 'backoff' -hotspots 5 -llm -glob '**/*.go' +# → src/retry/policy.go:12-64 score=1.4 patterns=p0,p1 +``` +Composes with `-elide` (render each hotspot's full match set as a compact chunk instead of a bare pointer) and `-llm` (flat one-line-per-file); default output is JSONL. Mutually exclusive with `-sample`. + +### Elided scope output — a compact excerpt, not the whole function +`-elide` prints each matched scope's signature and matched lines, folding untouched interior lines as `… (+N lines)` — implies `-scope auto` automatically. Cheaper than `-block-open`/`-block-close` when the function is large and only a couple of lines matter; matches outside any scope fall back to a plain context line. + +```bash +hprscript -p 'RetryPolicy' -elide -scope auto -glob '**/*.go' +``` + +### Budget-packed context — the closest thing to one RAG retrieval call +`-budget N` ranks every matching file (same formula as `-hotspots`) and renders them score-descending as `-elide` chunks until `N` bytes are spent: a file that doesn't fit in full degrades to a one-line summary, then to a named "dropped" entry in a trailing footer — nothing disappears silently. Defines its own output shape (no `-j`/`-f`/`-llm`/etc; no `-sample`/`-hotspots`). + +```bash +hprscript -p 'AuthMiddleware' -p 'validateToken' -budget 6000 -glob '**/*.go' +``` +This is usually the **first thing to try** when a task starts with "find everything relevant to X and summarize/fix it" — one call replaces search → read N files → manually trim to fit context. + +### `-seen` — stop re-paying tokens for code you already showed yourself +Iterating (search → edit → search again) re-reads unchanged functions every round. `-seen ` hashes each `-elide`/`-budget` chunk's raw source and collapses anything unchanged since the last run against that state file to a one-line `(unchanged, already shown)` pointer. + +```bash +hprscript -p 'AuthMiddleware' -elide -seen .hpr-seen -glob '**/*.go' +``` +Requires `-elide` or `-budget` (there's no "chunk" to collapse in any other mode). Worth reaching for whenever a multi-turn task keeps re-scanning the same files. + ### Byte budgets — protect your context from minified files ```bash hprscript -p 'TODO' -max-context-bytes 200 -max-output-bytes 50000 -glob '**/*.{js,ts}' @@ -279,7 +319,17 @@ hprscript -p 'func\s+LoadData\b' -p 'type LoadData\b' -p 'LoadData\s*=' -llm -gl # Who calls X — with the enclosing function of each call site hprscript -p '\bLoadData\(' -scope auto -llm -glob '**/*.go' -# Rank files by relevance to a feature (see Rank above) — start here for "where do I look?" +# Rank files by relevance to a feature — start here for "where do I look?" +hprscript -p 'RetryPolicy' -p 'backoff' -hotspots 5 -llm -glob '**/*.go' + +# "Find everything relevant to X and fix it" — one call, budget-fit, no manual trimming +hprscript -p 'AuthMiddleware' -p 'validateToken' -budget 8000 -glob '**/*.go' + +# Symbol search that survives naming-convention drift (parseConfig ~ parse_config) +hprscript -ident 'parse config' -glob '**/*.go' + +# Multi-turn task on the same tree — don't re-read functions you already showed yourself +hprscript -p 'AuthMiddleware' -elide -seen .hpr-seen -glob '**/*.go' # Files importing a package hprscript -p '^import\s+"net/http"' -f -glob '**/*.go' @@ -373,6 +423,10 @@ Exit codes: `0` previewed/applied · `1` no sites · `2` error · **`3` guard re - ❌ `edit … -write` straight away → ✅ dry-run first, read the diff and count, then re-run with `-write -expect `. - ❌ Multi-line code in `-content '…'` (shell-quoting hell) → ✅ write it to a scratch file and use `-content-file`. - ❌ A relation-qualifier pattern without `-ref` in edit mode → ✅ mark it `-ref` or its own matches get rewritten too. +- ❌ Reading N whole files to find "the relevant part" → ✅ `-hotspots`/`-budget` rank and pack it in one call. +- ❌ Enumerating every casing variant of a symbol by hand (`parseConfig|parse_config|ConfigParser`) → ✅ `-ident 'parse config'`. +- ❌ Re-reading unchanged functions every turn of a multi-step task → ✅ `-elide`/`-budget -seen `. +- ❌ Reaching for an external RAG/embedding index for "find relevant code" → ✅ hprscript computes relevance fresh from the live tree, no index to go stale. ## Going deeper diff --git a/src/cli.cpp b/src/cli.cpp index c765b05..5786a6e 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -63,7 +63,7 @@ bool parse_nonneg(const char *s, int64_t &out) { bool set_output_mode(Cli &cli, OutputMode mode) { if (cli.out_mode_set) { cli.error = true; - cli.error_message = "output modes -j/-f/-c/-o/-format/-absent/-llm are mutually exclusive"; + cli.error_message = "output modes -j/-f/-c/-o/-format/-absent/-llm/-elide are mutually exclusive"; return false; } cli.out_mode = mode; @@ -83,9 +83,20 @@ void validate_edit_cli(Cli &cli) { if (cli.out_mode_set && cli.out_mode != OutputMode::JsonLines) return fail("edit mode: output is a dry-run diff or -j edit records; " - "-f/-c/-o/-llm/-absent/-format do not apply"); + "-f/-c/-o/-llm/-elide/-absent/-format do not apply"); if (cli.sample_n > 0) return fail("edit mode cannot combine with -sample"); + if (cli.hotspots_n > 0) + return fail("edit mode cannot combine with -hotspots"); + if (cli.budget_bytes > 0) + return fail("edit mode cannot combine with -budget"); + for (const auto &p : cli.patterns) + if (!p.ident_terms.empty()) + return fail("edit mode cannot combine with -ident"); + if (cli.order_by != Cli::OrderBy::None) + return fail("edit mode cannot combine with -order-by"); + if (!cli.seen_path.empty()) + return fail("edit mode cannot combine with -seen"); if (cli.records != Cli::RecordMode::None) return fail("edit mode cannot combine with -records"); if (cli.context_before != 0 || cli.context_after != 0) @@ -176,6 +187,11 @@ void print_help(FILE *out) { " line {id, regexp|literal, case_insensitive, word_boundary,\n" " utf8}; '#' comment lines allowed (repeatable)\n" " -extract n1,n2,… Re-extract capture groups from the preceding -p/-pi\n" +" -ident 't1 t2 …' Match identifiers whose subtokens include ALL given\n" +" terms, regardless of casing/separator (parseConfig ~\n" +" parse_config ~ ConfigParser); repeatable, each\n" +" occurrence ORs with the others; search-mode only\n" +" (not edit mode)\n" " -glob Scan glob (e.g. \"**/*.go\"; repeatable)\n" " -exclude Exclude rule: glob, bare dir name, or path prefix (repeatable)\n" " -files-from Scan the literal paths listed in f, one per line ('-' = stdin)\n" @@ -210,6 +226,9 @@ void print_help(FILE *out) { " -llm Token-efficient text for LLM consumption (auto-detects\n" " block/scope; dedupes file paths; prints a 'limit reached'\n" " footer when -limit or -max-output-bytes truncates output)\n" +" -elide Scope-aware chunks: signature + matched lines with -A/-B\n" +" context; untouched interior lines fold as \"… (+N lines)\"\n" +" (implies -scope auto when no -scope config is given)\n" "\n" "Block extraction (with -p):\n" " -block-open Opening delimiter (e.g. \"{\")\n" @@ -224,6 +243,23 @@ void print_help(FILE *out) { "Sample mode:\n" " -sample Buffer & return n diverse matches (file/shape-stratified)\n" "\n" +"Ranking:\n" +" -hotspots Buffer the whole scan, emit the top n files by a\n" +" rarity/coverage/proximity score (same formula as script\n" +" mode's rank), each with its densest match window.\n" +" Composes with -llm (flat line) / -elide (rendered chunk);\n" +" default output is JSONL hotspot records.\n" +" -budget Rank every matching file, then render score-descending\n" +" in -elide's shape until n bytes are spent — degrading to\n" +" a one-line summary and finally to \"dropped\" once a\n" +" file no longer fits. Defines its own output shape\n" +" (mutually exclusive with -j/-f/-c/-o/-format/-absent/\n" +" -llm/-elide/-sample/-hotspots).\n" +" -seen Cross-invocation dedup for -elide/-budget: a chunk\n" +" unchanged since the last run against this state file\n" +" collapses to a one-line pointer instead of full source.\n" +" The file is rewritten after each run.\n" +"\n" "Pattern relations (filter matches by proximity/containment, repeatable):\n" " -near A:B:K Emit pattern A's matches with a B-match within K lines\n" " -far A:B:K Emit A's matches with NO B-match within K lines (K=0=same)\n" @@ -235,6 +271,11 @@ void print_help(FILE *out) { " -file-where Emit a file's matches only when the predicate over\n" " matched patterns holds: 'err AND NOT recovery',\n" " AND/OR/NOT or &&/||/!, parentheses, ids or p0/p1…\n" +" Conditions: count(pat) > n, churn(days) > n (git\n" +" commits touching the file), lang == \n" +" -order-by Sort -f/-c output by score|count|path instead of\n" +" walk order (mutually exclusive with -sample/\n" +" -hotspots/-budget, which define their own order)\n" "\n" "Enclosing-scope annotation (with -p):\n" " -scope Built-in pack (auto, go, rust, c, cpp, java, js, ts)\n" @@ -336,6 +377,27 @@ Cli parse_cli(int argc, char **argv) { cli.patterns.push_back(std::move(p)); continue; } + if (eq(a, "-ident")) { + const char *v = take(i, argc, argv, a, cli); if (!v) return cli; + CliPattern p; + const char *s = v; + while (*s) { + while (*s && std::isspace((unsigned char)*s)) ++s; + if (!*s) break; + const char *start = s; + while (*s && !std::isspace((unsigned char)*s)) ++s; + std::string term(start, static_cast(s - start)); + for (auto &c : term) c = static_cast(std::tolower((unsigned char)c)); + p.ident_terms.push_back(std::move(term)); + } + if (p.ident_terms.empty()) { + cli.error = true; + cli.error_message = "-ident: at least one term required"; + return cli; + } + cli.patterns.push_back(std::move(p)); + continue; + } if (eq(a, "-patterns-from") || eq(a, "--patterns-from")) { const char *v = take(i, argc, argv, a, cli); if (!v) return cli; cli.patterns_from.emplace_back(v); @@ -349,6 +411,13 @@ Cli parse_cli(int argc, char **argv) { return cli; } CliPattern &last = cli.patterns.back(); + if (!last.ident_terms.empty()) { + cli.error = true; + cli.error_message = + "-extract cannot follow -ident (no capture groups on " + "identifier matches)"; + return cli; + } if (!last.extract_names.empty()) { cli.error = true; cli.error_message = "-extract repeated for the same pattern"; @@ -450,6 +519,24 @@ Cli parse_cli(int argc, char **argv) { cli.file_where = v; continue; } + if (eq(a, "-order-by")) { + const char *v = take(i, argc, argv, a, cli); if (!v) return cli; + if (eq(v, "score")) cli.order_by = Cli::OrderBy::Score; + else if (eq(v, "count")) cli.order_by = Cli::OrderBy::Count; + else if (eq(v, "path")) cli.order_by = Cli::OrderBy::Path; + else { + cli.error = true; + cli.error_message = std::string("-order-by: unknown field '") + + v + "' (supported: score, count, path)"; + return cli; + } + continue; + } + if (eq(a, "-seen")) { + const char *v = take(i, argc, argv, a, cli); if (!v) return cli; + cli.seen_path = v; + continue; + } if (eq(a, "-w")) { cli.word_boundary = true; continue; } if (eq(a, "-no-utf8")) { cli.no_utf8 = true; continue; } if (eq(a, "-ucp")) { cli.ucp = true; continue; } @@ -486,6 +573,7 @@ Cli parse_cli(int argc, char **argv) { if (eq(a, "-o")) { if (!set_output_mode(cli, OutputMode::MatchOnly)) return cli; continue; } if (eq(a, "-absent")) { if (!set_output_mode(cli, OutputMode::Absent)) return cli; continue; } if (eq(a, "-llm")) { if (!set_output_mode(cli, OutputMode::Llm)) return cli; continue; } + if (eq(a, "-elide")) { if (!set_output_mode(cli, OutputMode::Elide)) return cli; continue; } if (eq(a, "-format")) { const char *v = take(i, argc, argv, a, cli); if (!v) return cli; if (!set_output_mode(cli, OutputMode::Custom)) return cli; @@ -604,6 +692,17 @@ Cli parse_cli(int argc, char **argv) { if (cli.sample_n < 0) cli.sample_n = 0; continue; } + if (eq(a, "-hotspots")) { + const char *v = take(i, argc, argv, a, cli); if (!v) return cli; + cli.hotspots_n = std::atoi(v); + if (cli.hotspots_n < 0) cli.hotspots_n = 0; + continue; + } + if (eq(a, "-budget")) { + const char *v = take(i, argc, argv, a, cli); if (!v) return cli; + cli.budget_bytes = static_cast(std::atoll(v)); + continue; + } if (eq(a, "-near") || eq(a, "-far")) { const char *v = take(i, argc, argv, a, cli); if (!v) return cli; @@ -955,11 +1054,20 @@ bool load_patterns_from(Cli &cli) { // collide with another pattern's name or auto `p` id, making relations // and `pat` attribution ambiguous. Called after -patterns-from loading. bool validate_pattern_ids(Cli &cli) { + // Mirrors build_patterns()'s numbering: regex patterns auto-number as + // p0/p1/... and -ident groups separately as ident0/ident1/..., each in + // the order given — independent counters, not a single shared index. std::vector ids; ids.reserve(cli.patterns.size()); - for (size_t i = 0; i < cli.patterns.size(); ++i) { - const std::string &n = cli.patterns[i].name; - ids.push_back(n.empty() ? "p" + std::to_string(i) : n); + size_t regex_i = 0, ident_i = 0; + for (const auto &cp : cli.patterns) { + if (!cp.ident_terms.empty()) { + ids.push_back(cp.name.empty() ? "ident" + std::to_string(ident_i++) + : cp.name); + } else { + ids.push_back(cp.name.empty() ? "p" + std::to_string(regex_i++) + : cp.name); + } } for (size_t i = 0; i < ids.size(); ++i) { for (size_t j = i + 1; j < ids.size(); ++j) { diff --git a/src/cli.hpp b/src/cli.hpp index f4c82fb..a611f43 100644 --- a/src/cli.hpp +++ b/src/cli.hpp @@ -32,6 +32,14 @@ struct CliPattern { // Edit mode only (-ref): reference-only pattern — usable in relations // and -file-where, but its matches never become edit sites. bool ref = false; + + // -ident terms (already lowercased at parse time). Non-empty marks this + // entry as an identifier-subtoken group instead of a regex pattern — + // `regexp` is unused for it. Terms inside one group AND together; + // separate -ident occurrences OR together (each is its own CliPattern + // entry, auto-numbered ident0/ident1/... independently of regex + // patterns' p0/p1/... numbering). See src/ident.hpp. + std::vector ident_terms; }; // Options for the `edit` subcommand — the only mode allowed to modify @@ -112,6 +120,19 @@ struct Cli { // (AND/OR/NOT, &&/||/!, parens). A file's matches are emitted only when // the predicate holds over the set of patterns that matched in it. std::string file_where; + + // -order-by : sort -f/-c output instead of streaming + // in walk order. `score` reuses the -hotspots ranking formula; `count` + // is total matches in the file; `path` is lexicographic. Mutually + // exclusive with -sample/-hotspots/-budget, which already define their + // own ordering. + enum class OrderBy { None, Score, Count, Path }; + OrderBy order_by = OrderBy::None; + + // -seen : cross-invocation dedup state file for -elide/-budget. + // Scope chunks unchanged since the last run against this file collapse + // to a one-line pointer instead of full source. Empty = off (default). + std::string seen_path; bool word_boundary = false; bool no_utf8 = false; // -no-utf8: byte-mode matching bool ucp = false; // -ucp: enable Unicode \w/\d/\s (opt-in) @@ -209,6 +230,20 @@ struct Cli { // collapsed). 0 means streaming (default). int sample_n = 0; + // Ranking mode (`-hotspots N`): buffer matches across the whole scan, + // then emit the top N files by the same rarity/coverage/proximity + // score script mode's `rank` uses (see src/rank.hpp), each annotated + // with its densest match window. 0 means off (default). + int hotspots_n = 0; + + // Budget-packing mode (`-budget N`): buffer the whole scan, rank every + // matching file, then render score-descending in -elide's shape until + // N bytes are spent — degrading to a one-line summary and finally to + // "dropped" once a file's full render no longer fits. Defines its own + // output shape, so it's mutually exclusive with -j/-f/-c/-o/-format/ + // -absent/-llm/-elide/-sample/-hotspots. 0 = off (default). + uint64_t budget_bytes = 0; + // Script mode. std::string script_inline; // -s '' std::string script_path; // -script diff --git a/src/edit.cpp b/src/edit.cpp index b494df7..00d6c26 100644 --- a/src/edit.cpp +++ b/src/edit.cpp @@ -49,6 +49,16 @@ namespace hpr { namespace { +// glibc (Linux) names `struct stat`'s POSIX nanosecond mtime field +// `st_mtim`; BSD/Darwin (macOS) names the same struct timespec +// `st_mtimespec`. Centralize the one platform difference here instead of +// scattering #ifdefs at each read site. +#if defined(__APPLE__) +struct timespec stat_mtime(const struct stat &st) { return st.st_mtimespec; } +#else +struct timespec stat_mtime(const struct stat &st) { return st.st_mtim; } +#endif + // One planned splice: bytes [start,end) of the file are replaced by `text`. // Inserts are zero-width (start == end). `span_*` is the resolved span the // site was derived from — for replace/delete it equals [start,end); for @@ -599,6 +609,14 @@ int run_edit(const Cli &cli) { const bool any_scope_rel = any_scope_relation(rels); FileWhere fw; if (!fw.init(cli.file_where, patterns)) return 2; + std::map> churn_map; + if (!fw.churn_windows().empty()) { + std::string cerr; + if (!build_churn_map(fw.churn_windows(), churn_map, cerr)) { + std::fprintf(stderr, "hprscript: git: %s\n", cerr.c_str()); + return 2; + } + } Matcher matcher; if (!anchorless) { @@ -742,7 +760,9 @@ int run_edit(const Cli &cli) { collector.collect(matcher, content, idx, scope_ptr, al, kept); tf.apply(kept, idx, scope_ptr); if (kept.empty()) return true; - if (fw.active() && !fw.pass(kept, patterns.size())) return true; + if (fw.active() && + !fw.pass(kept, patterns.size(), it.path, churn_map)) + return true; stats.matches_seen += kept.size(); // -ref patterns qualify (relations above, -file-where just now) @@ -795,7 +815,7 @@ int run_edit(const Cli &cli) { struct stat stbuf; if (::stat(pf.write_path.c_str(), &stbuf) == 0) { pf.st_size = stbuf.st_size; - pf.st_mtim = stbuf.st_mtim; + pf.st_mtim = stat_mtime(stbuf); pf.st_mode = stbuf.st_mode & 07777; } @@ -1165,8 +1185,8 @@ int run_edit(const Cli &cli) { struct stat stbuf; bool drifted = ::stat(pf.write_path.c_str(), &stbuf) != 0 || stbuf.st_size != pf.st_size || - stbuf.st_mtim.tv_sec != pf.st_mtim.tv_sec || - stbuf.st_mtim.tv_nsec != pf.st_mtim.tv_nsec; + stat_mtime(stbuf).tv_sec != pf.st_mtim.tv_sec || + stat_mtime(stbuf).tv_nsec != pf.st_mtim.tv_nsec; if (drifted) { Violation v; v.guard = "changed-during-run"; diff --git a/src/git.cpp b/src/git.cpp index 8b4c423..3ef0fb2 100644 --- a/src/git.cpp +++ b/src/git.cpp @@ -232,4 +232,41 @@ bool git_added_lines(const GitSelection &sel, return true; } +bool git_churn(int days, std::unordered_map &out, + std::string &err) { + std::string prefix; + if (!repo_prefix(prefix, err)) return false; + + std::string since = "--since=" + std::to_string(days) + ".days.ago"; + std::string blob; + if (!run_git({"-c", "core.quotePath=false", "log", since, "--name-only", + "--pretty=format:%H"}, + blob, err)) + return false; + + // Walk the blob line by line without relying on git's blank-line + // conventions between commits (those vary with --pretty=format:): any + // line that's a bare 40- or 64-char hex string is a commit hash + // (%H's exact width for SHA-1/SHA-256 repos), everything else + // non-blank is a changed filename belonging to the commit above it. + auto looks_like_hash = [](std::string_view s) { + if (s.size() != 40 && s.size() != 64) return false; + for (char c : s) { + bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + if (!hex) return false; + } + return true; + }; + size_t pos = 0; + while (pos < blob.size()) { + size_t eol = blob.find('\n', pos); + if (eol == std::string::npos) eol = blob.size(); + std::string_view line(blob.data() + pos, eol - pos); + pos = eol + 1; + if (line.empty() || looks_like_hash(line)) continue; + ++out[prefix + std::string(line)]; + } + return true; +} + } // namespace hpr diff --git a/src/git.hpp b/src/git.hpp index 5f094ff..08ec15b 100644 --- a/src/git.hpp +++ b/src/git.hpp @@ -45,4 +45,11 @@ bool git_added_lines(const GitSelection &sel, std::unordered_map &out, std::string &err); +// Commit-churn table: number of commits touching each file in the last +// `days` days (`git log --since=.days.ago --name-only`), one `git` +// invocation regardless of how many files exist — never one subprocess per +// file. Files with no commits in the window are simply absent from `out`. +bool git_churn(int days, std::unordered_map &out, + std::string &err); + } // namespace hpr diff --git a/src/ident.cpp b/src/ident.cpp new file mode 100644 index 0000000..5723f72 --- /dev/null +++ b/src/ident.cpp @@ -0,0 +1,112 @@ +#include "ident.hpp" + +#include + +namespace hpr { + +std::vector identifier_subtoken_starts(std::string_view s) { + std::vector starts; + const size_t n = s.size(); + auto is_lower = [](char c) { return c >= 'a' && c <= 'z'; }; + auto is_upper = [](char c) { return c >= 'A' && c <= 'Z'; }; + auto is_digit = [](char c) { return c >= '0' && c <= '9'; }; + + size_t i = 0; + while (i < n) { + char c = s[i]; + if (c == '_') { ++i; continue; } + + if (is_digit(c)) { + starts.push_back(i); + size_t j = i + 1; + while (j < n && is_digit(s[j])) ++j; + i = j; + continue; + } + if (is_upper(c)) { + starts.push_back(i); + size_t j = i + 1; + while (j < n && is_upper(s[j])) ++j; + if (j < n && is_lower(s[j]) && j > i + 1) { + // Acronym run followed by a lowercase word: "HTTPServer" — + // the last uppercase letter starts the next word, so back + // off one position ("HTTP" | "Server", not "HTTPS" | "erver"). + i = j - 1; + } else if (j == i + 1 && j < n && is_lower(s[j])) { + // Single leading capital + lowercase run: "Config". + size_t k = j; + while (k < n && is_lower(s[k])) ++k; + i = k; + } else { + i = j; // trailing acronym with nothing lowercase after it + } + continue; + } + if (is_lower(c)) { + starts.push_back(i); + size_t j = i + 1; + while (j < n && is_lower(s[j])) ++j; + i = j; + continue; + } + ++i; // unreachable given the caller's [A-Za-z0-9_]+ charset + } + return starts; +} + +namespace { + +bool group_matches(std::string_view ident, const std::vector &starts, + const std::vector &terms_lower) { + for (const auto &term : terms_lower) { + bool found = false; + for (size_t s : starts) { + if (s + term.size() > ident.size()) continue; + bool eq = true; + for (size_t k = 0; k < term.size(); ++k) { + if (std::tolower(static_cast(ident[s + k])) != + static_cast(term[k])) { + eq = false; + break; + } + } + if (eq) { found = true; break; } + } + if (!found) return false; // AND within the group + } + return true; +} + +} // namespace + +void scan_identifiers(std::string_view buf, const std::vector &groups, + uint32_t pattern_index_base, std::vector &out) { + if (groups.empty()) return; + const size_t n = buf.size(); + auto is_ident_start = [](unsigned char c) { + return std::isalpha(c) || c == '_'; + }; + auto is_ident_cont = [](unsigned char c) { + return std::isalnum(c) || c == '_'; + }; + + size_t i = 0; + while (i < n) { + if (!is_ident_start(static_cast(buf[i]))) { ++i; continue; } + size_t start = i++; + while (i < n && is_ident_cont(static_cast(buf[i]))) ++i; + std::string_view ident = buf.substr(start, i - start); + std::vector starts = identifier_subtoken_starts(ident); + for (size_t g = 0; g < groups.size(); ++g) { + if (group_matches(ident, starts, groups[g].terms)) { + Match m; + m.pattern_index = pattern_index_base + static_cast(g); + m.from = start; + m.to = i; + out.push_back(m); + } + } + } +} + +} // namespace hpr diff --git a/src/ident.hpp b/src/ident.hpp new file mode 100644 index 0000000..78132a8 --- /dev/null +++ b/src/ident.hpp @@ -0,0 +1,50 @@ +// Identifier-subtoken matching (`-ident`): find identifiers regardless of +// naming convention — `parseConfig` ~ `parse_config` ~ `ConfigParser` all +// contain the subtokens "parse" and "config". This is the recall gap plain +// regex search can't close without the caller enumerating every casing +// variant by hand. +// +// Deliberately ASCII-only ([A-Za-z_][A-Za-z0-9_]*) and hand-scanned rather +// than routed through Vectorscan: the term set is dynamic per invocation, +// so it doesn't fit the compiled-pattern-database model the rest of the +// engine uses. Non-ASCII identifiers (Unicode identifiers in Go, etc.) are +// silently not scanned. +#pragma once + +#include "common.hpp" + +#include +#include +#include +#include + +namespace hpr { + +// One `-ident` group: terms that must ALL appear as subtokens of the same +// identifier (AND within a group). Separate `-ident` occurrences OR +// together — each becomes its own group/synthetic pattern. Terms are +// lowercased by the CLI parser; matching is always case-insensitive. +struct IdentGroup { + std::vector terms; +}; + +// Byte offsets into `ident` where a subtoken begins: after each `_` +// (consumed, never itself part of a token), at lower→upper transitions +// (camelCase), at alpha↔digit transitions, and — for acronym runs like +// "HTTPServer" — at the last uppercase letter before a following lowercase +// run, so the split reads "HTTP", "Server" rather than "HTTPS", "erver". +// Exposed for testing; scan_identifiers() is the entry point callers use. +std::vector identifier_subtoken_starts(std::string_view ident); + +// Scan `buf` for identifier runs, testing each against every group in +// `groups`. A group matches an identifier when every one of its terms +// appears (case-insensitively) as a substring starting exactly at one of +// the identifier's subtoken boundaries — so "utf8" matches "UTF8Decoder" +// (starts at the identifier's own start, spanning the internal digit +// boundary) but "arse" does not match "parseConfig" (no boundary mid-word). +// One Match per (identifier, matching group), appended in scan order; +// pattern_index = `pattern_index_base` + the group's index in `groups`. +void scan_identifiers(std::string_view buf, const std::vector &groups, + uint32_t pattern_index_base, std::vector &out); + +} // namespace hpr diff --git a/src/output.cpp b/src/output.cpp index 2da4176..7bc59f9 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -1,6 +1,7 @@ #include "output.hpp" #include "block.hpp" +#include "seen.hpp" #include #include @@ -505,6 +506,186 @@ void Formatter::emit_llm(const std::string &file, const Pattern &pattern, write_out(s); } +namespace { + +// Small gaps inside a scope are shown plainly rather than elided — "… +// (+1 lines)" saves nothing and reads worse than just printing the line. +constexpr uint32_t ELIDE_MERGE_GAP = 2; + +struct LineWindow { + uint32_t lo, hi; +}; + +} // namespace + +void Formatter::on_file_elide(const std::string &file, + const std::vector &kept, + std::string_view buf, const LineIndex &idx, + const ScopeIndex *scope, const SeenStore *seen, + std::vector *marks_out) { + if (kept.empty()) return; + emitted_ += static_cast(kept.size()); + + bool first_block = true; + auto &s = scratch_; + + auto print_file_header = [&]() { + if (llm_last_file_ != file) { + llm_last_file_ = file; + s.clear(); + s.append(file); + s += '\n'; + write_out(s); + } + }; + + auto print_line_range = [&](uint32_t lo, uint32_t hi) { + for (uint32_t L = lo; L <= hi; ++L) { + std::string_view t = idx.line_text(L); + s.clear(); + s.append(t.data(), t.size()); + if (t.empty() || t.back() != '\n') s += '\n'; + write_out(s); + } + }; + + // Lines strictly between from_excl and to_excl: shown plainly if few, + // elided as "… (+N lines)" otherwise. + auto print_gap = [&](uint32_t from_excl, uint32_t to_excl) { + if (to_excl <= from_excl + 1) return; + uint32_t n = to_excl - from_excl - 1; + if (n <= ELIDE_MERGE_GAP) { + print_line_range(from_excl + 1, to_excl - 1); + } else { + s.clear(); + s += " \xE2\x80\xA6 (+"; + append_uint32(s, n); + s += " lines)\n"; + write_out(s); + } + }; + + auto render_scope_block = [&](const ScopeRange &sr, + const std::vector &ms) { + uint64_t hash = 0; + bool collapse = false; + if (seen || marks_out) { + hash = fnv1a(std::string_view( + buf.data() + sr.start_off, sr.end_off - sr.start_off)); + if (marks_out) + marks_out->push_back({file, sr.line_start, sr.line_end, hash}); + if (seen && seen->seen_unchanged(file, sr.line_start, sr.line_end, hash)) + collapse = true; + } + + print_file_header(); + s.clear(); + if (!first_block) s += '\n'; + first_block = false; + s += " "; + append_uint32(s, sr.line_start); + s += '-'; + append_uint32(s, sr.line_end); + s += ' '; + s += sr.kind; + s += ' '; + s += sr.name; + if (collapse) s += " (unchanged, already shown)"; + s += '\n'; + write_out(s); + if (collapse) return; + print_line_range(sr.line_start, sr.line_start); // signature line + + std::vector windows; + windows.reserve(ms.size()); + for (const Match *m : ms) { + uint32_t line = idx.line_of(m->from); + uint32_t lo = (line > static_cast(opts_.context_before)) + ? line - static_cast(opts_.context_before) + : 1; + uint32_t hi = line + static_cast(opts_.context_after); + lo = std::max(lo, sr.line_start + 1); + hi = std::min({hi, sr.line_end, idx.line_count()}); + if (lo > hi) continue; // whole window swallowed by the signature line + windows.push_back({lo, hi}); + } + // Merge overlapping/near windows (already position-sorted, since + // `ms` follows the position-sorted `kept` order). + std::vector merged; + for (const auto &w : windows) { + if (!merged.empty() && + w.lo <= merged.back().hi + ELIDE_MERGE_GAP + 1) { + merged.back().hi = std::max(merged.back().hi, w.hi); + } else { + merged.push_back(w); + } + } + + uint32_t prev_end = sr.line_start; + for (const auto &w : merged) { + print_gap(prev_end, w.lo); + print_line_range(w.lo, w.hi); + prev_end = w.hi; + } + if (prev_end < sr.line_end) { + print_gap(prev_end, sr.line_end); + print_line_range(sr.line_end, sr.line_end); // closing line + } + }; + + auto render_orphan = [&](const Match *m) { + print_file_header(); + if (!first_block) { + s.clear(); + s += '\n'; + write_out(s); + } + first_block = false; + uint32_t line = idx.line_of(m->from); + std::string_view ctx = context_block(buf, idx, *m); + std::string_view ctx_view = + truncate_safe(ctx, opts_.max_context_bytes, nullptr); + s.clear(); + s += " "; + append_uint32(s, line); + s += ": "; + s.append(ctx_view.data(), ctx_view.size()); + if (s.empty() || s.back() != '\n') s += '\n'; + write_out(s); + }; + + // Walk in position order, flushing an accumulated run whenever the + // innermost enclosing scope changes. `kept` is sorted by `from`, so + // same-scope matches are contiguous except for nesting, which — being + // rare in practice — isn't special-cased further. + const ScopeRange *cur_scope = nullptr; + bool run_is_scope = false; + std::vector run; + + auto flush = [&]() { + if (run.empty()) return; + if (run_is_scope) { + render_scope_block(*cur_scope, run); + } else { + for (const Match *m : run) render_orphan(m); + } + run.clear(); + }; + + for (const auto &m : kept) { + const ScopeRange *sr = scope ? scope->find_innermost(m.from) : nullptr; + bool is_scope = sr != nullptr; + if (!run.empty() && (is_scope != run_is_scope || sr != cur_scope)) + flush(); + if (run.empty()) { + run_is_scope = is_scope; + cur_scope = sr; + } + run.push_back(&m); + } + flush(); +} + void Formatter::on_match(const std::string &file, const Pattern &pattern, const Match &m, std::string_view buf, const LineIndex &idx, const ScopeIndex *scope) { @@ -524,6 +705,12 @@ void Formatter::on_match(const std::string &file, const Pattern &pattern, case OutputMode::MatchOnly: emit_match_only(buf, m, idx); break; case OutputMode::Custom: emit_custom(file, pattern, m, buf, idx, scope); break; case OutputMode::Llm: emit_llm(file, pattern, m, buf, idx, scope); break; + case OutputMode::Elide: + // Elide renders a whole file's matches at once via + // on_file_elide(); it never streams through on_match(). Undo + // the ++emitted_ above so callers can't double-count by mistake. + --emitted_; + break; case OutputMode::Absent: // Absent mode tracks per-file presence, no per-match output. per_file_counts_[file]++; @@ -554,7 +741,7 @@ void Formatter::on_file_end(const std::string &file, bool had_match) { } void Formatter::on_complete() { - if (opts_.mode == OutputMode::Llm) { + if (opts_.mode == OutputMode::Llm || opts_.mode == OutputMode::Elide) { char buf[160]; if (limit_hit_) { int n = std::snprintf(buf, sizeof(buf), diff --git a/src/output.hpp b/src/output.hpp index 549dd77..e7a6ed2 100644 --- a/src/output.hpp +++ b/src/output.hpp @@ -16,9 +16,13 @@ #include #include #include +#include namespace hpr { +class SeenStore; // src/seen.hpp — -seen cross-invocation dedup +struct SeenMark; + enum class OutputMode { JsonLines, // one JSON object per line (default) FilesOnly, // grep -l: dedup'd file paths @@ -27,6 +31,7 @@ enum class OutputMode { Custom, // -format template with $FILE etc. Absent, // grep -L: files where pattern is NOT present Llm, // -llm: token-efficient text for LLM consumption + Elide, // -elide: scope-aware chunk rendering, whole file at once }; // True iff the given output mode reads line/col/context from a LineIndex. @@ -36,6 +41,7 @@ inline bool needs_line_index(OutputMode m) { case OutputMode::JsonLines: case OutputMode::Custom: case OutputMode::Llm: + case OutputMode::Elide: return true; case OutputMode::FilesOnly: case OutputMode::Counts: @@ -86,6 +92,26 @@ class Formatter { const Match &m, std::string_view buf, const LineIndex &idx, const ScopeIndex *scope = nullptr); + // -elide mode: render one file's entire kept-match set in a single call, + // grouped by innermost enclosing scope. Each scope prints its signature + // line, then matched lines with -A/-B context, folding untouched + // interior lines as "… (+N lines)"; matches outside any scope fall back + // to a plain context-block line (like -llm's line branch). `kept` must + // be sorted by `from` ascending (the collector's normal emission order). + // + // `seen` (optional, -seen): before rendering a scope block in full, its + // raw byte range is hashed and checked against `seen` — an unchanged + // match collapses to a one-line "(unchanged, already shown)" pointer + // instead of the full render. Orphan (no-scope) matches are never + // collapsed. `marks_out` (optional), when given, receives one SeenMark + // per scope block examined (collapsed or not) — the caller decides + // whether to actually commit those into the persistent SeenStore (see + // src/seen.hpp's SeenMark doc comment for why that's a separate step). + void on_file_elide(const std::string &file, const std::vector &kept, + std::string_view buf, const LineIndex &idx, + const ScopeIndex *scope, const SeenStore *seen = nullptr, + std::vector *marks_out = nullptr); + // Record-level absence (-records line with -absent): emit one JSON line // for a record (line) of `file` that lacks `pattern`. Only meaningful in // Absent mode; `text` is the record's content (truncated to @@ -145,7 +171,7 @@ class Formatter { uint64_t bytes_emitted_ = 0; // total bytes written to out_ bool over_budget_ = false; // sticky once max_output_bytes is exceeded bool limit_hit_ = false; // set by mark_limit_hit() (LLM footer) - std::string llm_last_file_; // last file printed as LLM header (dedupe) + std::string llm_last_file_; // last file printed as a header (-llm/-elide dedupe) std::unordered_map per_file_counts_; // Per-line scratch buffer reused across matches to avoid reallocation. diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 6a56864..3e1ff26 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -14,13 +14,19 @@ namespace { using WhereNode = hpr::FileWhere::Node; +using WhereOp = hpr::FileWhere::Op; // ---- -file-where expression ------------------------------------------------- // Grammar: // expr := and (OR|'||' and)* // and := unary (AND|'&&' unary)* -// unary := (NOT|'!') unary | '(' expr ')' | pattern-id -// Keywords are case-insensitive; ids are names, `p`, or numeric indices. +// unary := (NOT|'!') unary | '(' expr ')' | leaf +// leaf := ident '(' [arg] ')' cmp number -- churn(30) > 2, count(p0) >= 3 +// | 'lang' cmp ident -- lang == go +// | ident -- bare pattern id (presence) +// cmp := '>=' | '<=' | '==' | '!=' | '>' | '<' +// Keywords are case-insensitive; pattern ids are names, `p`/`ident`, +// or numeric indices. struct WhereParser { std::string_view src; size_t pos = 0; @@ -63,6 +69,33 @@ struct WhereParser { } return false; } + // Longest-symbol-first so ">=" isn't swallowed as "> ' '='". + bool parse_cmp_op(WhereOp &op) { + skip_ws(); + if (eat_sym(">=")) { op = WhereOp::Ge; return true; } + if (eat_sym("<=")) { op = WhereOp::Le; return true; } + if (eat_sym("==")) { op = WhereOp::Eq; return true; } + if (eat_sym("!=")) { op = WhereOp::Ne; return true; } + if (eat_sym(">")) { op = WhereOp::Gt; return true; } + if (eat_sym("<")) { op = WhereOp::Lt; return true; } + return false; + } + bool parse_number(double &out) { + skip_ws(); + size_t s = pos; + size_t p = pos; + if (p < src.size() && (src[p] == '-' || src[p] == '+')) ++p; + bool any_digit = false; + while (p < src.size() && std::isdigit((unsigned char)src[p])) { ++p; any_digit = true; } + if (p < src.size() && src[p] == '.') { + ++p; + while (p < src.size() && std::isdigit((unsigned char)src[p])) { ++p; any_digit = true; } + } + if (!any_digit) return false; + out = std::strtod(std::string(src.substr(s, p - s)).c_str(), nullptr); + pos = p; + return true; + } bool parse_expr(WhereNode &out) { return parse_or(out); } bool parse_or(WhereNode &out) { @@ -108,29 +141,153 @@ struct WhereParser { return true; } std::string w; - if (!word(w)) { err = "expected a pattern id"; return false; } + if (!word(w)) { err = "expected a pattern id or condition"; return false; } if (is_kw(w, "and") || is_kw(w, "or") || is_kw(w, "not")) { err = "misplaced keyword '" + w + "'"; return false; } + + // Function-style condition: churn(N) OP number | count(pat) OP number. + skip_ws(); + if (pos < src.size() && src[pos] == '(') { + ++pos; + std::string arg; + skip_ws(); + if (pos < src.size() && src[pos] != ')') { + if (!word(arg)) { + err = "expected an argument inside " + w + "(...)"; + return false; + } + } + if (!eat_sym(")")) { err = "expected ')' after " + w + "(...)"; return false; } + WhereOp op; + if (!parse_cmp_op(op)) { + err = w + "(...) must be followed by a comparison " + "(>, <, >=, <=, ==, !=)"; + return false; + } + double num; + if (!parse_number(num)) { + err = "expected a number after " + w + "(...) " "comparison"; + return false; + } + out = WhereNode{}; + out.kind = WhereNode::Leaf; + out.op = op; + out.num_rhs = num; + if (is_kw(w, "churn")) { + char *endp = nullptr; + long days = arg.empty() ? 0 : std::strtol(arg.c_str(), &endp, 10); + if (arg.empty() || endp != arg.c_str() + arg.size() || days <= 0) { + err = "churn(...) needs a positive integer day-window, " + "e.g. churn(30)"; + return false; + } + out.leaf_kind = WhereNode::Churn; + out.churn_days = static_cast(days); + } else if (is_kw(w, "count")) { + if (arg.empty()) { + err = "count(...) needs a pattern id, e.g. count(p0)"; + return false; + } + out.leaf_kind = WhereNode::Count; + out.id = arg; + } else { + err = "unknown condition '" + w + + "(...)' (supported: churn, count)"; + return false; + } + return true; + } + + // Bare-ident comparison: currently just `lang == go` / `lang != go`. + { + size_t save = pos; + WhereOp op; + if (parse_cmp_op(op)) { + if (!is_kw(w, "lang")) { + err = "unknown condition field '" + w + + "' (supported: lang)"; + return false; + } + if (op != WhereOp::Eq && op != WhereOp::Ne) { + err = "lang only supports == and !="; + return false; + } + std::string rhs; + if (!word(rhs)) { + err = "expected a language name after 'lang " + + std::string(op == WhereOp::Eq ? "==" : "!=") + "'"; + return false; + } + for (auto &c : rhs) c = static_cast(std::tolower((unsigned char)c)); + out = WhereNode{}; + out.kind = WhereNode::Leaf; + out.leaf_kind = WhereNode::Lang; + out.op = op; + out.str_rhs = std::move(rhs); + return true; + } + pos = save; + } + out = WhereNode{}; out.kind = WhereNode::Leaf; + out.leaf_kind = WhereNode::PatternPresent; out.id = std::move(w); return true; } }; -bool eval_where(const WhereNode &n, const std::vector &matched) { +bool compare_num(double v, WhereOp op, double rhs) { + switch (op) { + case WhereOp::Gt: return v > rhs; + case WhereOp::Lt: return v < rhs; + case WhereOp::Ge: return v >= rhs; + case WhereOp::Le: return v <= rhs; + case WhereOp::Eq: return v == rhs; + case WhereOp::Ne: return v != rhs; + } + return false; +} + +bool eval_where(const WhereNode &n, const std::vector &matched, + const std::vector &counts, const std::string &file, + const std::map> &churn, + const std::string &lang) { switch (n.kind) { - case WhereNode::Leaf: return matched[n.pat] != 0; - case WhereNode::Not: return !eval_where(n.kids[0], matched); + case WhereNode::Leaf: + switch (n.leaf_kind) { + case WhereNode::PatternPresent: + return matched[n.pat] != 0; + case WhereNode::Count: + return compare_num(static_cast(counts[n.pat]), n.op, + n.num_rhs); + case WhereNode::Churn: { + double v = 0.0; + auto wit = churn.find(n.churn_days); + if (wit != churn.end()) { + auto fit = wit->second.find(file); + if (fit != wit->second.end()) + v = static_cast(fit->second); + } + return compare_num(v, n.op, n.num_rhs); + } + case WhereNode::Lang: { + bool eq = lang == n.str_rhs; + return n.op == WhereOp::Eq ? eq : !eq; + } + } + return false; + case WhereNode::Not: + return !eval_where(n.kids[0], matched, counts, file, churn, lang); case WhereNode::And: for (const auto &k : n.kids) - if (!eval_where(k, matched)) return false; + if (!eval_where(k, matched, counts, file, churn, lang)) return false; return true; case WhereNode::Or: for (const auto &k : n.kids) - if (eval_where(k, matched)) return true; + if (eval_where(k, matched, counts, file, churn, lang)) return true; return false; } return false; @@ -140,9 +297,12 @@ template bool resolve_where_leaves(WhereNode &n, const Resolve &resolve, std::string &err) { if (n.kind == WhereNode::Leaf) { - if (!resolve(n.id, n.pat)) { - err = "unknown pattern '" + n.id + "' in -file-where"; - return false; + if (n.leaf_kind == WhereNode::PatternPresent || + n.leaf_kind == WhereNode::Count) { + if (!resolve(n.id, n.pat)) { + err = "unknown pattern '" + n.id + "' in -file-where"; + return false; + } } return true; } @@ -151,6 +311,16 @@ bool resolve_where_leaves(WhereNode &n, const Resolve &resolve, return true; } +void collect_churn_windows(const WhereNode &n, std::vector &out) { + if (n.kind == WhereNode::Leaf) { + if (n.leaf_kind == WhereNode::Churn && + std::find(out.begin(), out.end(), n.churn_days) == out.end()) + out.push_back(n.churn_days); + return; + } + for (const auto &k : n.kids) collect_churn_windows(k, out); +} + } // namespace namespace hpr { @@ -235,12 +405,21 @@ bool add_walker_inputs(const Cli &cli, Walker &walker, ScanStats &stats, std::vector build_patterns(const Cli &cli) { // Per-pattern name/word_boundary/utf8 overrides come from -name / // -patterns-from entries; -1 means inherit the global flag. + // + // Two passes, not one: regex-backed patterns (p0/p1/...) always end up + // as the vector's prefix and -ident groups (ident0/ident1/...) as its + // suffix, regardless of how the user interleaved -p/-ident on the + // command line. That ordering is what lets Matcher compile just the + // prefix while its reported pattern ids still index correctly into + // this vector (see run_search() in runner.cpp) — and as a side effect, + // adding an -ident group never renumbers existing -p patterns' ids. std::vector patterns; patterns.reserve(cli.patterns.size()); - for (size_t i = 0; i < cli.patterns.size(); ++i) { - const CliPattern &cp = cli.patterns[i]; + size_t regex_i = 0; + for (const auto &cp : cli.patterns) { + if (!cp.ident_terms.empty()) continue; Pattern p; - p.id = cp.name.empty() ? "p" + std::to_string(i) : cp.name; + p.id = cp.name.empty() ? "p" + std::to_string(regex_i) : cp.name; p.regexp = cp.regexp; p.case_insensitive = cp.case_insensitive; p.word_boundary = @@ -250,6 +429,17 @@ std::vector build_patterns(const Cli &cli) { p.extract_names = cp.extract_names; p.ref = cp.ref; patterns.push_back(std::move(p)); + ++regex_i; + } + size_t ident_i = 0; + for (const auto &cp : cli.patterns) { + if (cp.ident_terms.empty()) continue; + Pattern p; + p.id = cp.name.empty() ? "ident" + std::to_string(ident_i) : cp.name; + // regexp deliberately left empty — ident-backed patterns never + // reach Matcher::compile. + patterns.push_back(std::move(p)); + ++ident_i; } return patterns; } @@ -316,15 +506,36 @@ bool FileWhere::init(const std::string &expr, std::fprintf(stderr, "hprscript: %s\n", werr.c_str()); return false; } + collect_churn_windows(root_, churn_windows_); active_ = true; return true; } -bool FileWhere::pass(const std::vector &kept, - size_t pattern_count) const { +bool FileWhere::pass( + const std::vector &kept, size_t pattern_count, + const std::string &file, + const std::map> &churn) const { std::vector matched(pattern_count, 0); - for (const auto &mm : kept) matched[mm.pattern_index] = 1; - return eval_where(root_, matched); + std::vector counts(pattern_count, 0); + for (const auto &mm : kept) { + matched[mm.pattern_index] = 1; + ++counts[mm.pattern_index]; + } + std::string lang = auto_lang_for_path(file); + for (auto &c : lang) c = static_cast(std::tolower((unsigned char)c)); + return eval_where(root_, matched, counts, file, churn, lang); +} + +bool build_churn_map( + const std::vector &windows, + std::map> &out, + std::string &err) { + for (int days : windows) { + std::unordered_map m; + if (!git_churn(days, m, err)) return false; + out[days] = std::move(m); + } + return true; } const ScopeIndex *build_file_scope(const std::string &scope_lang, @@ -404,9 +615,12 @@ void TargetFilter::apply(std::vector &kept, const LineIndex &idx, MatchCollector::MatchCollector(const std::vector &patterns, std::vector rels, - bool git_added_lines) - : patterns_(patterns), rels_(std::move(rels)), git_added_(git_added_lines) { + bool git_added_lines, + std::vector ident_groups) + : patterns_(patterns), rels_(std::move(rels)), git_added_(git_added_lines), + ident_groups_(std::move(ident_groups)) { any_scope_rel_ = any_scope_relation(rels_); + ident_base_ = static_cast(patterns_.size() - ident_groups_.size()); } void MatchCollector::collect(Matcher &matcher, std::string_view content, @@ -423,6 +637,8 @@ void MatchCollector::collect(Matcher &matcher, std::string_view content, return true; }; matcher.scan(content, cb); + if (!ident_groups_.empty()) + scan_identifiers(content, ident_groups_, ident_base_, raw_); // Single sort-based dedup pass: by (pattern, from, -to). Within a // pattern, after this sort the longest match at each `from` comes diff --git a/src/pipeline.hpp b/src/pipeline.hpp index 783800f..1bbe81b 100644 --- a/src/pipeline.hpp +++ b/src/pipeline.hpp @@ -10,12 +10,14 @@ #include "cli.hpp" #include "common.hpp" #include "git.hpp" +#include "ident.hpp" #include "line_index.hpp" #include "matcher.hpp" #include "scope.hpp" #include "walker.hpp" #include +#include #include #include #include @@ -38,7 +40,12 @@ bool add_walker_inputs(const Cli &cli, Walker &walker, ScanStats &stats, // Build the final Pattern list from CLI state: auto `p` ids, per-pattern // name/word_boundary/utf8 overrides (-name / -patterns-from entries), global -// -w / -no-utf8 / -ucp defaults. +// -w / -no-utf8 / -ucp defaults. Regex-backed patterns (-p/-pi/-F/-Fi/ +// -patterns-from) always come first, in order, auto-numbered p0/p1/...; +// -ident groups follow, auto-numbered ident0/ident1/... independently — +// this ordering is load-bearing: it's what lets Matcher compile just the +// regex prefix while Vectorscan's reported pattern ids still line up with +// this vector's indices (see run_search() in runner.cpp). std::vector build_patterns(const Cli &cli); // Resolve a pattern identifier — explicit name, auto "p", or a bare @@ -63,16 +70,26 @@ bool resolve_relations(const std::vector &relations, // True when any relation needs an active -scope config. bool any_scope_relation(const std::vector &rels); -// -file-where: boolean predicate over pattern ids, evaluated per file -// against "did this pattern match at least once here". +// -file-where: boolean predicate over pattern ids and file metadata, +// evaluated per file. Leaves are either bare pattern ids ("did this pattern +// match at least once here") or comparisons: `count(pat) >= 3` (occurrence +// count), `churn(30) > 2` (commits touching the file in the last 30 days — +// see git_churn()), or `lang == go` (auto_lang_for_path()'s guess). class FileWhere { public: + enum class Op { Gt, Lt, Ge, Le, Eq, Ne }; + // Expression tree node. Public so the parser (pipeline.cpp) can build // trees; not intended for use outside this module. struct Node { enum Kind { And, Or, Not, Leaf } kind = Leaf; - std::string id; // Leaf: pattern id as written - uint32_t pat = 0; // Leaf: resolved pattern index + enum LeafKind { PatternPresent, Count, Churn, Lang } leaf_kind = PatternPresent; + std::string id; // PatternPresent/Count: pattern id as written + uint32_t pat = 0; // PatternPresent/Count: resolved pattern index + Op op = Op::Gt; // Count/Churn/Lang: the comparison operator + double num_rhs = 0; // Count/Churn: right-hand side + int churn_days = 0; // Churn: the N inside churn(N) + std::string str_rhs; // Lang: right-hand side, lowercased std::vector kids; }; @@ -83,14 +100,31 @@ class FileWhere { bool active() const { return active_; } + // Distinct churn(N) day-windows referenced in the predicate (empty if + // none). The caller (runner.cpp/edit.cpp) should populate a churn map + // for each — see build_churn_map() — before calling pass(). + const std::vector &churn_windows() const { return churn_windows_; } + // Does the predicate hold for a file whose surviving matches are `kept`? - bool pass(const std::vector &kept, size_t pattern_count) const; + // `churn` may be empty when churn_windows() is empty; a file/window + // missing from it is treated as churn 0. + bool pass(const std::vector &kept, size_t pattern_count, + const std::string &file, + const std::map> &churn = {}) const; private: bool active_ = false; Node root_; + std::vector churn_windows_; }; +// Run git_churn() once per distinct window in `windows`, populating `out`. +// Shared by run_search() and the edit runner so both -file-where users get +// identical churn semantics from one implementation. +bool build_churn_map(const std::vector &windows, + std::map> &out, + std::string &err); + // Build `scope` for one file when a scope config resolves for it. Returns // &scope when built, nullptr otherwise; build errors go to stderr and the // scan continues without scope annotation. When `out_cfg` is non-null it @@ -141,9 +175,13 @@ class TargetFilter { // buffer so per-file collection doesn't reallocate. class MatchCollector { public: - // `patterns` is borrowed and must outlive the collector. + // `patterns` is borrowed and must outlive the collector. `ident_groups` + // (default empty) are the -ident groups occupying `patterns`' tail — + // see build_patterns(); their matches come from scan_identifiers() + // (src/ident.hpp), not Vectorscan. MatchCollector(const std::vector &patterns, - std::vector rels, bool git_added_lines); + std::vector rels, bool git_added_lines, + std::vector ident_groups = {}); // Produce the filtered match list for one file buffer. `added` is this // file's -git-added-lines entry (nullptr = no added lines; ignored @@ -160,6 +198,8 @@ class MatchCollector { std::vector rels_; bool git_added_ = false; bool any_scope_rel_ = false; + std::vector ident_groups_; + uint32_t ident_base_ = 0; // patterns_ index where ident groups start std::vector raw_; // reused across files }; diff --git a/src/rank.cpp b/src/rank.cpp new file mode 100644 index 0000000..4d28434 --- /dev/null +++ b/src/rank.cpp @@ -0,0 +1,160 @@ +#include "rank.hpp" + +#include +#include +#include + +namespace hpr { + +namespace { + +struct Cluster { + uint32_t lo, hi; + std::set ids; + size_t count = 0; +}; + +// Sort by line and sweep: a new cluster starts whenever the gap to the +// previous point exceeds K lines. Shared by count_prox_clusters and the +// best-window search so both agree on what "one cluster" means. +std::vector build_clusters(std::vector> pts, + uint32_t K) { + std::vector clusters; + if (pts.empty()) return clusters; + std::sort(pts.begin(), pts.end(), + [](const auto &a, const auto &b) { return a.first < b.first; }); + size_t i = 0; + while (i < pts.size()) { + size_t j = i + 1; + while (j < pts.size() && pts[j].first - pts[j - 1].first <= K) ++j; + Cluster c; + c.lo = pts[i].first; + c.hi = pts[j - 1].first; + c.count = j - i; + for (size_t k = i; k < j; ++k) c.ids.insert(pts[k].second); + clusters.push_back(std::move(c)); + i = j; + } + return clusters; +} + +// The single most "interesting" cluster: most distinct pattern ids first +// (that's what the proximity bonus itself rewards), then most points, then +// earliest — deterministic tiebreaking for stable output. +std::pair best_window( + const std::vector> &pts, uint32_t K) { + if (pts.empty()) return {0, 0}; + std::vector clusters = build_clusters(pts, K); + const Cluster *best = &clusters.front(); + for (const auto &c : clusters) { + if (c.ids.size() != best->ids.size()) { + if (c.ids.size() > best->ids.size()) best = &c; + } else if (c.count > best->count) { + best = &c; + } + } + return {best->lo, best->hi}; +} + +} // namespace + +uint32_t count_prox_clusters(std::vector> pts, + bool rich) { + static constexpr uint32_t K = 20; + if (pts.size() < 2) return 0; + uint32_t total = 0; + for (const auto &c : build_clusters(std::move(pts), K)) { + if (c.ids.size() >= 2) + total += rich ? static_cast(c.ids.size() - 1) : 1u; + } + return total; +} + +std::vector rank_files(const RankInput &in) { + static constexpr uint32_t K = 20; + static constexpr double kCoverageExp = 1.5; + static constexpr double kProximityWeight = 0.5; + const double queried = in.total_queried > 0 + ? static_cast(in.total_queried) + : 1.0; + + // Corpus-surprise weighting: surprise_p = log((N+1)/(df_p+1)) + 1. Below + // 3 files the corpus is too small for document-frequency to mean + // anything, so every factor collapses to 1 (equivalent to being off). + std::map surprise_p; + const bool surprise_active = in.surprise && in.file_order.size() >= 3; + if (in.surprise) { + const double N = static_cast(in.file_order.size()); + std::map df; + for (const auto &id : in.queried_ids) df[id] = 0; + for (const auto &f : in.file_order) { + auto fit = in.per_file.find(f); + if (fit == in.per_file.end()) continue; + for (const auto &id : fit->second.matched_pat_ids) { + auto it = df.find(id); + if (it != df.end()) ++it->second; + } + } + for (const auto &id : in.queried_ids) { + double s = 1.0; + if (surprise_active) { + double dfp = static_cast(df[id]); + s = std::log((N + 1.0) / (dfp + 1.0)) + 1.0; + if (s < 1.0) s = 1.0; + } + surprise_p[id] = s; + } + } + + auto effective_weight = [&](const std::string &id) -> double { + double base = 1.0; + if (auto it = in.pattern_weights.find(id); + it != in.pattern_weights.end()) base = it->second; + if (in.surprise) { + auto sit = surprise_p.find(id); + if (sit != surprise_p.end()) base *= sit->second; + } + return base; + }; + + std::vector rows; + rows.reserve(in.file_order.size()); + for (const auto &f : in.file_order) { + auto fit = in.per_file.find(f); + if (fit == in.per_file.end()) continue; + const FileRank &fr = fit->second; + + double sum_weight = 0.0; + for (const auto &id : fr.matched_pat_ids) sum_weight += effective_weight(id); + double matched = static_cast(fr.matched_pat_ids.size()); + double coverage = matched / queried; + if (coverage > 1.0) coverage = 1.0; + double cov_factor = std::pow(coverage, kCoverageExp); + double divisor = std::log(static_cast(fr.line_count) + 10.0); + if (divisor <= 0.0) divisor = 1.0; + uint32_t clusters = count_prox_clusters(fr.match_points, in.rich_clusters); + double prox_bonus = kProximityWeight * static_cast(clusters); + + RankRow r; + r.file = f; + r.score = cov_factor * sum_weight / divisor + prox_bonus; + r.density = fr.line_count > 0 ? sum_weight / static_cast(fr.line_count) : 0.0; + r.matched_patterns.assign(fr.matched_pat_ids.begin(), fr.matched_pat_ids.end()); + if (in.surprise) { + r.surprise.reserve(fr.matched_pat_ids.size()); + for (const auto &id : fr.matched_pat_ids) { + auto sit = surprise_p.find(id); + r.surprise.emplace_back(id, sit != surprise_p.end() ? sit->second : 1.0); + } + } + std::tie(r.window_lo, r.window_hi) = best_window(fr.match_points, K); + rows.push_back(std::move(r)); + } + std::sort(rows.begin(), rows.end(), [](const RankRow &a, const RankRow &b) { + if (a.score != b.score) return a.score > b.score; + return a.density > b.density; + }); + return rows; +} + +} // namespace hpr diff --git a/src/rank.hpp b/src/rank.hpp new file mode 100644 index 0000000..0b5d923 --- /dev/null +++ b/src/rank.hpp @@ -0,0 +1,76 @@ +// File-relevance scoring shared by script-mode `rank` and quick-search +// `-hotspots`. Extracted from script.cpp so both can call the identical +// formula instead of maintaining two implementations. +// +// The formula combines three signals: coverage (fraction of queried pattern +// ids the file matches, exponentiated so matching everything dominates), +// weighted hits (Σ weight over distinct matched pattern ids, normalized by +// file size), and a proximity bonus (matches from ≥2 distinct patterns +// within a small line window — "matches that live together in one +// function"). See HPRSCRIPT.md's "Match ranking" section for the full +// derivation and the `rank_surprise` / `rank_rich_clusters` opt-ins. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace hpr { + +// Per-file accumulator fed incrementally while scanning. +struct FileRank { + std::set matched_pat_ids; + double raw_score = 0.0; // Σ weight of distinct matched pattern IDs. + uint32_t line_count = 1; + // Per-file pattern-id → local index (0..N-1) for compact match_points. + std::map pat_local_ids; + // Every match recorded as (1-based line, pat_local_id) for proximity + // sweep and best-window selection. + std::vector> match_points; +}; + +// Inputs the scoring formula needs, decoupled from any one caller's state. +struct RankInput { + std::vector file_order; // first-seen order (stable output) + std::map per_file; + std::set queried_ids; // non-absent pattern ids in the query + uint32_t total_queried = 0; // coverage denominator + std::map pattern_weights; // id -> weight + // Opt-in: fold a corpus-derived surprise factor (IDF-style) into the + // per-pattern weight — a pattern matching almost every file barely + // distinguishes them; a rare pattern is boosted. + bool surprise = false; + // Opt-in: scale the proximity bonus by (distinct_pat_ids_in_cluster - 1) + // instead of a flat contribution per qualifying cluster. + bool rich_clusters = false; +}; + +struct RankRow { + std::string file; + double score = 0.0; + double density = 0.0; // Σweight / line_count — diagnostic + tiebreaker + std::vector matched_patterns; + // Populated only when RankInput::surprise is set: per-pattern factor. + std::vector> surprise; + // The file's single densest match cluster (most distinct pattern ids, + // ties broken by point count), 1-based inclusive line range. {0,0} when + // the file has no recorded match points. + uint32_t window_lo = 0; + uint32_t window_hi = 0; +}; + +// Count proximity clusters: contiguous groups of match points where each +// adjacent pair is within K=20 lines (roughly "same function" in typical +// code) and the group spans ≥2 distinct pattern IDs. `rich` scales each +// cluster's contribution by (distinct_ids - 1) instead of a flat 1. +uint32_t count_prox_clusters(std::vector> pts, + bool rich = false); + +// Score every file in `in`, sorted score descending (density is the +// tiebreaker) — same order `flush_rank` prints in script mode. +std::vector rank_files(const RankInput &in); + +} // namespace hpr diff --git a/src/runner.cpp b/src/runner.cpp index 0c8e1c2..58de0e8 100644 --- a/src/runner.cpp +++ b/src/runner.cpp @@ -3,11 +3,14 @@ #include "extract.hpp" #include "file_io.hpp" #include "git.hpp" +#include "ident.hpp" #include "line_index.hpp" #include "matcher.hpp" #include "output.hpp" #include "pipeline.hpp" +#include "rank.hpp" #include "scope.hpp" +#include "seen.hpp" #include "walker.hpp" #include @@ -16,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -52,13 +56,103 @@ std::string normalise_shape(std::string_view line) { return s; } +// Shared per-file buffering for modes that need to look at a file's matches +// again after the whole scan completes (-sample, -hotspots): owns the +// content (the original mmap'd buffer doesn't outlive the walk callback) +// and rebuilds LineIndex/ScopeIndex against the owned copy so pointers stay +// valid. `kept` is left empty by callers that don't need per-match replay. +struct BufferedFile { + std::string path; + std::string content; + hpr::LineIndex idx; + hpr::ScopeIndex scope; + bool scope_built = false; + std::vector kept; +}; + +BufferedFile buffer_file(const std::string &display_name, + std::string_view content, + const std::string &eff_scope_lang, + const hpr::ScopeConfig &user_scope_custom, + bool rebuild_scope) { + BufferedFile bf; + bf.path = display_name; + bf.content.assign(content.data(), content.size()); + bf.idx.build(bf.content); + if (rebuild_scope) { + hpr::ScopeConfig sc = hpr::resolve_scope_for_file( + eff_scope_lang, user_scope_custom, display_name); + if (!sc.anchor_regex.empty()) { + std::string serr; + if (bf.scope.build(bf.content, sc, bf.idx, &serr)) + bf.scope_built = true; + } + } + return bf; +} + +// Fold one file's kept matches into a RankInput accumulator — shared by +// -hotspots and -budget, which both feed the same scoring formula +// (src/rank.hpp) from quick-search's per-file match lists. +void accumulate_rank_input(hpr::RankInput &in, const std::string &display_name, + const std::vector &kept, + const std::vector &patterns, + const hpr::LineIndex &idx) { + auto &fr = in.per_file[display_name]; + if (fr.matched_pat_ids.empty() && fr.match_points.empty()) + in.file_order.push_back(display_name); + fr.line_count = idx.line_count(); + static constexpr size_t kMaxMatchPoints = 4096; + for (const auto &m : kept) { + const hpr::Pattern &pat = patterns[m.pattern_index]; + if (fr.matched_pat_ids.insert(pat.id).second) + fr.raw_score += pat.weight; + if (fr.match_points.size() < kMaxMatchPoints) { + auto pit = fr.pat_local_ids.find(pat.id); + uint16_t local; + if (pit == fr.pat_local_ids.end()) { + local = static_cast( + std::min(fr.pat_local_ids.size(), 0xFFFFu)); + fr.pat_local_ids[pat.id] = local; + } else { + local = pit->second; + } + fr.match_points.emplace_back(idx.line_of(m.from), local); + } + } +} + +// Render a buffered file's matches via -elide's logic into an owned string +// instead of stdout, so -budget can measure the size before committing to +// it. Reuses Formatter/on_file_elide verbatim through a memory-backed +// FILE* (open_memstream is POSIX; this codebase already assumes POSIX +// throughout — mmap, fork/execvp, poll). +std::string render_elide_to_string(const BufferedFile &bf, + const hpr::OutputOptions &render_oo, + const hpr::SeenStore *seen, + std::vector *marks_out) { + char *buf = nullptr; + size_t len = 0; + FILE *mem = open_memstream(&buf, &len); + if (!mem) return {}; + { + hpr::Formatter fmt(render_oo, mem); + fmt.on_file_elide(bf.path, bf.kept, bf.content, bf.idx, + bf.scope_built ? &bf.scope : nullptr, seen, marks_out); + } + std::fclose(mem); + std::string result(buf, len); + std::free(buf); + return result; +} + } // namespace namespace hpr { int run_search(const Cli &cli) { if (cli.patterns.empty()) { - std::fprintf(stderr, "hprscript: -p required\n"); + std::fprintf(stderr, "hprscript: -p or -ident required\n"); return 2; } const auto t_start = std::chrono::steady_clock::now(); @@ -66,27 +160,50 @@ int run_search(const Cli &cli) { std::vector patterns = build_patterns(cli); + // -ident groups occupy patterns' tail (build_patterns guarantees this); + // they're matched by scan_identifiers(), never by Vectorscan, so only + // the regex-backed prefix is compiled. Vectorscan's reported pattern + // ids are 0..num_regex-1 either way, which is exactly where those + // patterns sit in the unified `patterns` vector too. + std::vector ident_groups; + for (const auto &cp : cli.patterns) { + if (!cp.ident_terms.empty()) ident_groups.push_back(IdentGroup{cp.ident_terms}); + } + const size_t num_regex = patterns.size() - ident_groups.size(); + std::vector rels; if (!resolve_relations(cli.relations, patterns, rels)) return 2; const bool any_scope_rel = any_scope_relation(rels); FileWhere fw; if (!fw.init(cli.file_where, patterns)) return 2; + std::map> churn_map; + if (!fw.churn_windows().empty()) { + std::string cerr; + if (!build_churn_map(fw.churn_windows(), churn_map, cerr)) { + std::fprintf(stderr, "hprscript: git: %s\n", cerr.c_str()); + return 2; + } + } TargetFilter tf; if (!tf.init(cli)) return 2; Matcher matcher; CompileError ce; - if (!matcher.compile(patterns, &ce)) { - std::fprintf(stderr, "hprscript: pattern compile failed: %s\n", - ce.message.c_str()); - if (ce.pattern_index >= 0 && - static_cast(ce.pattern_index) < patterns.size()) { - std::fprintf(stderr, " in pattern: %s\n", - patterns[ce.pattern_index].regexp.c_str()); + if (num_regex > 0) { + std::vector regex_patterns(patterns.begin(), + patterns.begin() + num_regex); + if (!matcher.compile(regex_patterns, &ce)) { + std::fprintf(stderr, "hprscript: pattern compile failed: %s\n", + ce.message.c_str()); + if (ce.pattern_index >= 0 && + static_cast(ce.pattern_index) < patterns.size()) { + std::fprintf(stderr, " in pattern: %s\n", + patterns[ce.pattern_index].regexp.c_str()); + } + return 2; } - return 2; } ExtractTable extract_table; @@ -158,8 +275,9 @@ int run_search(const Cli &cli) { // effect: records gain the `enclosing` annotation, which is useful // context anyway. std::string eff_scope_lang = cli.scope_lang; - if (tf.scope_needed() && eff_scope_lang.empty() && - cli.scope_pattern.empty()) + if ((tf.scope_needed() || oo.mode == OutputMode::Elide || + cli.budget_bytes > 0) && + eff_scope_lang.empty() && cli.scope_pattern.empty()) eff_scope_lang = "auto"; bool scope_enabled = !eff_scope_lang.empty() || (!cli.scope_pattern.empty() && !cli.scope_open.empty() && @@ -216,30 +334,144 @@ int run_search(const Cli &cli) { // Sample is incompatible with output modes that don't emit per-match // data — let the user know rather than silently doing nothing. if (oo.mode == OutputMode::FilesOnly || oo.mode == OutputMode::Counts || - oo.mode == OutputMode::Absent) { + oo.mode == OutputMode::Absent || oo.mode == OutputMode::Elide) { std::fprintf(stderr, - "hprscript: -sample requires a per-match output mode (default JSONL, -o, -format)\n"); + "hprscript: -sample requires a per-match output mode " + "(default JSONL, -o, -format, -llm) — -elide renders " + "whole-file batches and doesn't compose with -sample\n"); return 2; } } - struct SampleFile { - std::string path; - std::string content; - LineIndex idx; - ScopeIndex scope; - bool scope_built = false; - }; struct SampleRec { size_t file_idx; Match m; }; - std::vector sample_files; + std::vector sample_files; std::vector sample_recs; const size_t SAMPLE_REC_CAP = std::max( 100u * static_cast(cli.sample_n > 0 ? cli.sample_n : 1), 10000u); - MatchCollector collector(patterns, std::move(rels), cli.git_added_lines); + // Hotspot-mode buffering. When hotspots_n > 0 we accumulate the same + // rarity/coverage/proximity signal script mode's `rank` uses (see + // src/rank.hpp), one FileRank per file with ≥1 match, then score and + // emit the top N after the whole scan. Full file content is only + // buffered under -elide, which needs to re-render the file; JSONL/-llm + // hotspot rows need nothing but the accumulated RankInput. + const bool hotspotting = cli.hotspots_n > 0; + if (hotspotting) { + if (sampling) { + std::fprintf(stderr, + "hprscript: -hotspots cannot combine with -sample " + "(both buffer the scan to pick a subset)\n"); + return 2; + } + if (oo.mode != OutputMode::JsonLines && oo.mode != OutputMode::Llm && + oo.mode != OutputMode::Elide) { + std::fprintf(stderr, + "hprscript: -hotspots output is JSONL (default), " + "-llm, or -elide\n"); + return 2; + } + } + RankInput hs_input; + if (hotspotting) { + for (const auto &p : patterns) { + hs_input.pattern_weights[p.id] = p.weight; + hs_input.queried_ids.insert(p.id); + } + hs_input.total_queried = static_cast(patterns.size()); + } + std::vector hs_files; + std::unordered_map hs_file_idx; + const size_t HOTSPOT_REC_CAP = std::max( + 100u * static_cast(cli.hotspots_n > 0 ? cli.hotspots_n : 1), + 10000u); + size_t hs_buffered_matches = 0; + + // Budget-packing mode. Unlike -hotspots (a fixed top-N), -budget doesn't + // know ahead of time how many files will fit, so every file with ≥1 + // match is ranked and buffered in full (bounded by BUDGET_REC_CAP, same + // "silently truncates" convention as -sample/-hotspots). It defines its + // own output shape, so no other output-mode flag may be set. + const bool budgeting = cli.budget_bytes > 0; + if (budgeting) { + if (sampling || hotspotting) { + std::fprintf(stderr, + "hprscript: -budget cannot combine with " + "-sample/-hotspots\n"); + return 2; + } + if (cli.out_mode_set) { + std::fprintf(stderr, + "hprscript: -budget defines its own output shape — " + "drop -j/-f/-c/-o/-format/-absent/-llm/-elide\n"); + return 2; + } + } + RankInput bg_input; + if (budgeting) { + for (const auto &p : patterns) { + bg_input.pattern_weights[p.id] = p.weight; + bg_input.queried_ids.insert(p.id); + } + bg_input.total_queried = static_cast(patterns.size()); + } + std::vector bg_files; + std::unordered_map bg_file_idx; + static constexpr size_t BUDGET_REC_CAP = 20000; + size_t bg_buffered_matches = 0; + OutputOptions bg_render_oo; + if (budgeting) { + bg_render_oo.mode = OutputMode::Elide; + bg_render_oo.context_before = cli.context_before; + bg_render_oo.context_after = cli.context_after; + bg_render_oo.max_match_bytes = cli.max_match_bytes; + bg_render_oo.max_context_bytes = cli.max_context_bytes; + bg_render_oo.max_block_bytes = cli.max_block_bytes; + } + + // -order-by: sorts -f/-c output instead of streaming it in walk order. + // `score` needs the same whole-scan RankInput accumulation as + // -hotspots/-budget; `count`/`path` just need the per-file row buffered. + // No separate check against -sample/-hotspots/-budget is needed here: + // each of those already requires an output mode that isn't -f/-c, so + // they can never reach this point already combined with -order-by. + const bool ordering = cli.order_by != Cli::OrderBy::None; + if (ordering && oo.mode != OutputMode::FilesOnly && + oo.mode != OutputMode::Counts) { + std::fprintf(stderr, "hprscript: -order-by requires -f or -c output\n"); + return 2; + } + struct OrderRow { + std::string file; + uint64_t count; + }; + std::vector order_rows; + RankInput ord_input; + if (ordering && cli.order_by == Cli::OrderBy::Score) { + for (const auto &p : patterns) { + ord_input.pattern_weights[p.id] = p.weight; + ord_input.queried_ids.insert(p.id); + } + ord_input.total_queried = static_cast(patterns.size()); + } + + // -seen: cross-invocation dedup, only meaningful where there's a + // "chunk" to collapse (-elide's own mode, or -budget's internal use of + // it). Loaded once up front; rewritten once at the end with whatever + // this run actually displayed in full (see SeenMark's doc comment for + // why -budget only commits marks for its full-render tier). + const bool seen_active = !cli.seen_path.empty(); + if (seen_active && oo.mode != OutputMode::Elide && !budgeting) { + std::fprintf(stderr, "hprscript: -seen requires -elide or -budget\n"); + return 2; + } + SeenStore seen_store; + if (seen_active) seen_store.load(cli.seen_path); + + MatchCollector collector(patterns, std::move(rels), cli.git_added_lines, + ident_groups); const bool have_rels = !cli.relations.empty(); auto scan_buf = [&](const std::string &display_name, @@ -267,7 +499,8 @@ int run_search(const Cli &cli) { // -file-where: emit this file's matches only when the predicate over // its matched-pattern set holds. - if (fw.active() && !fw.pass(kept, patterns.size())) { + if (fw.active() && + !fw.pass(kept, patterns.size(), display_name, churn_map)) { fmt.on_file_end(display_name, false); return true; } @@ -308,25 +541,11 @@ int run_search(const Cli &cli) { if (sampling) { if (kept.empty()) return true; if (sample_recs.size() >= SAMPLE_REC_CAP) return true; - // Take ownership of file content + indices the sampler will need. - SampleFile sf; - sf.path = display_name; - sf.content.assign(content.data(), content.size()); - sf.idx.build(sf.content); - if (scope_ptr) { - // Re-build against the owned content so pointers stay valid. - ScopeConfig sc = resolve_scope_for_file(eff_scope_lang, - user_scope_custom, - display_name); - if (!sc.anchor_regex.empty()) { - std::string serr; - if (sf.scope.build(sf.content, sc, sf.idx, &serr)) { - sf.scope_built = true; - } - } - } size_t fidx = sample_files.size(); - sample_files.push_back(std::move(sf)); + sample_files.push_back(buffer_file(display_name, content, + eff_scope_lang, + user_scope_custom, + scope_ptr != nullptr)); for (const auto &m : kept) { if (sample_recs.size() >= SAMPLE_REC_CAP) break; sample_recs.push_back({fidx, m}); @@ -334,17 +553,78 @@ int run_search(const Cli &cli) { return true; } - uint64_t per_file = 0; - for (const auto &m : kept) { - had_match = true; - ++per_file; - const Pattern &pat = patterns[m.pattern_index]; - fmt.on_match(display_name, pat, m, content, idx, scope_ptr); - if (fmt.over_budget()) break; + if (hotspotting) { + if (kept.empty()) return true; + accumulate_rank_input(hs_input, display_name, kept, patterns, idx); + if (oo.mode == OutputMode::Elide && + hs_buffered_matches < HOTSPOT_REC_CAP) { + BufferedFile bf = buffer_file(display_name, content, + eff_scope_lang, user_scope_custom, + scope_ptr != nullptr); + bf.kept = kept; + hs_buffered_matches += kept.size(); + hs_file_idx[display_name] = hs_files.size(); + hs_files.push_back(std::move(bf)); + } + return true; + } + + if (budgeting) { + if (kept.empty()) return true; + accumulate_rank_input(bg_input, display_name, kept, patterns, idx); + if (bg_buffered_matches < BUDGET_REC_CAP) { + BufferedFile bf = buffer_file(display_name, content, + eff_scope_lang, user_scope_custom, + scope_ptr != nullptr); + bf.kept = kept; + bg_buffered_matches += kept.size(); + bg_file_idx[display_name] = bg_files.size(); + bg_files.push_back(std::move(bf)); + } + return true; + } + + if (ordering) { + // Match -f's existing semantics (files with zero matches are + // omitted) and -c's (every scanned file gets a row, even :0). + if (oo.mode == OutputMode::FilesOnly && kept.empty()) return true; + order_rows.push_back({display_name, static_cast(kept.size())}); + if (cli.order_by == Cli::OrderBy::Score && !kept.empty()) + accumulate_rank_input(ord_input, display_name, kept, patterns, idx); + return true; + } + + if (oo.mode == OutputMode::Elide) { + // Whole-file batch render — -m caps how many of this file's + // matches participate, but there's no natural mid-file stopping + // point the way per-match modes have one. + std::vector capped = kept; if (cli.per_file_limit > 0 && - per_file >= static_cast(cli.per_file_limit)) break; - if (cli.limit > 0 && - fmt.emitted() >= static_cast(cli.limit)) break; + capped.size() > static_cast(cli.per_file_limit)) + capped.resize(static_cast(cli.per_file_limit)); + if (!capped.empty()) { + had_match = true; + std::vector marks; + fmt.on_file_elide(display_name, capped, content, idx, scope_ptr, + seen_active ? &seen_store : nullptr, + seen_active ? &marks : nullptr); + // -elide's render is the final output — nothing measured + // and discarded like -budget — so every mark commits. + if (seen_active) for (const auto &m : marks) seen_store.mark(m); + } + } else { + uint64_t per_file = 0; + for (const auto &m : kept) { + had_match = true; + ++per_file; + const Pattern &pat = patterns[m.pattern_index]; + fmt.on_match(display_name, pat, m, content, idx, scope_ptr); + if (fmt.over_budget()) break; + if (cli.per_file_limit > 0 && + per_file >= static_cast(cli.per_file_limit)) break; + if (cli.limit > 0 && + fmt.emitted() >= static_cast(cli.limit)) break; + } } fmt.on_file_end(display_name, had_match); @@ -401,7 +681,7 @@ int run_search(const Cli &cli) { for (size_t ri = 0; ri < sample_recs.size(); ++ri) { const auto &r = sample_recs[ri]; - const SampleFile &sf = sample_files[r.file_idx]; + const BufferedFile &sf = sample_files[r.file_idx]; uint32_t line = sf.idx.line_of(r.m.from); std::string_view ltext = sf.idx.line_text(line); std::string shape = normalise_shape(ltext.data() ? ltext : std::string_view{}); @@ -457,7 +737,7 @@ int run_search(const Cli &cli) { for (size_t ri : selected) { const auto &r = sample_recs[ri]; - const SampleFile &sf = sample_files[r.file_idx]; + const BufferedFile &sf = sample_files[r.file_idx]; const Pattern &pat = patterns[r.m.pattern_index]; fmt.on_match(sf.path, pat, r.m, sf.content, sf.idx, sf.scope_built ? &sf.scope : nullptr); @@ -469,13 +749,216 @@ int run_search(const Cli &cli) { } } + uint64_t hs_emitted_rows = 0; + if (hotspotting) { + std::vector rows = rank_files(hs_input); + if (rows.size() > static_cast(cli.hotspots_n)) + rows.resize(static_cast(cli.hotspots_n)); + for (const auto &r : rows) { + if (oo.mode == OutputMode::Elide) { + auto it = hs_file_idx.find(r.file); + if (it == hs_file_idx.end()) + continue; // buffering cap hit before this file — skip + const BufferedFile &bf = hs_files[it->second]; + if (bf.kept.empty()) continue; + std::vector marks; + fmt.on_file_elide(bf.path, bf.kept, bf.content, bf.idx, + bf.scope_built ? &bf.scope : nullptr, + seen_active ? &seen_store : nullptr, + seen_active ? &marks : nullptr); + if (seen_active) for (const auto &m : marks) seen_store.mark(m); + ++hs_emitted_rows; + } else if (oo.mode == OutputMode::Llm) { + std::string line = r.file; + line += ':'; + line += std::to_string(r.window_lo); + line += '-'; + line += std::to_string(r.window_hi); + line += " score="; + char buf[32]; + std::snprintf(buf, sizeof(buf), "%g", r.score); + line += buf; + line += " patterns="; + for (size_t i = 0; i < r.matched_patterns.size(); ++i) { + if (i) line += ','; + line += r.matched_patterns[i]; + } + line += '\n'; + std::fwrite(line.data(), 1, line.size(), stdout); + ++hs_emitted_rows; + } else { + std::string s = "{\"type\":\"hotspot\",\"file\":\""; + json_escape_to(s, r.file); + s += "\",\"score\":"; + char buf[32]; + std::snprintf(buf, sizeof(buf), "%g", r.score); + s += buf; + s += ",\"line_start\":"; + s += std::to_string(r.window_lo); + s += ",\"line_end\":"; + s += std::to_string(r.window_hi); + s += ",\"patterns\":["; + for (size_t i = 0; i < r.matched_patterns.size(); ++i) { + if (i) s += ','; + s += '"'; + json_escape_to(s, r.matched_patterns[i]); + s += '"'; + } + s += "]}\n"; + std::fwrite(s.data(), 1, s.size(), stdout); + ++hs_emitted_rows; + } + } + } + + uint64_t bg_emitted_rows = 0; + if (budgeting) { + std::vector rows = rank_files(bg_input); + int64_t remaining = static_cast(cli.budget_bytes); + std::vector dropped; + size_t full_count = 0, compact_count = 0; + + auto compact_line = [](const RankRow &r) { + std::string line = r.file; + line += ':'; + line += std::to_string(r.window_lo); + line += '-'; + line += std::to_string(r.window_hi); + line += " score="; + char buf[32]; + std::snprintf(buf, sizeof(buf), "%g", r.score); + line += buf; + line += " patterns="; + for (size_t i = 0; i < r.matched_patterns.size(); ++i) { + if (i) line += ','; + line += r.matched_patterns[i]; + } + line += " (compact — full render didn't fit the budget)\n"; + return line; + }; + + for (const auto &r : rows) { + if (remaining <= 0) { dropped.push_back(r.file); continue; } + auto it = bg_file_idx.find(r.file); + if (it == bg_file_idx.end()) { // buffering cap hit — never captured + dropped.push_back(r.file); + continue; + } + const BufferedFile &bf = bg_files[it->second]; + if (bf.kept.empty()) { dropped.push_back(r.file); continue; } + + // Measured against `seen` for collapse sizing either way, but + // marks are only committed to the real store if this render is + // the one that actually ends up on screen — a render measured + // here and then degraded to compact/dropped must not be + // recorded as "shown" (see SeenMark's doc comment). + std::vector marks; + std::string full_text = render_elide_to_string( + bf, bg_render_oo, seen_active ? &seen_store : nullptr, + seen_active ? &marks : nullptr); + if (static_cast(full_text.size()) <= remaining) { + std::fwrite(full_text.data(), 1, full_text.size(), stdout); + remaining -= static_cast(full_text.size()); + ++full_count; + ++bg_emitted_rows; + if (seen_active) for (const auto &m : marks) seen_store.mark(m); + continue; + } + std::string compact = compact_line(r); + if (static_cast(compact.size()) <= remaining) { + std::fwrite(compact.data(), 1, compact.size(), stdout); + remaining -= static_cast(compact.size()); + ++compact_count; + ++bg_emitted_rows; + continue; + } + dropped.push_back(r.file); + } + + if (compact_count > 0 || !dropped.empty()) { + std::string footer = "--- budget: "; + footer += std::to_string(full_count); + footer += " file(s) in full, "; + footer += std::to_string(compact_count); + footer += " compact, "; + footer += std::to_string(dropped.size()); + footer += " dropped"; + if (!dropped.empty()) { + footer += ": "; + static constexpr size_t kMaxNamed = 10; + for (size_t i = 0; i < dropped.size() && i < kMaxNamed; ++i) { + if (i) footer += ", "; + footer += dropped[i]; + } + if (dropped.size() > kMaxNamed) { + footer += " (+"; + footer += std::to_string(dropped.size() - kMaxNamed); + footer += " more)"; + } + } + footer += " ---\n"; + std::fwrite(footer.data(), 1, footer.size(), stdout); + } + } + + uint64_t ord_emitted = 0; + if (ordering) { + std::unordered_map score_by_file; + if (cli.order_by == Cli::OrderBy::Score) { + for (const auto &r : rank_files(ord_input)) score_by_file[r.file] = r.score; + } + std::stable_sort(order_rows.begin(), order_rows.end(), + [&](const OrderRow &a, const OrderRow &b) { + switch (cli.order_by) { + case Cli::OrderBy::Path: + return a.file < b.file; + case Cli::OrderBy::Count: + return a.count > b.count; // descending + case Cli::OrderBy::Score: { + double sa = score_by_file.count(a.file) ? score_by_file[a.file] : 0.0; + double sb = score_by_file.count(b.file) ? score_by_file[b.file] : 0.0; + return sa > sb; // descending; stable_sort keeps walk order on ties + } + default: + return false; + } + }); + for (const auto &r : order_rows) { + std::string line = r.file; + if (oo.mode == OutputMode::Counts) { + line += ':'; + line += std::to_string(r.count); + } + line += '\n'; + std::fwrite(line.data(), 1, line.size(), stdout); + // Mirrors Formatter::emitted()'s existing -f/-c semantics: it + // counts matches, not rows/files — -c prints a ":0" row for a + // clean file, but that shouldn't flip -summary/exit code to + // "found something". + ord_emitted += r.count; + } + } + + if (seen_active) { + std::string serr; + if (!seen_store.save(cli.seen_path, &serr)) { + // The real output already printed successfully — a state-file + // write failure shouldn't flip the exit code, just warn. + std::fprintf(stderr, "hprscript: -seen: %s\n", serr.c_str()); + } + } + fmt.on_complete(); if (cli.summary) { auto elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - t_start) .count(); - emit_summary_record(stats, fmt.emitted(), + uint64_t emitted_for_summary = fmt.emitted(); + if (hotspotting) emitted_for_summary = hs_emitted_rows; + if (budgeting) emitted_for_summary = bg_emitted_rows; + if (ordering) emitted_for_summary = ord_emitted; + emit_summary_record(stats, emitted_for_summary, static_cast(elapsed)); } if (cli.require_complete && @@ -490,6 +973,9 @@ int run_search(const Cli &cli) { // Exit code semantics follow grep: 0 if any match emitted (or absent // files printed), 1 if no output, 2 already returned earlier on errors. + if (hotspotting) return hs_emitted_rows > 0 ? 0 : 1; + if (budgeting) return bg_emitted_rows > 0 ? 0 : 1; + if (ordering) return ord_emitted > 0 ? 0 : 1; return fmt.emitted() > 0 ? 0 : 1; } diff --git a/src/script.cpp b/src/script.cpp index 05c2200..e08b9c9 100644 --- a/src/script.cpp +++ b/src/script.cpp @@ -29,6 +29,7 @@ #include "line_index.hpp" #include "matcher.hpp" #include "output.hpp" +#include "rank.hpp" #include "scope.hpp" #include "value.hpp" #include "walker.hpp" @@ -262,18 +263,6 @@ class VarStore { std::map values_; }; -// ---- Per-file rank info ----------------------------------------------------- - -struct FileRank { - std::set matched_pat_ids; - double raw_score = 0.0; // Σ weight of distinct matched pattern IDs. - uint32_t line_count = 1; - // Per-file pattern-id → local index (0..N-1) for compact match_points. - std::map pat_local_ids; - // Every match recorded as (1-based line, pat_local_id) for proximity sweep. - std::vector> match_points; -}; - // ---- Script-wide state ------------------------------------------------------ struct ScriptState { @@ -2227,142 +2216,22 @@ void flush_groups(ScriptState &state) { state.group_order.clear(); } -// Count proximity clusters: contiguous groups of match points where each -// adjacent pair is within K lines, and the group spans ≥2 distinct pattern -// IDs. K=20 is roughly "same function" in typical code. -// -// When `rich` is false, each qualifying cluster contributes 1 (original -// behaviour). When `rich` is true, each cluster contributes -// (distinct_pat_ids_in_cluster − 1) so that denser co-occurrences score -// higher — at the 2-distinct minimum this still contributes 1, preserving -// the original score for two-pattern clusters. -static uint32_t count_prox_clusters( - std::vector> pts, - bool rich = false) { - static constexpr uint32_t K = 20; - if (pts.size() < 2) return 0; - std::sort(pts.begin(), pts.end(), - [](const auto &a, const auto &b) { return a.first < b.first; }); - uint32_t total = 0; - size_t i = 0; - while (i < pts.size()) { - size_t j = i + 1; - while (j < pts.size() && pts[j].first - pts[j - 1].first <= K) ++j; - if (j - i >= 2) { - std::set ids; - for (size_t k = i; k < j; ++k) ids.insert(pts[k].second); - if (ids.size() >= 2) { - total += rich ? (uint32_t)(ids.size() - 1) : 1u; - } - } - i = j; - } - return total; -} - +// Scoring itself lives in rank.hpp/rank.cpp (shared with quick-search's +// -hotspots); this just adapts ScriptState's accumulated fields into a +// RankInput and prints the resulting rows in script mode's JSON shape. void flush_rank(ScriptState &state) { if (!state.rank_enabled) return; - struct Row { - std::string file; - double score; - double density; - std::vector pats; - // Only populated when rank_surprise is on; lists surprise factor of - // each matched pattern in this file. - std::vector> surprise; - }; - // Tunables. Coverage exponent makes "matches all queried patterns" a - // strong multiplier; density divisor gently penalizes huge files; - // proximity bonus rewards co-located matches (cohesion). - static constexpr double kCoverageExp = 1.5; - static constexpr double kProximityWeight = 0.5; - const double queried = state.rank_total_queried > 0 - ? (double)state.rank_total_queried - : 1.0; - - // Change 1 — corpus-surprise weighting. surprise_p = log((N+1)/(df_p+1)) + 1. - // N = number of files in the rank table (contributed at least one match); - // df_p = number of those files whose matched-pattern set contains p. - // With N<3 the corpus is too small for document-frequency to be - // meaningful, so all factors collapse to 1 (i.e. base user_weight). - std::map surprise_p; - const bool surprise_active = - state.rank_surprise && state.rank_file_order.size() >= 3; - if (state.rank_surprise) { - const double N = (double)state.rank_file_order.size(); - std::map df; - for (const auto &id : state.rank_queried_ids) df[id] = 0; - for (const auto &f : state.rank_file_order) { - const FileRank &fr = state.rank_per_file[f]; - for (const auto &id : fr.matched_pat_ids) { - auto it = df.find(id); - if (it != df.end()) ++it->second; - } - } - for (const auto &id : state.rank_queried_ids) { - double s = 1.0; - if (surprise_active) { - double dfp = (double)df[id]; - s = std::log((N + 1.0) / (dfp + 1.0)) + 1.0; - if (s < 1.0) s = 1.0; - } - surprise_p[id] = s; - } - } - auto effective_weight = [&](const std::string &id) -> double { - double base = 1.0; - if (auto it = state.pattern_weights.find(id); - it != state.pattern_weights.end()) base = it->second; - if (state.rank_surprise) { - auto sit = surprise_p.find(id); - if (sit != surprise_p.end()) base *= sit->second; - } - return base; - }; + RankInput in; + in.file_order = state.rank_file_order; + in.per_file = state.rank_per_file; + in.queried_ids = state.rank_queried_ids; + in.total_queried = state.rank_total_queried; + in.pattern_weights = state.pattern_weights; + in.surprise = state.rank_surprise; + in.rich_clusters = state.rank_rich_clusters; - std::vector rows; - rows.reserve(state.rank_file_order.size()); - for (const auto &f : state.rank_file_order) { - const FileRank &fr = state.rank_per_file[f]; - // Recompute Σ effective_weight over distinct matched patterns. When - // rank_surprise is off, effective_weight == user_weight and this - // matches the value already accumulated in fr.raw_score. - double sum_weight = 0.0; - for (const auto &id : fr.matched_pat_ids) { - sum_weight += effective_weight(id); - } - double matched = (double)fr.matched_pat_ids.size(); - double coverage = matched / queried; - if (coverage > 1.0) coverage = 1.0; - double cov_factor = std::pow(coverage, kCoverageExp); - double divisor = std::log((double)fr.line_count + 10.0); - if (divisor <= 0.0) divisor = 1.0; - uint32_t clusters = count_prox_clusters(fr.match_points, - state.rank_rich_clusters); - double prox_bonus = kProximityWeight * (double)clusters; - - Row r; - r.file = f; - r.score = cov_factor * sum_weight / divisor + prox_bonus; - r.density = fr.line_count > 0 - ? sum_weight / (double)fr.line_count - : 0.0; - r.pats.assign(fr.matched_pat_ids.begin(), fr.matched_pat_ids.end()); - if (state.rank_surprise) { - r.surprise.reserve(fr.matched_pat_ids.size()); - for (const auto &id : fr.matched_pat_ids) { - auto sit = surprise_p.find(id); - r.surprise.emplace_back( - id, sit != surprise_p.end() ? sit->second : 1.0); - } - } - rows.push_back(std::move(r)); - } - std::sort(rows.begin(), rows.end(), [](const Row &a, const Row &b) { - if (a.score != b.score) return a.score > b.score; - return a.density > b.density; - }); + std::vector rows = rank_files(in); for (const auto &r : rows) { std::string line = "{\"file\":\""; json_escape_to(line, r.file); @@ -2382,10 +2251,10 @@ void flush_rank(ScriptState &state) { line += buf; } line += ",\"matched_patterns\":["; - for (size_t i = 0; i < r.pats.size(); ++i) { + for (size_t i = 0; i < r.matched_patterns.size(); ++i) { if (i) line += ','; line += '"'; - json_escape_to(line, r.pats[i]); + json_escape_to(line, r.matched_patterns[i]); line += '"'; } line += "]"; diff --git a/src/seen.cpp b/src/seen.cpp new file mode 100644 index 0000000..12be2d2 --- /dev/null +++ b/src/seen.cpp @@ -0,0 +1,116 @@ +#include "seen.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hpr { + +uint64_t fnv1a(std::string_view data) { + uint64_t h = 0xcbf29ce484222325ULL; // FNV-1a 64-bit offset basis + for (unsigned char c : data) { + h ^= c; + h *= 0x100000001b3ULL; // FNV-1a 64-bit prime + } + return h; +} + +void SeenStore::load(const std::string &path) { + std::ifstream in(path); + if (!in) return; // missing/unreadable = empty store, not an error + std::string line; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.empty()) continue; + // \t\t\t + size_t t1 = line.find('\t'); + if (t1 == std::string::npos) continue; + size_t t2 = line.find('\t', t1 + 1); + if (t2 == std::string::npos) continue; + size_t t3 = line.find('\t', t2 + 1); + if (t3 == std::string::npos) continue; + std::string file = line.substr(0, t1); + SeenEntry e; + char *endp = nullptr; + e.line_start = static_cast( + std::strtoul(line.c_str() + t1 + 1, &endp, 10)); + if (endp != line.c_str() + t2) continue; // malformed — skip the line + e.line_end = static_cast( + std::strtoul(line.c_str() + t2 + 1, &endp, 10)); + if (endp != line.c_str() + t3) continue; + e.hash = std::strtoull(line.c_str() + t3 + 1, &endp, 16); + if (endp != line.c_str() + line.size()) continue; + prior_[file].push_back(e); + } +} + +bool SeenStore::seen_unchanged(const std::string &file, uint32_t line_start, + uint32_t line_end, uint64_t hash) const { + auto it = prior_.find(file); + if (it == prior_.end()) return false; + for (const auto &e : it->second) { + if (e.line_start == line_start && e.line_end == line_end) + return e.hash == hash; + } + return false; +} + +void SeenStore::mark(const std::string &file, uint32_t line_start, + uint32_t line_end, uint64_t hash) { + current_[file].push_back({line_start, line_end, hash}); +} + +bool SeenStore::save(const std::string &path, std::string *err) const { + std::ostringstream out; + for (const auto &kv : current_) { + for (const auto &e : kv.second) { + out << kv.first << '\t' << e.line_start << '\t' << e.line_end + << '\t' << std::hex << e.hash << std::dec << '\n'; + } + } + std::string data = out.str(); + + std::filesystem::path target(path); + std::filesystem::path dir = + target.parent_path().empty() ? "." : target.parent_path(); + std::string tmpl = dir.string() + "/.hpr-seen." + target.filename().string() + + ".XXXXXX"; + std::vector tmp(tmpl.begin(), tmpl.end()); + tmp.push_back('\0'); + int fd = ::mkstemp(tmp.data()); + if (fd < 0) { + if (err) *err = std::string("cannot create temp file near ") + path + + ": " + std::strerror(errno); + return false; + } + std::string tmp_path(tmp.data()); + bool ok = true; + size_t off = 0; + while (off < data.size()) { + ssize_t n = ::write(fd, data.data() + off, data.size() - off); + if (n < 0) { + if (errno == EINTR) continue; + if (err) *err = std::string("write failed for ") + tmp_path + ": " + + std::strerror(errno); + ok = false; + break; + } + off += static_cast(n); + } + if (ok) ::fchmod(fd, 0644); + ::close(fd); + if (ok && std::rename(tmp_path.c_str(), path.c_str()) != 0) { + if (err) *err = std::string("rename failed for ") + path + ": " + + std::strerror(errno); + ok = false; + } + if (!ok) ::unlink(tmp_path.c_str()); + return ok; +} + +} // namespace hpr diff --git a/src/seen.hpp b/src/seen.hpp new file mode 100644 index 0000000..655a6d6 --- /dev/null +++ b/src/seen.hpp @@ -0,0 +1,69 @@ +// Cross-invocation "already shown" state for -seen: lets repeated agent +// queries against -elide/-budget skip re-paying tokens for chunks that +// haven't changed since the last run. +#pragma once + +#include +#include +#include +#include +#include + +namespace hpr { + +// FNV-1a over raw bytes — fast, non-cryptographic, good enough for change +// detection (not a security boundary). +uint64_t fnv1a(std::string_view data); + +// One remembered chunk: a scope's line range and content hash as of the +// last run that rendered it in full. +struct SeenEntry { + uint32_t line_start = 0; + uint32_t line_end = 0; + uint64_t hash = 0; +}; + +// A chunk this run examined, recorded whether it collapsed or rendered in +// full — the caller decides whether to commit it (see mark()) once it +// knows the chunk actually made it into the final output. This split +// matters for -budget: it measures a file's full render to decide whether +// the render fits the byte budget, and a measurement that's ultimately +// discarded (degraded to a compact summary or dropped) must not be +// recorded as "shown". +struct SeenMark { + std::string file; + uint32_t line_start = 0; + uint32_t line_end = 0; + uint64_t hash = 0; +}; + +class SeenStore { +public: + // Load `path`'s prior entries. A missing or unreadable file is treated + // as an empty store (first run) rather than an error — -seen is a soft + // cache, not a hard contract; a corrupt line is skipped, not fatal. + void load(const std::string &path); + + // True when `file`'s [line_start,line_end] chunk was recorded last run + // with this exact hash — i.e. safe to collapse. + bool seen_unchanged(const std::string &file, uint32_t line_start, + uint32_t line_end, uint64_t hash) const; + + // Commit a chunk into this run's state (called only for chunks that + // actually reached the final output — see SeenMark's doc comment). + void mark(const std::string &file, uint32_t line_start, uint32_t line_end, + uint64_t hash); + void mark(const SeenMark &m) { mark(m.file, m.line_start, m.line_end, m.hash); } + + // Atomically rewrite `path` with this run's marked entries (mkstemp + + // rename). Entries never marked this run — collapsed away, or simply + // not visited — are dropped, so the file always reflects exactly what + // the most recent run actually displayed. + bool save(const std::string &path, std::string *err) const; + +private: + std::unordered_map> prior_; + std::unordered_map> current_; +}; + +} // namespace hpr diff --git a/tests/run.sh b/tests/run.sh index 1e6a456..b41e14a 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1888,6 +1888,505 @@ expect_contains "-list-scopes rejects -lines" "does not apply" "$OUT" rm -rf "$SC" +# --------------------------------------------------------------------------- +section "elided scope output (-elide)" +EL=$(mktemp -d) +cat > "$EL/el.go" <<'GOEOF' +package main + +// target: top-level marker + +func ShortFn() { + target() +} + +func LongFn() { + target() + step1() + step2() + step3() + step4() + step5() + step6() + step7() + step8() + target() +} +GOEOF + +OUT=$("$BIN" -p 'target' -elide -scope go "$EL/el.go") +expect_contains "-elide: file header" "$EL/el.go" "$OUT" +expect_contains "-elide: scope header shows kind+name" "5-7 func ShortFn" "$OUT" +expect_contains "-elide: signature line kept" "func LongFn() {" "$OUT" +expect_contains "-elide: big gap elided" "… (+8 lines)" "$OUT" +expect_contains "-elide: closing line kept" "}" "$OUT" +expect_contains "-elide: orphan match outside any scope" "3: // target: top-level marker" "$OUT" +SHORT_BLOCK=$(printf '%s\n' "$OUT" | awk '/5-7 func ShortFn/{f=1} f{print} f && /^$/{exit}') +if [[ "$SHORT_BLOCK" != *"…"* ]]; then + report ok "-elide: short function has no elision marker" +else + report fail "-elide: short function has no elision marker" +fi + +# --- implicit -scope auto: no -scope flag needed to trigger eliding +OUT=$("$BIN" -p 'target' -elide "$EL/el.go") +expect_contains "-elide implies -scope auto" "func ShortFn" "$OUT" + +# --- mutually exclusive with other output modes and with -sample +OUT=$("$BIN" -p x -elide -llm 2>&1) ; RC=$? +expect_contains "-elide output-mode conflict" "mutually exclusive" "$OUT" +[[ "$RC" == "2" ]] && report ok "-elide output-mode conflict exit 2" || report fail "-elide output-mode conflict exit (got $RC)" +OUT=$("$BIN" -p x -elide -sample 5 2>&1) ; RC=$? +expect_contains "-elide + -sample rejected" "doesn't compose with -sample" "$OUT" +[[ "$RC" == "2" ]] && report ok "-elide + -sample exit 2" || report fail "-elide + -sample exit (got $RC)" + +# --- -m caps how many matches participate in the render (pattern excludes +# the top-level orphan so match #1 is unambiguously the ShortFn hit) +OUT=$("$BIN" -p 'target\(\)' -elide -scope go -m 1 "$EL/el.go") +expect_contains "-elide -m 1 renders first match's scope" "func ShortFn" "$OUT" +if [[ "$OUT" != *"LongFn"* ]]; then + report ok "-elide -m 1 excludes later matches" +else + report fail "-elide -m 1 excludes later matches" +fi + +rm -rf "$EL" + +# --------------------------------------------------------------------------- +section "hotspot ranking (-hotspots)" +HS=$(mktemp -d) +cat > "$HS/hot.go" <<'GOEOF' +package main + +func Busy() { + alpha() + beta() +} +GOEOF +cat > "$HS/cold.go" <<'GOEOF' +package main + +func Quiet() { + alpha() +} +GOEOF + +# hot.go matches both patterns (full coverage + a proximity cluster) so it +# must outrank cold.go, which matches only one. +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -hotspots 2 "$HS/hot.go" "$HS/cold.go") +FIRST_LINE=$(printf '%s\n' "$OUT" | head -n1) +expect_contains "-hotspots ranks fuller-coverage file first" "hot.go" "$FIRST_LINE" +expect_lines "-hotspots emits one row per file" 2 "$OUT" +expect_contains "-hotspots JSONL shape" '"type":"hotspot"' "$OUT" + +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -hotspots 1 -llm "$HS/hot.go" "$HS/cold.go") +expect_contains "-hotspots -llm flat line" "hot.go:" "$OUT" +expect_contains "-hotspots -llm shows patterns" "patterns=" "$OUT" + +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -hotspots 1 -elide -scope go "$HS/hot.go" "$HS/cold.go") +expect_contains "-hotspots -elide renders top file's scope" "func Busy" "$OUT" +if [[ "$OUT" != *"Quiet"* ]]; then + report ok "-hotspots 1 -elide excludes lower-ranked file" +else + report fail "-hotspots 1 -elide excludes lower-ranked file" +fi + +# --- validation +OUT=$("$BIN" -p x -hotspots 5 -sample 3 2>&1) ; RC=$? +expect_contains "-hotspots + -sample rejected" "cannot combine with -sample" "$OUT" +[[ "$RC" == "2" ]] && report ok "-hotspots + -sample exit 2" || report fail "-hotspots + -sample exit (got $RC)" +OUT=$("$BIN" -p x -hotspots 5 -f 2>&1) ; RC=$? +expect_contains "-hotspots rejects -f output mode" "JSONL (default), -llm, or -elide" "$OUT" +[[ "$RC" == "2" ]] && report ok "-hotspots -f exit 2" || report fail "-hotspots -f exit (got $RC)" +OUT=$("$BIN" edit -p x -hotspots 5 -content y "$HS/hot.go" 2>&1) ; RC=$? +expect_contains "edit mode rejects -hotspots" "cannot combine with -hotspots" "$OUT" +[[ "$RC" == "2" ]] && report ok "edit mode -hotspots exit 2" || report fail "edit mode -hotspots exit (got $RC)" +OUT=$("$BIN" -p 'zzz_no_such_token_zzz' -hotspots 5 "$HS/hot.go" ; echo "rc=$?") +expect_contains "-hotspots no matches exit 1" "rc=1" "$OUT" + +rm -rf "$HS" + +# --------------------------------------------------------------------------- +section "budget-packed context (-budget)" +BG=$(mktemp -d) +cat > "$BG/hot.go" <<'GOEOF' +package main + +func Busy() { + alpha() + beta() +} +GOEOF +cat > "$BG/cold.go" <<'GOEOF' +package main + +func Quiet() { + alpha() +} +GOEOF + +# Measure each file's standalone full-render size so budget thresholds are +# derived, not hardcoded — robust to any future change in -elide's shape. +HOT_ELIDE=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -elide -scope go "$BG/hot.go") +HOT_LEN=$(printf '%s' "$HOT_ELIDE" | wc -c) +COLD_ELIDE=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -elide -scope go "$BG/cold.go") +COLD_LEN=$(printf '%s' "$COLD_ELIDE" | wc -c) + +# --- huge budget: both files render in full, no footer at all +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -budget $((HOT_LEN + COLD_LEN + 1000)) "$BG/hot.go" "$BG/cold.go") +expect_contains "-budget huge: hot.go in full" "func Busy" "$OUT" +expect_contains "-budget huge: cold.go in full" "func Quiet" "$OUT" +if [[ "$OUT" != *"--- budget:"* ]]; then + report ok "-budget huge: no footer when nothing degraded" +else + report fail "-budget huge: no footer when nothing degraded" +fi + +# --- mid budget: only room for the top-ranked file (hot.go: full coverage + +# proximity cluster outranks cold.go), cold.go can't fit in any form +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -budget $((HOT_LEN + 3)) "$BG/hot.go" "$BG/cold.go") +expect_contains "-budget mid: top file rendered in full" "func Busy" "$OUT" +if [[ "$OUT" != *"Quiet"* ]]; then + report ok "-budget mid: lower-ranked file not shown in any form" +else + report fail "-budget mid: lower-ranked file not shown in any form" +fi +expect_contains "-budget mid: footer counts one file in full" "1 file(s) in full" "$OUT" +expect_contains "-budget mid: footer names the dropped file" "cold.go" "$OUT" + +# --- a budget below the full render but above the compact line degrades to +# the one-line compact summary instead of the source text. A single tiny +# function's full render can be as small as (or smaller than) its own +# compact line, especially under a long mktemp path, so use a file with +# several matched functions — its multi-block full render is reliably +# bigger than one compact line regardless of path length — and probe a +# range rather than assume a fixed threshold. +cat > "$BG/multi.go" <<'GOEOF' +package main + +func Busy1() { + alpha() + beta() +} + +func Busy2() { + alpha() + beta() +} + +func Busy3() { + alpha() + beta() +} + +func Busy4() { + alpha() + beta() +} +GOEOF +MULTI_ELIDE=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -elide -scope go "$BG/multi.go") +MULTI_LEN=$(printf '%s' "$MULTI_ELIDE" | wc -c) +COMPACT_OUT="" +for b in $(seq 20 10 "$MULTI_LEN"); do + PROBE=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -budget "$b" "$BG/multi.go") + if [[ "$PROBE" == *"(compact"* ]]; then COMPACT_OUT="$PROBE"; break; fi +done +expect_contains "-budget: compact summary reachable" "(compact — full render didn't fit the budget)" "$COMPACT_OUT" +if [[ "$COMPACT_OUT" != *"func Busy"* ]]; then + report ok "-budget: compact summary omits source text" +else + report fail "-budget: compact summary omits source text" +fi + +# --- essentially zero budget: everything dropped, exit 1 +OUT=$("$BIN" -p 'alpha\(\)' -p 'beta\(\)' -budget 1 "$BG/hot.go" "$BG/cold.go" ; echo "rc=$?") +expect_contains "-budget ~0: both dropped" "2 dropped" "$OUT" +expect_contains "-budget ~0: exit 1" "rc=1" "$OUT" + +# --- validation +OUT=$("$BIN" -p x -budget 1000 -llm 2>&1) ; RC=$? +expect_contains "-budget + explicit output mode rejected" "defines its own output shape" "$OUT" +[[ "$RC" == "2" ]] && report ok "-budget + -llm exit 2" || report fail "-budget + -llm exit (got $RC)" +OUT=$("$BIN" -p x -budget 1000 -sample 5 2>&1) ; RC=$? +expect_contains "-budget + -sample rejected" "cannot combine with -sample/-hotspots" "$OUT" +[[ "$RC" == "2" ]] && report ok "-budget + -sample exit 2" || report fail "-budget + -sample exit (got $RC)" +OUT=$("$BIN" edit -p x -budget 1000 -content y "$BG/hot.go" 2>&1) ; RC=$? +expect_contains "edit mode rejects -budget" "cannot combine with -budget" "$OUT" +[[ "$RC" == "2" ]] && report ok "edit mode -budget exit 2" || report fail "edit mode -budget exit (got $RC)" + +rm -rf "$BG" + +# --------------------------------------------------------------------------- +section "identifier matching (-ident)" +ID=$(mktemp -d) +cat > "$ID/id.go" <<'GOEOF' +package main + +func parseConfig() {} +func ParseConfig() {} +func parse_config() {} +func ConfigParser() {} +func HTTPServerConfig() {} +func utf8Decoder() {} +func unrelated() {} +func MAX_RETRY_COUNT() {} +GOEOF + +# --- casing/separator-agnostic AND-within-group matching +OUT=$("$BIN" -ident 'parse config' -o "$ID/id.go") +expect_lines "-ident: matches all casing variants" 4 "$OUT" +expect_contains "-ident: camelCase" "parseConfig" "$OUT" +expect_contains "-ident: PascalCase" "ParseConfig" "$OUT" +expect_contains "-ident: snake_case" "parse_config" "$OUT" +expect_contains "-ident: reversed word order still matches (AND, not sequence)" "ConfigParser" "$OUT" +if [[ "$OUT" != *"unrelated"* && "$OUT" != *"HTTPServerConfig"* ]]; then + report ok "-ident: non-matching identifiers excluded" +else + report fail "-ident: non-matching identifiers excluded" +fi + +# --- acronym and digit-boundary splitting +OUT=$("$BIN" -ident 'http server' -o "$ID/id.go") +expect_eq "-ident: acronym split (HTTPServer)" "HTTPServerConfig" "$OUT" +OUT=$("$BIN" -ident 'utf8' -o "$ID/id.go") +expect_eq "-ident: digit-spanning term" "utf8Decoder" "$OUT" +OUT=$("$BIN" -ident 'max retry' -o "$ID/id.go") +expect_eq "-ident: underscore-separated all-caps" "MAX_RETRY_COUNT" "$OUT" + +# --- OR across repeated -ident groups +OUT=$("$BIN" -ident 'parse config' -ident 'unrelated' -f "$ID/id.go") +expect_contains "-ident: repeated groups OR together" "id.go" "$OUT" +COUNT=$("$BIN" -ident 'parse config' -ident 'unrelated' -o "$ID/id.go" | wc -l) +[[ "$COUNT" -eq 5 ]] && report ok "-ident: OR group adds its own matches" || report fail "-ident: OR group adds its own matches (got $COUNT)" + +# --- synthetic pattern id: auto ident0/ident1 and explicit -name +OUT=$("$BIN" -ident 'parse config' -format '$PAT_ID' "$ID/id.go") +expect_eq "-ident: auto id is ident0" "ident0" "$(printf '%s\n' "$OUT" | head -n1)" +OUT=$("$BIN" -ident 'parse config' -name myid -format '$PAT_ID' "$ID/id.go") +expect_eq "-ident: -name overrides auto id" "myid" "$(printf '%s\n' "$OUT" | head -n1)" +OUT=$("$BIN" -p 'func' -ident 'parse config' -name pc -format '$PAT_ID' "$ID/id.go") +if [[ "$OUT" == *"p0"* && "$OUT" == *"pc"* ]]; then + report ok "-ident: coexists with -p's independent p numbering" +else + report fail "-ident: coexists with -p's independent p numbering" +fi + +# --- composes with relations and -file-where +cat > "$ID/rel.go" <<'GOEOF' +package main + +func Handler() { + parseConfig() + doWork() +} + +func Other() { + doWork() +} +GOEOF +OUT=$("$BIN" -p 'doWork' -name work -ident 'parse config' -name pc -near work:pc:2 -format '$LINE $PAT_ID' "$ID/rel.go") +expect_eq "-ident: participates in -near" "$(printf '4 pc\n5 work')" "$OUT" +OUT=$("$BIN" -p 'Other' -name has_other -ident 'parse config' -name pc -file-where 'NOT has_other' -f "$ID/rel.go" ; true) +expect_lines "-ident: participates in -file-where" 0 "$OUT" + +# --- validation +OUT=$("$BIN" -ident 'parse config' -extract x "$ID/id.go" 2>&1) ; RC=$? +expect_contains "-extract after -ident rejected" "cannot follow -ident" "$OUT" +[[ "$RC" == "2" ]] && report ok "-extract after -ident exit 2" || report fail "-extract after -ident exit (got $RC)" +OUT=$("$BIN" -ident ' ' "$ID/id.go" 2>&1) ; RC=$? +expect_contains "-ident: empty terms rejected" "at least one term required" "$OUT" +[[ "$RC" == "2" ]] && report ok "-ident empty terms exit 2" || report fail "-ident empty terms exit (got $RC)" +OUT=$("$BIN" edit -ident 'parse config' -content x "$ID/id.go" 2>&1) ; RC=$? +expect_contains "edit mode rejects -ident" "cannot combine with -ident" "$OUT" +[[ "$RC" == "2" ]] && report ok "edit mode -ident exit 2" || report fail "edit mode -ident exit (got $RC)" +OUT=$("$BIN" -ident 'zzz nonexistent' "$ID/id.go" ; echo "rc=$?") +expect_contains "-ident: no matches exit 1" "rc=1" "$OUT" + +rm -rf "$ID" + +# --------------------------------------------------------------------------- +section "-file-where metadata conditions (count/churn/lang)" +FW=$(mktemp -d) +printf 'foo\nfoo\nfoo\n' > "$FW/hot.txt" +printf 'foo\n' > "$FW/cold.txt" +printf 'package main\n' > "$FW/a.go" + +# --- count(pat) +OUT=$("$BIN" -p foo -name f -file-where 'count(f) >= 2' -f "$FW/hot.txt" "$FW/cold.txt") +expect_eq "-file-where count(): threshold met" "$FW/hot.txt" "$OUT" +OUT=$("$BIN" -p foo -name f -file-where 'count(f) >= 5' -f "$FW/hot.txt" "$FW/cold.txt" ; true) +expect_lines "-file-where count(): threshold not met" 0 "$OUT" + +# --- lang +OUT=$("$BIN" -p 'package' -file-where 'lang == go' -f "$FW/a.go" "$FW/hot.txt") +expect_eq "-file-where lang ==" "$FW/a.go" "$OUT" +OUT=$("$BIN" -p 'foo' -file-where 'lang != go' -f "$FW/a.go" "$FW/hot.txt") +expect_eq "-file-where lang !=" "$FW/hot.txt" "$OUT" + +# --- compound +OUT=$("$BIN" -p foo -name f -file-where 'count(f) >= 2 AND lang != go' -f "$FW/hot.txt" "$FW/cold.txt" "$FW/a.go") +expect_eq "-file-where compound count+lang" "$FW/hot.txt" "$OUT" + +# --- churn(): fresh repo, commit hot.txt 3x and cold.txt once, all "now" — +# well within any day-window regardless of when the test runs. +GITREPO=$(mktemp -d) +( + cd "$GITREPO" + git init -q + git config user.email test@example.com + git config user.name Test + printf 'foo\n' > hot.txt + printf 'foo\n' > cold.txt + git add -A && git commit -q -m 1 + printf 'foo\nfoo\n' > hot.txt + git add -A && git commit -q -m 2 + printf 'foo\nfoo\nfoo\n' > hot.txt + git add -A && git commit -q -m 3 +) +OUT=$(cd "$GITREPO" && "$BIN" -p foo -file-where 'churn(1) > 1' -f hot.txt cold.txt) +expect_eq "-file-where churn(): more-committed file selected" "hot.txt" "$OUT" +OUT=$(cd "$GITREPO" && "$BIN" -p foo -file-where 'churn(1) > 5' -f hot.txt cold.txt ; true) +expect_lines "-file-where churn(): threshold not met" 0 "$OUT" + +# --- edit mode: same predicates work for targeting +OUT=$(cd "$GITREPO" && "$BIN" edit -p foo -file-where 'churn(1) > 1' -content bar -j hot.txt) +expect_contains "edit mode: churn() predicate targets correctly" '"file":"hot.txt"' "$OUT" +OUT=$(cd "$GITREPO" && "$BIN" edit -p foo -file-where 'churn(1) > 1' -content bar -j cold.txt 2>&1) ; RC=$? +[[ "$RC" == "1" ]] && report ok "edit mode: churn() predicate excludes non-matching file" || report fail "edit mode: churn() exclude (got $RC)" + +rm -rf "$GITREPO" + +# --- validation +OUT=$("$BIN" -p x -file-where 'churn(abc) > 2' 2>&1) ; RC=$? +expect_contains "-file-where: non-numeric churn arg rejected" "positive integer day-window" "$OUT" +[[ "$RC" == "2" ]] && report ok "-file-where churn(abc) exit 2" || report fail "-file-where churn(abc) exit (got $RC)" +OUT=$("$BIN" -p x -file-where 'lang > go' 2>&1) ; RC=$? +expect_contains "-file-where: lang ordering operator rejected" "only supports == and !=" "$OUT" +[[ "$RC" == "2" ]] && report ok "-file-where lang> exit 2" || report fail "-file-where lang> exit (got $RC)" +OUT=$("$BIN" -p x -file-where 'foo(bar) > 2' 2>&1) ; RC=$? +expect_contains "-file-where: unknown condition function rejected" "unknown condition" "$OUT" +[[ "$RC" == "2" ]] && report ok "-file-where foo(bar) exit 2" || report fail "-file-where foo(bar) exit (got $RC)" +OUT=$("$BIN" -p x -name f -file-where 'count(zzz) > 2' 2>&1) ; RC=$? +expect_contains "-file-where: unknown pattern in count() rejected" "unknown pattern" "$OUT" +[[ "$RC" == "2" ]] && report ok "-file-where count(zzz) exit 2" || report fail "-file-where count(zzz) exit (got $RC)" + +rm -rf "$FW" + +# --------------------------------------------------------------------------- +section "sorting file-grouped output (-order-by)" +OB=$(mktemp -d) +printf 'foo\nfoo\nfoo\n' > "$OB/c_three.txt" +printf 'foo\n' > "$OB/a_one.txt" +printf 'foo\nfoo\n' > "$OB/b_two.txt" + +OUT=$("$BIN" -p foo -c -order-by path "$OB/c_three.txt" "$OB/a_one.txt" "$OB/b_two.txt") +expect_eq "-order-by path" "$(printf '%s:1\n%s:2\n%s:3' "$OB/a_one.txt" "$OB/b_two.txt" "$OB/c_three.txt")" "$OUT" + +OUT=$("$BIN" -p foo -c -order-by count "$OB/c_three.txt" "$OB/a_one.txt" "$OB/b_two.txt") +FIRST_LINE=$(printf '%s\n' "$OUT" | head -n1) +expect_eq "-order-by count: highest first" "$OB/c_three.txt:3" "$FIRST_LINE" + +OUT=$("$BIN" -p foo -f -order-by score "$OB/c_three.txt" "$OB/a_one.txt" "$OB/b_two.txt") +expect_eq "-order-by score: densest file first" "$OB/c_three.txt" "$(printf '%s\n' "$OUT" | head -n1)" + +# --- -c lists every file (including :0) even when ordered +printf 'nothing\n' > "$OB/d_zero.txt" +OUT=$("$BIN" -p foo -c -order-by path "$OB/d_zero.txt" "$OB/a_one.txt" ; echo "rc=$?") +expect_contains "-order-by: -c still lists 0-match files" "d_zero.txt:0" "$OUT" +expect_contains "-order-by: exit code reflects matches, not rows" "rc=0" "$OUT" +OUT=$("$BIN" -p foo -c -order-by path "$OB/d_zero.txt" ; echo "rc=$?") +expect_contains "-order-by: exit 1 when the only row has 0 matches" "rc=1" "$OUT" + +# --- validation +OUT=$("$BIN" -p foo -order-by path 2>&1) ; RC=$? +expect_contains "-order-by requires -f/-c" "requires -f or -c output" "$OUT" +[[ "$RC" == "2" ]] && report ok "-order-by without -f/-c exit 2" || report fail "-order-by without -f/-c exit (got $RC)" +OUT=$("$BIN" -p foo -order-by bogus 2>&1) ; RC=$? +expect_contains "-order-by: unknown field rejected" "unknown field" "$OUT" +[[ "$RC" == "2" ]] && report ok "-order-by bogus field exit 2" || report fail "-order-by bogus field exit (got $RC)" +OUT=$("$BIN" edit -p foo -order-by path -content bar "$OB/a_one.txt" 2>&1) ; RC=$? +expect_contains "edit mode rejects -order-by" "cannot combine with -order-by" "$OUT" +[[ "$RC" == "2" ]] && report ok "edit mode -order-by exit 2" || report fail "edit mode -order-by exit (got $RC)" + +rm -rf "$OB" + +# --------------------------------------------------------------------------- +section "cross-invocation dedup (-seen)" +SN=$(mktemp -d) +cat > "$SN/f.go" <<'GOEOF' +package main + +func Alpha() { + target() +} + +func Beta() { + target() +} +GOEOF + +# --- first run: full render, state file written in the documented format +OUT=$("$BIN" -p target -elide -scope go -seen "$SN/state.txt" "$SN/f.go") +expect_contains "-seen first run: full render" "func Alpha() {" "$OUT" +expect_contains "-seen first run: full render (2nd fn)" "func Beta() {" "$OUT" +[[ -f "$SN/state.txt" ]] && report ok "-seen: state file created" || report fail "-seen: state file created" +STATE_LINES=$(wc -l < "$SN/state.txt") +[[ "$STATE_LINES" -eq 2 ]] && report ok "-seen: one state line per scope" || report fail "-seen: one state line per scope (got $STATE_LINES)" + +# --- second run, nothing changed: both collapse +OUT=$("$BIN" -p target -elide -scope go -seen "$SN/state.txt" "$SN/f.go") +expect_contains "-seen: unchanged Alpha collapses" "func Alpha (unchanged, already shown)" "$OUT" +expect_contains "-seen: unchanged Beta collapses" "func Beta (unchanged, already shown)" "$OUT" +if [[ "$OUT" != *"func Alpha() {"* && "$OUT" != *"func Beta() {"* ]]; then + report ok "-seen: collapsed chunks omit source text" +else + report fail "-seen: collapsed chunks omit source text" +fi + +# --- change one function's body (same line count/range): hash mismatch +# still forces a full re-render even though the range didn't move. +cat > "$SN/f.go" <<'GOEOF' +package main + +func Alpha() { + target2() +} + +func Beta() { + target() +} +GOEOF +OUT=$("$BIN" -p 'target2|target' -elide -scope go -seen "$SN/state.txt" "$SN/f.go") +expect_contains "-seen: content change at same range re-renders" "func Alpha() {" "$OUT" +expect_contains "-seen: untouched function still collapses" "func Beta (unchanged, already shown)" "$OUT" + +# --- -budget: a render measured to decide fit, then degraded/dropped, must +# NOT be recorded as seen — a later run with enough budget renders in full. +rm -f "$SN/bstate.txt" +"$BIN" -p target -budget 10 -seen "$SN/bstate.txt" "$SN/f.go" > /dev/null +if [[ ! -s "$SN/bstate.txt" ]]; then + report ok "-seen + -budget: dropped render isn't marked seen" +else + report fail "-seen + -budget: dropped render isn't marked seen" +fi +OUT=$("$BIN" -p target -elide -scope go -seen "$SN/bstate.txt" "$SN/f.go") +expect_contains "-seen + -budget: later full run still shows real content" "func Beta() {" "$OUT" + +# --- missing state file on first use: no error, just an empty store +rm -f "$SN/fresh.txt" +OUT=$("$BIN" -p target -elide -scope go -seen "$SN/fresh.txt" "$SN/f.go" 2>&1) ; RC=$? +[[ "$RC" == "0" ]] && report ok "-seen: missing state file is not an error" || report fail "-seen: missing state file (got rc=$RC)" + +# --- validation +OUT=$("$BIN" -p target -seen "$SN/state.txt" "$SN/f.go" 2>&1) ; RC=$? +expect_contains "-seen requires -elide/-budget" "requires -elide or -budget" "$OUT" +[[ "$RC" == "2" ]] && report ok "-seen without -elide/-budget exit 2" || report fail "-seen without -elide/-budget exit (got $RC)" +OUT=$("$BIN" edit -p target -seen "$SN/state.txt" -content x "$SN/f.go" 2>&1) ; RC=$? +expect_contains "edit mode rejects -seen" "cannot combine with -seen" "$OUT" +[[ "$RC" == "2" ]] && report ok "edit mode -seen exit 2" || report fail "edit mode -seen exit (got $RC)" + +rm -rf "$SN" + # --------------------------------------------------------------------------- section "summary" TOTAL=$((PASS + FAIL))