From 020fe3d8ce5cc9fbad6e30973014e981209bcb89 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 03:46:58 -0400 Subject: [PATCH 1/8] test(retrieval): add source-native comparison path --- .github/workflows/ci.yml | 3 + .gitignore | 1 + README.md | 22 +- docs/README.md | 2 + docs/guides/consumer-handoff.md | 19 +- docs/guides/source-native-retrieval.md | 166 ++++++ examples/README.md | 24 +- examples/javascript/rag-quickstart.mjs | 67 ++- examples/python/rag_quickstart.py | 83 +-- examples/python/source_native_search.py | 64 ++ runtimes/python/README.md | 21 +- .../src/vyral_runtime/_local_experience.py | 96 ++- .../vyral_runtime/integrations/__init__.py | 20 + .../src/vyral_runtime/integrations/ripgrep.py | 553 ++++++++++++++++++ .../python/tests/test_local_experience.py | 27 + .../python/tests/test_ripgrep_integration.py | 149 +++++ scripts/export-public-tree.py | 2 + scripts/verify-lexical-http-quickstarts.sh | 76 +++ scripts/verify-python-runtime.sh | 4 + 19 files changed, 1259 insertions(+), 140 deletions(-) create mode 100644 docs/guides/source-native-retrieval.md create mode 100644 examples/python/source_native_search.py create mode 100644 runtimes/python/src/vyral_runtime/integrations/ripgrep.py create mode 100644 runtimes/python/tests/test_ripgrep_integration.py create mode 100755 scripts/verify-lexical-http-quickstarts.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0490d0..e5763cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,9 @@ jobs: - name: Run deterministic tests run: dotnet test Vyral.sln --configuration Release --no-build --logger 'console;verbosity=minimal' + - name: Verify zero-embedding HTTP quickstarts + run: scripts/verify-lexical-http-quickstarts.sh + - name: Prove core contract TFMs run: | dotnet build src/Vyral.Abstractions/Vyral.Abstractions.csproj --configuration Release --framework net8.0 --no-restore diff --git a/.gitignore b/.gitignore index 4a9b72f..10a12e1 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ docs/contributing/* docs/guides/* !docs/guides/consumer-handoff.md !docs/guides/extropic-execution.md +!docs/guides/source-native-retrieval.md !docs/maintainers/ docs/maintainers/* !docs/maintainers/releasing.md diff --git a/README.md b/README.md index 71dd12e..ca51070 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Vyral is a local-first contract layer and runtime for applications that need **records and objects**, **lexical/vector/hybrid retrieval and RAG**, +**source-native retrieval experiments**, **durable jobs**, and **provider-backed AI runs** without binding application code to a single cloud, vector store, orchestration engine, or model vendor. @@ -33,9 +34,9 @@ From a source checkout, run: That is the complete local path. It needs only Python 3.10 or newer—no package installation, .NET SDK, container runtime, cloud account, external service, or -model download. It runs retrieval with citations, admits durable work, closes -and reopens the runtime, and completes the preserved run. State remains visible -under `./.vyral/quickstart`. +model download. It runs zero-embedding lexical retrieval with citations, admits +durable work, closes and reopens the runtime, and completes the preserved run. +State remains visible under `./.vyral/quickstart`. Create a small editable application when you are ready to build: @@ -54,6 +55,7 @@ output, reset, and server instructions are in the | You need… | Start here | | --- | --- | | Local RAG / retrieval substrate | [Quick start](#quick-start) | +| Source-native, lexical, vector, or hybrid choice | [Retrieval guide](docs/guides/source-native-retrieval.md) | | Embedded Python runtime, no server or .NET | [Python runtime](runtimes/python) | | HTTP clients (Python / JavaScript) | [Client packages](#client-packages) | | Go external-worker client | [Go client](clients/go) | @@ -72,7 +74,7 @@ Vyral keeps the capability model application-owned: - **Records** — ids, partition keys, metadata, content, sources, vectors, and etags - **Objects** — large immutable payloads and manifests outside the record index -- **Retrieval** — filters, lexical, vector, hybrid, profiles, and evaluation +- **Retrieval** — filters, lexical, vector, hybrid, profiles, evaluation, and experimental source-native search - **RAG** — deterministic chunk plans, dry-run/commit, manifests, context, and citations - **Provider runs** — one envelope for local, CLI, and ONNX-backed AI capabilities - **Execution** — durable runs and plugins with local and provider-backed adapters @@ -81,6 +83,13 @@ Vyral keeps the capability model application-owned: Use Vyral when you want realistic local development and a rehearsed path to re-home storage, retrieval, or execution without rewriting the product surface. +For local, current, structured sources that an application can safely access, +searching the source directly is often the best first retrieval path: exact, +immediately fresh, and free of index synchronization. Use indexed lexical +retrieval when you need a governed record boundary, vectors when labeled tests +show semantic recall is worth the added index, and hybrid or reranking only +when the same evaluation demonstrates an improvement. + ## Status Vyral is an Apache-2.0, pre-1.0 project in active development. The local paths @@ -93,6 +102,7 @@ compatibility policy. | .NET local server, SQLite, retrieval, and RAG | Primary supported implementation | | Python and JavaScript HTTP clients | Implemented and tested against the public SDK catalog | | Python-native embedded runtime and REST/MCP host | Prototype; implemented and clean-artifact qualified on Python 3.10–3.12 across Linux, macOS, and Windows; independent security review remains | +| Source-native ripgrep integration | Experimental; bounded, read-only Python integration outside the stable wire/MCP contract | | Stateless MCP `2026-07-28` endpoint | Preview; opt-in and REST-backed | | Local SQLite and Azure Durable execution | Preview; locally conformant | | AWS, Google, Temporal, pgvector, and Cloudflare adapters | Preview or prototype; live qualification is opt-in | @@ -111,8 +121,8 @@ scripts/start-local-server.sh ``` It listens on `http://127.0.0.1:5220` and writes local state beneath `.vyral/`. -Run either HTTP client example to create a collection, ingest three documents, -retrieve context, and print citations: +Run either HTTP client example to create a collection, store three documents, +retrieve lexical context without embeddings, and print citations: ```bash python3 examples/python/rag_quickstart.py diff --git a/docs/README.md b/docs/README.md index 16e7b26..8906cca 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,8 @@ here so each document has a clear audience and owner. portable guarantees and adapter-specific boundaries. - [Extropic execution](guides/extropic-execution.md) — experimental, fail-closed dispatch of registered Python workloads to Extropic compute. +- [Source-native and indexed retrieval](guides/source-native-retrieval.md) — + choose and evaluate direct, lexical, vector, hybrid, and reranked retrieval. - [Stability and versioning](reference/stability.md) — maturity labels, version lines, and compatibility promises. diff --git a/docs/guides/consumer-handoff.md b/docs/guides/consumer-handoff.md index 8994e12..12d7f7c 100644 --- a/docs/guides/consumer-handoff.md +++ b/docs/guides/consumer-handoff.md @@ -160,12 +160,25 @@ If `VYRAL_API_KEY` is set, pass the same key to the client constructor. Failed c ## Retrieval Defaults -For evidence-sensitive or exact-reference retrieval, start with lexical retrieval or a retrieval profile that keeps lexical scoring dominant. Use vectors for semantic discovery, recall expansion, or exploratory search unless your own evaluation shows vector-only retrieval is strong enough for the corpus. +For local or structured text that the application can safely read directly, +test source-native search first. It provides exact matching and current source +state without an index. Use Vyral lexical retrieval when the corpus needs a +governed record boundary, filters, stable snapshots, or provider-portable +retrieval semantics. Use vectors for semantic discovery, recall expansion, +remote or unstructured corpora, and non-greppable modalities when your own +evaluation supports the added index. Treat hybrid and reranking as evaluated +policies rather than defaults. See the +[source-native and indexed retrieval guide](source-native-retrieval.md) for the +bounded ripgrep experiment and a five-variant comparison recipe. Recommended first path: -1. Create a RAG collection with `create_rag_collection(...)` / `createRagCollection(...)`. -2. Ingest with `plan_rag_text_ingestion(...)` then `commit_rag_text_ingestion(...)`, or use `ingest_rag_text(...)` for simple loops. +1. Create a record-only collection for lexical retrieval, or a RAG collection + with `create_rag_collection(...)` / `createRagCollection(...)` when vectors + are justified. +2. Store source-backed records directly for zero-embedding lexical RAG. For + vector ingestion, use `plan_rag_text_ingestion(...)` then + `commit_rag_text_ingestion(...)`, or `ingest_rag_text(...)` for simple loops. 3. Build context with `build_rag_context(...)`. 4. Evaluate with `/retrieval/evaluate` or `evaluate_retrieval(...)` before changing retrieval profiles. 5. Snapshot with collection export before destructive corpus changes. diff --git a/docs/guides/source-native-retrieval.md b/docs/guides/source-native-retrieval.md new file mode 100644 index 0000000..49f7889 --- /dev/null +++ b/docs/guides/source-native-retrieval.md @@ -0,0 +1,166 @@ +# Source-native and indexed retrieval + +Retrieval should follow the corpus rather than forcing every source through an +embedding pipeline. For local, current, structured text that an application is +already authorized to read, direct source search is often the strongest first +path. It preserves exact identifiers, reflects edits immediately, and avoids an +index lifecycle. + +Choose the least complex path that meets a measured need: + +| Corpus need | First path to test | +| --- | --- | +| Code, Markdown, configuration, or other greppable text | Source-native fixed-string search | +| Governed records, filters, tenant boundaries, or stable snapshots | Vyral lexical retrieval | +| Paraphrases, cross-lingual queries, or weak keyword overlap | Semantic vector retrieval | +| Exact precision plus demonstrated semantic recall | Evaluated hybrid retrieval | +| Top-rank quality improves enough to justify another model call | Evaluated reranking | +| PDFs, images, audio, or other non-greppable sources | Extract or index through an appropriate adapter | + +Vector retrieval remains valuable for large unstructured or remote corpora, +semantic discovery, aggregation, access-controlled retrieval services, and +modalities that cannot be searched as text. Hybrid retrieval is a policy to +earn with corpus evidence, not an automatic upgrade over a simpler method. + +## Experimental ripgrep integration + +The Python runtime includes an experimental, read-only ripgrep integration. It +is not part of the stable Vyral wire contract and is not automatically exposed +through REST or MCP. An application owns the authorized root and may place the +result behind its own Vyral handler after applying its authorization policy. + +```python +from pathlib import Path + +from vyral_runtime.integrations.ripgrep import ( + RipgrepAdapterOptions, + RipgrepSearchAdapter, + RipgrepSearchRequest, +) + +search = RipgrepSearchAdapter( + Path("./knowledge"), + RipgrepAdapterOptions( + include_globs=("*.py", "*.md"), + exclude_globs=("generated/**",), + max_results=40, + timeout_seconds=3, + ), +) + +result = search.search( + RipgrepSearchRequest( + "durable receipt", + limit=10, + case_sensitive=False, + ) +) + +for match in result.matches: + print(match.source_uri, match.source_revision, match.line_text) +``` + +The adapter: + +- accepts one fixed root and static include/exclude globs at construction; +- passes the query over standard input, never through a shell or process + argument; +- disables ripgrep configuration, regular expressions, hidden-file search, + and symbolic-link following; +- applies non-overridable credential-file exclusions and filters sensitive + filenames from parsed results; +- bounds query length, result count, source size, line size, process output, + and execution time; and +- emits root-relative line citations plus a SHA-256 source revision without + disclosing the absolute root. + +Filename filtering is not data-loss prevention. Point the adapter only at a +root whose contents the caller is authorized to retrieve, keep secrets outside +that root, and authorize any handler or MCP tool before search begins. Treat +returned lines as untrusted source content when they enter an agent context. + +The executable is resolved once when the adapter starts and must identify +itself as ripgrep. Install `rg` separately and keep its version in evaluation +receipts when results need to be reproduced. + +## Corpus comparison recipe + +Use a labeled fixture that resembles the real corpus. Keep the source revision, +query set, expected records or files, hard negatives, and relevance grades in +version control. Include exact identifiers, paraphrases, recently edited +content, ambiguous terms, and queries that should return nothing. + +Run five variants against the same fixture: + +1. Source-native `rg` through `RipgrepSearchAdapter`. +2. Vyral lexical retrieval over source-backed records. +3. Vyral vector retrieval with the intended production embedding provider. +4. Vyral hybrid retrieval with declared fusion weights. +5. The best prior candidate set with the intended reranker. + +Vyral can compare the four indexed variants in one +`POST /retrieval/evaluate/compare` request. The essential variant shape is: + +```json +{ + "variants": [ + { + "id": "lexical", + "searchMode": "lexical", + "lexical": { "fields": ["/content/text", "/metadata/title"] } + }, + { + "id": "vector", + "searchMode": "vector", + "embedding": { "field": "contentEmbedding", "purpose": "query" } + }, + { + "id": "hybrid", + "searchMode": "hybrid", + "embedding": { "field": "contentEmbedding", "purpose": "query" }, + "lexical": { "fields": ["/content/text", "/metadata/title"] }, + "hybrid": { + "fusion": "rrf", + "lexicalWeight": 0.65, + "vectorWeight": 0.35 + } + }, + { + "id": "reranked-lexical", + "searchMode": "lexical", + "lexical": { "fields": ["/content/text", "/metadata/title"] }, + "rerank": { "enabled": true, "candidateLimit": 20 } + } + ] +} +``` + +Each `cases` entry supplies a normal retrieval request, expected matches, hard +negatives, and `k`. Normalize each source-native result to the same labeled +file or source id so it can be scored beside the indexed variants. + +Record at least these measures: + +| Measure | What to retain | +| --- | --- | +| Accuracy | hit rate, recall@k, precision@k, MRR, nDCG, and hard-negative rate | +| Latency | warm and cold p50/p95, including agent/tool handoff where applicable | +| Context | returned characters and tokens measured with the actual consuming tokenizer | +| Freshness | time from a committed source edit until that edit can be retrieved | +| Cost | embedding, rerank, storage, indexing, request, and agent-token units | +| Reliability | timeouts, fallbacks, empty results, malformed citations, and provider failures | + +For a freshness check, modify one labeled source after the initial run. Search +the source immediately, then measure each index's documented update path until +the new text is retrievable. Do not compare direct search's zero index lag with +an index that was never refreshed. + +The bundled `local-token-hash` embedding provider is appropriate for vector +mechanics and deterministic tests, but it is not semantic. Label any comparison +that uses it as a mechanics baseline. A semantic-quality conclusion requires +the embedding and rerank providers intended for the target deployment. + +Promote a more complex retrieval path only when the gain is repeatable and +worth its freshness, latency, cost, and operational tradeoffs. Retain the +fixture and comparison output so later adapter or model changes can be judged +against the same boundary. diff --git a/examples/README.md b/examples/README.md index 8ab84b9..a6b803f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,7 +28,12 @@ Run the JavaScript quickstart: node examples/javascript/rag-quickstart.mjs ``` -Both examples create a collection, call `/collections/{collection}/rag/ingest-text` with the server's configured embedding provider, call `/rag/context`, and print returned citation IDs plus the deterministic `contextText` block. Set `VYRAL_URL` to point at a non-default server URL. Set `VYRAL_COLLECTION` to override the sample collection name. +Both examples create a collection without a vector policy, store three small +source-backed records, run lexical RAG with no embeddings, and print citation IDs +plus the deterministic `contextText` block. Set `VYRAL_URL` to point at a +non-default server URL. Set `VYRAL_COLLECTION` to override the sample +collection name. `scripts/verify-lexical-http-quickstarts.sh` runs both examples +against one isolated Release server as a CI regression gate. Run the broader consumer workflows when you want to exercise lexical RAG, vector RAG, lexical plus rerank, GraphRAG expansion/evaluation, `ai.extract`, provider model listing, and quota discovery: @@ -39,6 +44,23 @@ node examples/javascript/consumer-workflows.mjs These scripts create isolated example collections by default. Override `VYRAL_COLLECTION`, `VYRAL_GRAPH_COLLECTION`, `VYRAL_RECIPE_AI_PROVIDER`, and `VYRAL_RECIPE_RERANK_PROVIDER` to point them at different local test surfaces. +## Source-native retrieval + +For a local code or Markdown corpus, test the bounded experimental ripgrep +integration before creating an index: + +```bash +python3 examples/python/source_native_search.py \ + "durable receipt" \ + --root . +``` + +The example searches only allowlisted `*.py` and `*.md` paths by default and +prints root-relative line citations with source revisions. It does not expose a +REST or MCP tool. See the +[source-native retrieval guide](../docs/guides/source-native-retrieval.md) for +the safety boundary and comparison recipe. + ## Execution Runtime Sample The execution runtime sample is a .NET console app rather than an HTTP client script. It runs diff --git a/examples/javascript/rag-quickstart.mjs b/examples/javascript/rag-quickstart.mjs index 2ffd398..a905dc1 100644 --- a/examples/javascript/rag-quickstart.mjs +++ b/examples/javascript/rag-quickstart.mjs @@ -2,18 +2,17 @@ import { VyralClient } from "../../clients/javascript/src/index.js"; const collection = process.env.VYRAL_COLLECTION ?? "quickstart-chunks-js"; const partitionKey = "tenant:quickstart"; -const vectorField = "contentEmbedding"; const client = new VyralClient(process.env.VYRAL_URL ?? "http://localhost:5220"); -const health = await client.health(); -const embedding = health.embedding; const deleteRun = await client.deleteCollection(collection, { idempotencyKey: `quickstart:${collection}:delete` }); await client.waitExecutionRun(deleteRun.id); -const createRun = await client.createRagCollection(collection, { - dimensions: embedding.dimensions, - embeddingField: vectorField, - indexedMetadata: ["/metadata/status", "/metadata/topic"], +const createRun = await client.createCollection({ + name: collection, + partitionKeyPath: "/partitionKey", + vectorPolicies: [], + indexedMetadata: ["/metadata/status", "/metadata/topic"] +}, { idempotencyKey: `quickstart:${collection}:create` }); await client.waitExecutionRun(createRun.id); @@ -36,46 +35,46 @@ const documents = [ } ]; -const ingestion = await client.ingestRagTexts(collection, documents.map((document) => ({ - documentId: document.id, - partitionKey, - text: document.text, - embeddingField: vectorField, - sourceUri: `memory://quickstart/${document.id}`, - sourceKind: "example", - metadata: { - status: "active", - topic: document.topic - } - })), { idempotencyKey: `quickstart:${collection}:ingest` }); -await client.waitRagIngestionJob(ingestion.id); +await client.upsertRecords(collection, documents.map((document) => ({ + id: document.id, + partitionKey, + type: "rag.chunk", + content: { text: document.text }, + metadata: { + status: "active", + topic: document.topic + }, + sources: [{ + id: document.id, + kind: "example", + uri: `memory://quickstart/${document.id}`, + label: document.id + }] +})), { idempotencyKey: `quickstart:${collection}:ingest` }); -const context = await client.buildRagContext({ +const retrieval = { query: "Retention holds keep protected records from deletion until the hold is released.", collections: [collection], partitionKeys: [partitionKey], - embeddingField: vectorField, + searchMode: "lexical", + lexical: { fields: ["/content/text"] }, limit: 2, + includeTrace: true +}; +const contextRequest = { + retrieval, maxChars: 2000, maxCharsPerChunk: 800, includeContextText: true, includeTrace: true -}); +}; +const context = await client.buildRagContext(contextRequest); const prompt = await client.buildRagPrompt({ - context: { - query: "Retention holds keep protected records from deletion until the hold is released.", - collections: [collection], - partitionKeys: [partitionKey], - embeddingField: vectorField, - limit: 2, - maxChars: 2000, - maxCharsPerChunk: 800, - includeTrace: true - }, + context: contextRequest, template: { failOnEmptyContext: true } }); -console.log(`provider=${embedding.provider} model=${embedding.modelId} dimensions=${embedding.dimensions}`); +console.log("retrieval=lexical embeddings=unused"); for (const chunk of context.chunks) { const citations = chunk.citationIds?.join(", ") || "none"; console.log(`${chunk.rank}. ${chunk.id} score=${chunk.score.toFixed(4)} citations=${citations} text=${chunk.text}`); diff --git a/examples/python/rag_quickstart.py b/examples/python/rag_quickstart.py index 22cbd8d..c0ef7d5 100644 --- a/examples/python/rag_quickstart.py +++ b/examples/python/rag_quickstart.py @@ -11,23 +11,22 @@ COLLECTION = os.environ.get("VYRAL_COLLECTION", "quickstart-chunks-python") PARTITION_KEY = "tenant:quickstart" -VECTOR_FIELD = "contentEmbedding" def main() -> None: client = VyralClient(os.environ.get("VYRAL_URL", "http://localhost:5220")) - health = client.health() - embedding = health["embedding"] delete_run = client.delete_collection( COLLECTION, idempotency_key=f"quickstart:{COLLECTION}:delete" ) client.wait_execution_run(delete_run["id"]) - create_run = client.create_rag_collection( - COLLECTION, - dimensions=embedding["dimensions"], - embedding_field=VECTOR_FIELD, - indexed_metadata=["/metadata/status", "/metadata/topic"], + create_run = client.create_collection( + { + "name": COLLECTION, + "partitionKeyPath": "/partitionKey", + "vectorPolicies": [], + "indexedMetadata": ["/metadata/status", "/metadata/topic"], + }, idempotency_key=f"quickstart:{COLLECTION}:create", ) client.wait_execution_run(create_run["id"]) @@ -50,49 +49,55 @@ def main() -> None: }, ] - ingestion = client.ingest_rag_texts(COLLECTION, [ - { - "documentId": document["id"], - "partitionKey": PARTITION_KEY, - "text": document["text"], - "embeddingField": VECTOR_FIELD, - "sourceUri": f"memory://quickstart/{document['id']}", - "sourceKind": "example", - "metadata": { - "status": "active", - "topic": document["topic"], - }, - } - for document in documents - ], idempotency_key=f"quickstart:{COLLECTION}:ingest") - client.wait_rag_ingestion_job(ingestion["id"]) - - context = client.build_rag_context({ + client.upsert_records( + COLLECTION, + [ + { + "id": document["id"], + "partitionKey": PARTITION_KEY, + "type": "rag.chunk", + "content": {"text": document["text"]}, + "metadata": { + "status": "active", + "topic": document["topic"], + }, + "sources": [ + { + "id": document["id"], + "kind": "example", + "uri": f"memory://quickstart/{document['id']}", + "label": document["id"], + } + ], + } + for document in documents + ], + idempotency_key=f"quickstart:{COLLECTION}:ingest", + ) + + retrieval = { "query": "Retention holds keep protected records from deletion until the hold is released.", "collections": [COLLECTION], "partitionKeys": [PARTITION_KEY], - "embeddingField": VECTOR_FIELD, + "searchMode": "lexical", + "lexical": {"fields": ["/content/text"]}, "limit": 2, + "includeTrace": True, + } + context_request = { + "retrieval": retrieval, "maxChars": 2000, "maxCharsPerChunk": 800, "includeContextText": True, "includeTrace": True, - }) + } + context = client.build_rag_context(context_request) prompt = client.build_rag_prompt({ - "context": { - "query": "Retention holds keep protected records from deletion until the hold is released.", - "collections": [COLLECTION], - "partitionKeys": [PARTITION_KEY], - "embeddingField": VECTOR_FIELD, - "limit": 2, - "maxChars": 2000, - "maxCharsPerChunk": 800, - "includeTrace": True, - }, + "context": context_request, "template": {"failOnEmptyContext": True}, }) - print(f"provider={embedding['provider']} model={embedding['modelId']} dimensions={embedding['dimensions']}") + print("retrieval=lexical embeddings=unused") for chunk in context["chunks"]: citations = ", ".join(chunk.get("citationIds", [])) or "none" print(f"{chunk['rank']}. {chunk['id']} score={chunk['score']:.4f} citations={citations} text={chunk['text']}") diff --git a/examples/python/source_native_search.py b/examples/python/source_native_search.py new file mode 100644 index 0000000..4838b3d --- /dev/null +++ b/examples/python/source_native_search.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +sys.path.insert( + 0, + str(Path(__file__).resolve().parents[2] / "runtimes/python/src"), +) + +from vyral_runtime.integrations.ripgrep import ( # noqa: E402 + RipgrepAdapterOptions, + RipgrepSearchAdapter, + RipgrepSearchRequest, +) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run bounded, source-native retrieval through ripgrep." + ) + parser.add_argument("query") + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument( + "--include", + action="append", + dest="includes", + default=None, + help="Allowlisted glob; repeat as needed (defaults to Python and Markdown).", + ) + parser.add_argument("--limit", type=int, default=10) + parser.add_argument("--case-sensitive", action="store_true") + parser.add_argument("--json", action="store_true") + arguments = parser.parse_args() + + adapter = RipgrepSearchAdapter( + arguments.root, + RipgrepAdapterOptions( + include_globs=tuple(arguments.includes or ("*.py", "*.md")), + ), + ) + result = adapter.search( + RipgrepSearchRequest( + arguments.query, + limit=arguments.limit, + case_sensitive=arguments.case_sensitive, + ) + ) + if arguments.json: + print(json.dumps(result.to_dict(), indent=2, sort_keys=True)) + return + for match in result.matches: + print(f"{match.source_uri} [{match.source_revision}]") + print(f" {match.line_text}") + print( + f"matches={len(result.matches)} truncated={str(result.truncated).lower()} " + f"durationMs={result.duration_ms}" + ) + + +if __name__ == "__main__": + main() diff --git a/runtimes/python/README.md b/runtimes/python/README.md index e9e62fa..919632d 100644 --- a/runtimes/python/README.md +++ b/runtimes/python/README.md @@ -80,15 +80,17 @@ same proof: vyral ``` -The quickstart creates and ingests a three-document corpus, returns cited hybrid -retrieval context, admits a decorated handler with a stable idempotency key, +The quickstart creates a three-document, source-backed record corpus without +vectors, returns cited lexical context, admits a decorated handler with a +stable idempotency key, closes the runtime before dispatch, reopens the same SQLite/filesystem state, and completes the preserved run identity. It reports the queued receipt before the close/reopen boundary so acceptance is not confused with completion. -The default `local-token-hash` provider is CPU-only, model-free, and requires no -network or downloaded assets. It provides deterministic lexical-overlap vectors -for local development; its ranking is not a semantic-model quality claim. +The quickstart does not invoke an embedding provider. The runtime's available +default `local-token-hash` provider remains CPU-only, model-free, and requires +no network or downloaded assets; when explicitly selected for vector mechanics, +its lexical-overlap ranking is not a semantic-model quality claim. Inspect the state and its material limitations independently: @@ -197,6 +199,15 @@ naming or dynamic registration is clearer. Handler and plugin IDs remain explicit because they are durable contract identities; renaming or moving a Python function must not silently create a different operation. +### Experimental source-native retrieval + +`vyral_runtime.integrations.ripgrep` provides a bounded, read-only experiment +for code, Markdown, and other safely accessible text sources. It uses a static +root and glob allowlist, fixed-string queries over standard input, source +revision citations, and strict resource limits. It is not part of the stable +wire contract and is not automatically exposed through REST or MCP. See the +[source-native retrieval guide](../../docs/guides/source-native-retrieval.md). + ### Experimental Extropic execution Install the optional `extropic` extra to place a registered Python workload diff --git a/runtimes/python/src/vyral_runtime/_local_experience.py b/runtimes/python/src/vyral_runtime/_local_experience.py index 77aea16..807910c 100644 --- a/runtimes/python/src/vyral_runtime/_local_experience.py +++ b/runtimes/python/src/vyral_runtime/_local_experience.py @@ -18,14 +18,10 @@ from .local import ( LexicalSearchOptions, RecordCollectionPolicy, - VectorFieldPolicy, + VyralRecord, ) -from .rag import ( - RagContextRequest, - RagIngestTextRequest, - RagIngestionOptions, -) -from .retrieval import EmbeddingOptions, HybridSearchOptions, RetrievalRequest +from .rag import RagContextRequest +from .retrieval import RetrievalRequest from .runtime import VyralRuntime @@ -93,6 +89,8 @@ class LocalQuickstartResult: contract_version: str maturity: str full_local_ready: bool + retrieval_mode: str + embedding_used: bool embedding_provider: str embedding_model: str embedding_dimensions: int @@ -124,6 +122,7 @@ def to_dict(self) -> dict[str, object]: "fullLocalReady": self.full_local_ready, "topology": "local-single-node", "embedding": { + "used": self.embedding_used, "provider": self.embedding_provider, "model": self.embedding_model, "dimensions": self.embedding_dimensions, @@ -131,6 +130,7 @@ def to_dict(self) -> dict[str, object]: "requiresNetwork": False, }, "retrieval": { + "mode": self.retrieval_mode, "query": self.query, "contextText": self.context_text, "contextHash": self.context_hash, @@ -221,51 +221,50 @@ async def run_local_quickstart( _emit( emit, ( - f"Embedding: {provider.provider_id}/{provider.model_id} " - f"({provider.dimensions} dimensions, " - f"{descriptor.semantic_quality}, no network)" + "Retrieval: lexical (SQLite, no embeddings generated, " + "no network)" ), ) runtime.records.create_collection( RecordCollectionPolicy( name=_COLLECTION, - vector_policies=( - VectorFieldPolicy( - name="contentEmbedding", - path="/vectors/contentEmbedding/values", - dimensions=provider.dimensions, - ), - ), indexed_metadata=("/metadata/topic",), ) ) for document_id, label, text, uri in _DOCUMENTS: - ingestion = runtime.rag_ingestion.ingest_text( - _COLLECTION, - RagIngestTextRequest( - document_id=document_id, - partition_key=_PARTITION, - text=text, - embedding=EmbeddingOptions( - field="contentEmbedding", - purpose="passage", - ), - metadata={"topic": document_id}, - source_uri=uri, - source_kind="documentation", - source_label=label, - options=RagIngestionOptions( - chunk_chars=600, - chunk_overlap_chars=0, - replace_document_chunks=True, - skip_unchanged_chunks=True, - persist_manifest=True, - ), + record = VyralRecord( + id=document_id, + partition_key=_PARTITION, + type="rag.chunk", + metadata={"topic": document_id}, + content={"text": text}, + sources=( + { + "id": document_id, + "kind": "documentation", + "uri": uri, + "label": label, + "span": None, + }, ), ) - created_chunks += ingestion.created_count - reused_chunks += ingestion.reused_count + existing = runtime.records.get_record( + _COLLECTION, + _PARTITION, + document_id, + ) + if ( + existing is not None + and existing.type == record.type + and existing.metadata == record.metadata + and existing.content == record.content + and existing.sources == record.sources + ): + reused_chunks += 1 + else: + runtime.records.upsert_record(_COLLECTION, record) + created_chunks += 1 context = runtime.rag_context.build_context( RagContextRequest( @@ -273,19 +272,10 @@ async def run_local_quickstart( query=_QUERY, collections=(_COLLECTION,), partition_keys=(_PARTITION,), - search_mode="hybrid", - embedding=EmbeddingOptions( - field="contentEmbedding", - purpose="query", - ), + search_mode="lexical", lexical=LexicalSearchOptions( fields=("/content/text",), ), - hybrid=HybridSearchOptions( - vector_weight=0.4, - lexical_weight=0.6, - fusion="weighted", - ), limit=3, include_trace=True, ), @@ -313,8 +303,8 @@ async def run_local_quickstart( emit, ( f"Retrieved {len(context.chunks)} chunks with " - f"{len(citations)} citations. Token-hash rankings are " - "model-free local-development evidence, not semantic-model quality." + f"{len(citations)} citations using exact, current lexical " + "content; embeddings were not used." ), ) @@ -383,6 +373,8 @@ async def run_local_quickstart( contract_version=readiness.contract_version, maturity=readiness.maturity, full_local_ready=readiness.full_local_ready, + retrieval_mode="lexical", + embedding_used=False, embedding_provider=provider.provider_id, embedding_model=provider.model_id, embedding_dimensions=provider.dimensions, diff --git a/runtimes/python/src/vyral_runtime/integrations/__init__.py b/runtimes/python/src/vyral_runtime/integrations/__init__.py index b024bc5..36a0173 100644 --- a/runtimes/python/src/vyral_runtime/integrations/__init__.py +++ b/runtimes/python/src/vyral_runtime/integrations/__init__.py @@ -17,6 +17,17 @@ ExtropicSdkBackend, ExtropicTransportError, ) +from .ripgrep import ( + RipgrepAdapterOptions, + RipgrepIntegrationError, + RipgrepNotAvailableError, + RipgrepSearchAdapter, + RipgrepSearchError, + RipgrepSearchLimitError, + RipgrepSearchRequest, + RipgrepSearchResult, + RipgrepSourceMatch, +) __all__ = [ "EXTROPIC_CHECKPOINT_KEY", @@ -34,4 +45,13 @@ "ExtropicRateLimitedError", "ExtropicSdkBackend", "ExtropicTransportError", + "RipgrepAdapterOptions", + "RipgrepIntegrationError", + "RipgrepNotAvailableError", + "RipgrepSearchAdapter", + "RipgrepSearchError", + "RipgrepSearchLimitError", + "RipgrepSearchRequest", + "RipgrepSearchResult", + "RipgrepSourceMatch", ] diff --git a/runtimes/python/src/vyral_runtime/integrations/ripgrep.py b/runtimes/python/src/vyral_runtime/integrations/ripgrep.py new file mode 100644 index 0000000..75fb4b4 --- /dev/null +++ b/runtimes/python/src/vyral_runtime/integrations/ripgrep.py @@ -0,0 +1,553 @@ +"""Bounded, read-only source search using a locally installed ripgrep. + +This integration is experimental and intentionally sits outside Vyral's stable +wire contract. It lets an application evaluate source-native retrieval before +copying an already-searchable corpus into a durable retrieval index. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import subprocess +from threading import Event, Thread +from time import perf_counter +from typing import Any, cast +from urllib.parse import quote + + +_BASELINE_EXCLUDED_GLOBS = ( + "**/.git/**", + "**/.env", + "**/.env.*", + "**/*.key", + "**/*.pem", + "**/*.p12", + "**/*.pfx", + "**/*.kdbx", + "**/id_rsa", + "**/id_ed25519", + "**/credentials.json", + "**/secrets.json", + "**/secrets.yaml", + "**/secrets.yml", +) +_SENSITIVE_FILENAMES = frozenset( + { + ".env", + "credentials", + "credentials.json", + "id_ed25519", + "id_rsa", + "secrets", + "secrets.json", + "secrets.yaml", + "secrets.yml", + } +) +_SENSITIVE_SUFFIXES = frozenset({".key", ".kdbx", ".p12", ".pem", ".pfx"}) + + +class RipgrepIntegrationError(RuntimeError): + """Base error for the experimental ripgrep integration.""" + + +class RipgrepNotAvailableError(RipgrepIntegrationError): + """Raised when a suitable ripgrep executable cannot be resolved.""" + + +class RipgrepSearchError(RipgrepIntegrationError): + """Raised when ripgrep cannot safely complete a search.""" + + +class RipgrepSearchLimitError(RipgrepSearchError): + """Raised when ripgrep exceeds a configured output boundary.""" + + +@dataclass(frozen=True) +class RipgrepAdapterOptions: + """Static policy for one allowlisted source root.""" + + include_globs: tuple[str, ...] + exclude_globs: tuple[str, ...] = () + executable: str = "rg" + max_results: int = 50 + max_query_chars: int = 512 + max_line_chars: int = 2_000 + max_file_bytes: int = 8 * 1024 * 1024 + max_output_bytes: int = 1024 * 1024 + timeout_seconds: float = 5.0 + + def __post_init__(self) -> None: + if not self.include_globs: + raise ValueError("ripgrep include_globs must contain at least one glob") + for pattern in self.include_globs: + _validate_glob(pattern, "include") + if pattern.startswith("!"): + raise ValueError("ripgrep include globs must not start with '!'") + for pattern in self.exclude_globs: + _validate_glob(pattern, "exclude") + if pattern.startswith("!"): + raise ValueError("ripgrep exclude globs must not start with '!'") + if not self.executable.strip(): + raise ValueError("ripgrep executable must not be empty") + _positive(self.max_results, "ripgrep max_results") + _positive(self.max_query_chars, "ripgrep max_query_chars") + _positive(self.max_line_chars, "ripgrep max_line_chars") + _positive(self.max_file_bytes, "ripgrep max_file_bytes") + _positive(self.max_output_bytes, "ripgrep max_output_bytes") + if ( + isinstance(self.timeout_seconds, bool) + or not isinstance(self.timeout_seconds, (int, float)) + or not math.isfinite(self.timeout_seconds) + or self.timeout_seconds <= 0 + ): + raise ValueError("ripgrep timeout_seconds must be greater than zero") + + +@dataclass(frozen=True) +class RipgrepSearchRequest: + """One fixed-string search within the adapter's static policy.""" + + query: str + limit: int = 20 + case_sensitive: bool = False + + +@dataclass(frozen=True) +class RipgrepSourceMatch: + relative_path: str + line_number: int + byte_column: int + line_text: str + matched_text: str + source_uri: str + source_revision: str + + def to_dict(self) -> dict[str, object]: + return { + "relativePath": self.relative_path, + "lineNumber": self.line_number, + "byteColumn": self.byte_column, + "lineText": self.line_text, + "matchedText": self.matched_text, + "sourceUri": self.source_uri, + "sourceRevision": self.source_revision, + } + + +@dataclass(frozen=True) +class RipgrepSearchResult: + query: str + matches: tuple[RipgrepSourceMatch, ...] + truncated: bool + filtered_sensitive_paths: int + duration_ms: float + executable_version: str + + def to_dict(self) -> dict[str, object]: + return { + "query": self.query, + "matches": [match.to_dict() for match in self.matches], + "truncated": self.truncated, + "filteredSensitivePaths": self.filtered_sensitive_paths, + "durationMs": self.duration_ms, + "executableVersion": self.executable_version, + } + + +class RipgrepSearchAdapter: + """Search one fixed root without shell expansion or caller-selected paths.""" + + def __init__( + self, + root_path: str | Path, + options: RipgrepAdapterOptions, + ) -> None: + self._root = _resolve_root(root_path) + self._options = options + self._executable = _resolve_executable(options.executable) + self._version = self._read_version() + + @property + def root_path(self) -> Path: + return self._root + + @property + def executable_version(self) -> str: + return self._version + + def search( + self, + request: RipgrepSearchRequest | str, + ) -> RipgrepSearchResult: + selected = ( + RipgrepSearchRequest(query=request) + if isinstance(request, str) + else request + ) + query = _validate_query(selected.query, self._options.max_query_chars) + if selected.limit <= 0 or selected.limit > self._options.max_results: + raise ValueError( + "ripgrep request limit must be between 1 and " + f"{self._options.max_results}" + ) + + command = [ + str(self._executable), + "--no-config", + "--json", + "--color", + "never", + "--line-number", + "--column", + "--max-columns", + str(self._options.max_line_chars), + "--max-columns-preview", + "--max-filesize", + str(self._options.max_file_bytes), + "--fixed-strings", + "--case-sensitive" if selected.case_sensitive else "--ignore-case", + ] + for pattern in self._options.include_globs: + command.extend(("--glob", pattern)) + for pattern in (*_BASELINE_EXCLUDED_GLOBS, *self._options.exclude_globs): + command.extend(("--glob", f"!{pattern}")) + command.extend(("--file", "-", ".")) + + started_at = perf_counter() + return_code, output, exceeded = _run_bounded_process( + command, + cwd=self._root, + stdin_data=f"{query}\n".encode("utf-8"), + max_output_bytes=self._options.max_output_bytes, + timeout_seconds=self._options.timeout_seconds, + ) + if exceeded: + raise RipgrepSearchLimitError( + "ripgrep exceeded the configured output boundary" + ) + if return_code not in (0, 1): + raise RipgrepSearchError( + f"ripgrep exited without a usable result (code {return_code})" + ) + + raw_matches = _parse_match_events(output) + revisions: dict[str, str] = {} + matches: list[RipgrepSourceMatch] = [] + filtered_sensitive_paths = 0 + truncated = False + for event in raw_matches: + relative_path = _safe_relative_path(event["path"]) + if _is_sensitive_path(relative_path): + filtered_sensitive_paths += 1 + continue + if len(matches) >= selected.limit: + truncated = True + break + source_path = _resolve_match_path(self._root, relative_path) + revision = revisions.get(relative_path) + if revision is None: + revision = _hash_regular_file( + source_path, + self._options.max_file_bytes, + ) + revisions[relative_path] = revision + line_number = event["line_number"] + matches.append( + RipgrepSourceMatch( + relative_path=relative_path, + line_number=line_number, + byte_column=event["byte_column"], + line_text=event["line_text"][: self._options.max_line_chars], + matched_text=event["matched_text"], + source_uri=( + "vyral-source://ripgrep/" + f"{quote(relative_path, safe='/')}#L{line_number}" + ), + source_revision=f"sha256:{revision}", + ) + ) + + return RipgrepSearchResult( + query=query, + matches=tuple(matches), + truncated=truncated, + filtered_sensitive_paths=filtered_sensitive_paths, + duration_ms=round((perf_counter() - started_at) * 1000, 3), + executable_version=self._version, + ) + + def _read_version(self) -> str: + return_code, output, exceeded = _run_bounded_process( + [str(self._executable), "--version"], + cwd=self._root, + stdin_data=None, + max_output_bytes=16 * 1024, + timeout_seconds=min(self._options.timeout_seconds, 2.0), + ) + if exceeded or return_code != 0: + raise RipgrepNotAvailableError( + "ripgrep did not return a bounded version response" + ) + first_line = output.decode("utf-8", errors="replace").splitlines() + if not first_line or not first_line[0].startswith("ripgrep "): + raise RipgrepNotAvailableError( + "the configured executable did not identify itself as ripgrep" + ) + return first_line[0] + + +def _positive(value: int, name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be greater than zero") + + +def _validate_glob(pattern: str, kind: str) -> None: + if not isinstance(pattern, str) or not pattern.strip(): + raise ValueError(f"ripgrep {kind} glob must not be empty") + if "\x00" in pattern or "\\" in pattern: + raise ValueError(f"ripgrep {kind} glob contains an unsupported character") + candidate = PurePosixPath(pattern) + if candidate.is_absolute() or ".." in candidate.parts: + raise ValueError(f"ripgrep {kind} glob must stay within the source root") + + +def _validate_query(query: str, max_chars: int) -> str: + if not isinstance(query, str) or not query.strip(): + raise ValueError("ripgrep query must not be empty") + if len(query) > max_chars: + raise ValueError(f"ripgrep query cannot exceed {max_chars} characters") + if any(ord(character) < 32 or ord(character) == 127 for character in query): + raise ValueError("ripgrep query must not contain control characters") + return query + + +def _resolve_root(root_path: str | Path) -> Path: + requested = Path(root_path).expanduser() + if requested.is_symlink(): + raise ValueError("ripgrep source root must not be a symbolic link") + root = requested.resolve() + if not root.is_dir(): + raise ValueError(f"ripgrep source root is not a directory: {root}") + anchor = Path(root.anchor) + if root == anchor or root == Path.home().resolve(): + raise ValueError("ripgrep source root is too broad") + return root + + +def _resolve_executable(executable: str) -> Path: + candidate = Path(executable).expanduser() + located = ( + str(candidate) + if candidate.is_absolute() + else shutil.which(executable) + ) + if located is None: + raise RipgrepNotAvailableError("ripgrep executable was not found") + resolved = Path(located).resolve() + if not resolved.is_file() or not os.access(resolved, os.X_OK): + raise RipgrepNotAvailableError( + "ripgrep executable is not a runnable regular file" + ) + return resolved + + +def _run_bounded_process( + command: list[str], + *, + cwd: Path, + stdin_data: bytes | None, + max_output_bytes: int, + timeout_seconds: float, +) -> tuple[int, bytes, bool]: + environment = dict(os.environ) + environment.pop("RIPGREP_CONFIG_PATH", None) + process = subprocess.Popen( + command, + cwd=cwd, + env=environment, + stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + assert process.stdout is not None + stdout = process.stdout + chunks: list[bytes] = [] + output_size = 0 + exceeded = Event() + read_failed = Event() + + def drain_stdout() -> None: + nonlocal output_size + try: + while True: + chunk = stdout.read(64 * 1024) + if not chunk: + return + remaining = max_output_bytes - output_size + if remaining <= 0 or len(chunk) > remaining: + if remaining > 0: + chunks.append(chunk[:remaining]) + output_size += remaining + exceeded.set() + process.kill() + return + chunks.append(chunk) + output_size += len(chunk) + except OSError: + read_failed.set() + process.kill() + + reader = Thread(target=drain_stdout, name="vyral-ripgrep-stdout", daemon=True) + reader.start() + if stdin_data is not None: + assert process.stdin is not None + try: + process.stdin.write(stdin_data) + process.stdin.close() + except BrokenPipeError: + pass + try: + return_code = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired as error: + process.kill() + process.wait() + reader.join(timeout=1.0) + stdout.close() + raise RipgrepSearchError( + "ripgrep exceeded the configured timeout" + ) from error + reader.join(timeout=1.0) + if reader.is_alive() or read_failed.is_set(): + process.kill() + stdout.close() + raise RipgrepSearchError("ripgrep output could not be read safely") + stdout.close() + return return_code, b"".join(chunks), exceeded.is_set() + + +def _parse_match_events(output: bytes) -> tuple[dict[str, Any], ...]: + matches: list[dict[str, Any]] = [] + for raw_line in output.splitlines(): + try: + envelope = json.loads(raw_line) + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise RipgrepSearchError("ripgrep returned malformed JSON") from error + if not isinstance(envelope, dict) or envelope.get("type") != "match": + continue + data = envelope.get("data") + if not isinstance(data, dict): + raise RipgrepSearchError("ripgrep returned an invalid match event") + path = _nested_text(data, "path") + line_text = _nested_text(data, "lines").rstrip("\r\n") + line_number = data.get("line_number") + submatches = data.get("submatches") + if ( + isinstance(line_number, bool) + or not isinstance(line_number, int) + or line_number <= 0 + or not isinstance(submatches, list) + or not submatches + or not isinstance(submatches[0], dict) + ): + raise RipgrepSearchError("ripgrep returned an invalid match position") + first = submatches[0] + start = first.get("start") + matched = first.get("match") + if isinstance(start, bool) or not isinstance(start, int) or start < 0: + raise RipgrepSearchError("ripgrep returned an invalid byte column") + if not isinstance(matched, dict) or not isinstance(matched.get("text"), str): + raise RipgrepSearchError("ripgrep returned a non-text match") + matches.append( + { + "path": path, + "line_number": line_number, + "byte_column": start + 1, + "line_text": line_text, + "matched_text": matched["text"], + } + ) + return tuple(matches) + + +def _nested_text(data: dict[str, Any], field: str) -> str: + value = data.get(field) + if not isinstance(value, dict) or not isinstance(value.get("text"), str): + raise RipgrepSearchError(f"ripgrep returned a non-text {field}") + return cast(str, value["text"]) + + +def _safe_relative_path(value: str) -> str: + normalized = value.removeprefix("./").replace("\\", "/") + candidate = PurePosixPath(normalized) + if not normalized or candidate.is_absolute() or ".." in candidate.parts: + raise RipgrepSearchError("ripgrep returned a path outside the source root") + return candidate.as_posix() + + +def _is_sensitive_path(relative_path: str) -> bool: + for component in PurePosixPath(relative_path).parts: + lowered = component.casefold() + if ( + lowered in _SENSITIVE_FILENAMES + or lowered.startswith(".env.") + or lowered.startswith("secrets.") + or Path(lowered).suffix in _SENSITIVE_SUFFIXES + ): + return True + return False + + +def _resolve_match_path(root: Path, relative_path: str) -> Path: + candidate = root.joinpath(*PurePosixPath(relative_path).parts) + if candidate.is_symlink(): + raise RipgrepSearchError("ripgrep matched a symbolic link") + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (FileNotFoundError, ValueError) as error: + raise RipgrepSearchError( + "ripgrep matched a path outside the source root or a changed source" + ) from error + return resolved + + +def _hash_regular_file(path: Path, max_file_bytes: int) -> str: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + file_descriptor = os.open(path, flags) + except OSError as error: + raise RipgrepSearchError("a matched source could not be opened safely") from error + digest = hashlib.sha256() + try: + with os.fdopen(file_descriptor, "rb") as source: + details = os.fstat(source.fileno()) + if not stat.S_ISREG(details.st_mode) or details.st_size > max_file_bytes: + raise RipgrepSearchError( + "a matched source is not a bounded regular file" + ) + while chunk := source.read(64 * 1024): + digest.update(chunk) + except OSError as error: + raise RipgrepSearchError("a matched source changed during hashing") from error + return digest.hexdigest() + + +__all__ = [ + "RipgrepAdapterOptions", + "RipgrepIntegrationError", + "RipgrepNotAvailableError", + "RipgrepSearchAdapter", + "RipgrepSearchError", + "RipgrepSearchLimitError", + "RipgrepSearchRequest", + "RipgrepSearchResult", + "RipgrepSourceMatch", +] diff --git a/runtimes/python/tests/test_local_experience.py b/runtimes/python/tests/test_local_experience.py index 7589a58..4e23aa1 100644 --- a/runtimes/python/tests/test_local_experience.py +++ b/runtimes/python/tests/test_local_experience.py @@ -10,6 +10,8 @@ reset_local_quickstart, run_local_quickstart_sync, ) +from vyral_runtime.local import QueryEnvelope +from vyral_runtime.runtime import VyralRuntime class LocalExperienceTests(unittest.TestCase): @@ -24,6 +26,8 @@ def test_citations_and_execution_survive_a_real_reopen(self) -> None: self.assertEqual("prototype", first.maturity) self.assertFalse(first.full_local_ready) + self.assertEqual("lexical", first.retrieval_mode) + self.assertFalse(first.embedding_used) self.assertEqual("local-token-hash", first.embedding_provider) self.assertEqual("lexical", first.embedding_semantic_quality) self.assertEqual(3, first.created_chunks) @@ -58,6 +62,15 @@ def test_citations_and_execution_survive_a_real_reopen(self) -> None: "accepted work survived the runtime restart", first.completed_result["message"], ) + serialized = first.to_dict() + retrieval = serialized["retrieval"] + embedding = serialized["embedding"] + self.assertIsInstance(retrieval, dict) + self.assertIsInstance(embedding, dict) + assert isinstance(retrieval, dict) + assert isinstance(embedding, dict) + self.assertEqual("lexical", retrieval["mode"]) + self.assertFalse(embedding["used"]) self.assertLess( next( index @@ -79,6 +92,20 @@ def test_citations_and_execution_survive_a_real_reopen(self) -> None: self.assertEqual(0, second.dispatched_runs) self.assertTrue(second.admission_replayed) + with VyralRuntime.open_local(root) as runtime: + policy = runtime.records.get_collection_policy( + "vyral-quickstart" + ) + self.assertIsNotNone(policy) + assert policy is not None + self.assertEqual((), policy.vector_policies) + records = runtime.records.query_all_records( + "vyral-quickstart", + QueryEnvelope(partition_keys=("local",)), + ) + self.assertEqual(3, len(records)) + self.assertTrue(all(not record.vectors for record in records)) + inspection = inspect_local_runtime(root) self.assertEqual("local-single-node", inspection["topology"]) providers = inspection["providers"] diff --git a/runtimes/python/tests/test_ripgrep_integration.py b/runtimes/python/tests/test_ripgrep_integration.py new file mode 100644 index 0000000..957bdf4 --- /dev/null +++ b/runtimes/python/tests/test_ripgrep_integration.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path +import shutil +import tempfile +import unittest + +from vyral_runtime.integrations.ripgrep import ( + RipgrepAdapterOptions, + RipgrepNotAvailableError, + RipgrepSearchAdapter, + RipgrepSearchLimitError, + RipgrepSearchRequest, +) + + +@unittest.skipUnless(shutil.which("rg"), "ripgrep is not installed") +class RipgrepIntegrationTests(unittest.TestCase): + def test_search_is_bounded_cited_and_filters_sensitive_paths(self) -> None: + with tempfile.TemporaryDirectory(prefix="vyral-ripgrep-") as temporary: + root = Path(temporary) / "source" + root.mkdir() + python_source = root / "worker.py" + python_source.write_text( + "def preserve_receipt():\n" + " return 'accepted work survives restart'\n", + encoding="utf-8", + ) + (root / "notes.md").write_text( + "Accepted work survives restart through a durable receipt.\n", + encoding="utf-8", + ) + (root / "secrets.md").write_text( + "accepted work survives restart: do-not-return\n", + encoding="utf-8", + ) + (root / ".env").write_text( + "MESSAGE=accepted work survives restart\n", + encoding="utf-8", + ) + + adapter = RipgrepSearchAdapter( + root, + RipgrepAdapterOptions(include_globs=("*.py", "*.md", ".env")), + ) + result = adapter.search("accepted work survives restart") + + self.assertTrue(result.executable_version.startswith("ripgrep ")) + self.assertEqual(1, result.filtered_sensitive_paths) + self.assertEqual({"notes.md", "worker.py"}, { + match.relative_path for match in result.matches + }) + self.assertFalse(result.truncated) + self.assertNotIn(str(root), str(result.to_dict())) + for match in result.matches: + self.assertTrue(match.source_uri.startswith("vyral-source://ripgrep/")) + self.assertIn("#L", match.source_uri) + expected = hashlib.sha256( + (root / match.relative_path).read_bytes() + ).hexdigest() + self.assertEqual(f"sha256:{expected}", match.source_revision) + self.assertGreaterEqual(match.line_number, 1) + self.assertGreaterEqual(match.byte_column, 1) + + def test_request_limit_and_case_policy_are_enforced(self) -> None: + with tempfile.TemporaryDirectory(prefix="vyral-ripgrep-") as temporary: + root = Path(temporary) / "source" + root.mkdir() + (root / "a.md").write_text("Needle\n", encoding="utf-8") + (root / "b.md").write_text("needle\n", encoding="utf-8") + adapter = RipgrepSearchAdapter( + root, + RipgrepAdapterOptions(include_globs=("*.md",), max_results=2), + ) + + insensitive = adapter.search(RipgrepSearchRequest("needle", limit=1)) + self.assertEqual(1, len(insensitive.matches)) + self.assertTrue(insensitive.truncated) + + sensitive = adapter.search( + RipgrepSearchRequest("Needle", limit=2, case_sensitive=True) + ) + self.assertEqual(("a.md",), tuple( + match.relative_path for match in sensitive.matches + )) + + with self.assertRaisesRegex(ValueError, "between 1 and 2"): + adapter.search(RipgrepSearchRequest("needle", limit=3)) + with self.assertRaisesRegex(ValueError, "control characters"): + adapter.search("needle\nsecond-pattern") + + def test_static_policy_rejects_unsafe_roots_and_globs(self) -> None: + with self.assertRaisesRegex(ValueError, "at least one"): + RipgrepAdapterOptions(include_globs=()) + with self.assertRaisesRegex(ValueError, "within the source root"): + RipgrepAdapterOptions(include_globs=("../*.md",)) + with self.assertRaisesRegex(ValueError, "must not start"): + RipgrepAdapterOptions(include_globs=("!*.key",)) + with self.assertRaisesRegex(ValueError, "too broad"): + RipgrepSearchAdapter( + Path.home(), + RipgrepAdapterOptions(include_globs=("*.md",)), + ) + + def test_output_and_executable_boundaries_fail_closed(self) -> None: + with tempfile.TemporaryDirectory(prefix="vyral-ripgrep-") as temporary: + root = Path(temporary) / "source" + root.mkdir() + (root / "large.md").write_text( + "needle with enough output to exceed the boundary\n" * 200, + encoding="utf-8", + ) + adapter = RipgrepSearchAdapter( + root, + RipgrepAdapterOptions( + include_globs=("*.md",), + max_output_bytes=256, + ), + ) + with self.assertRaisesRegex( + RipgrepSearchLimitError, + "output boundary", + ): + adapter.search("needle") + + with self.assertRaises(RipgrepNotAvailableError): + RipgrepSearchAdapter( + root, + RipgrepAdapterOptions( + include_globs=("*.md",), + executable=str(root / "missing-rg"), + ), + ) + + alias = Path(temporary) / "source-link" + try: + alias.symlink_to(root, target_is_directory=True) + except OSError: + return + with self.assertRaisesRegex(ValueError, "symbolic link"): + RipgrepSearchAdapter( + alias, + RipgrepAdapterOptions(include_globs=("*.md",)), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/export-public-tree.py b/scripts/export-public-tree.py index 27ace5d..afc7670 100755 --- a/scripts/export-public-tree.py +++ b/scripts/export-public-tree.py @@ -63,6 +63,7 @@ "docs/contributing/adapter-contributor.md", "docs/guides/consumer-handoff.md", "docs/guides/extropic-execution.md", + "docs/guides/source-native-retrieval.md", "docs/maintainers/releasing.md", "docs/reference/execution-runtime-limitations.md", "docs/reference/stability.md", @@ -135,6 +136,7 @@ "docs/concepts/canonical-store.md", "docs/contributing/adapter-contributor.md", "docs/guides/consumer-handoff.md", + "docs/guides/source-native-retrieval.md", "docs/maintainers/releasing.md", "docs/reference/execution-runtime-limitations.md", "docs/reference/stability.md", diff --git a/scripts/verify-lexical-http-quickstarts.sh b/scripts/verify-lexical-http-quickstarts.sh new file mode 100755 index 0000000..c650265 --- /dev/null +++ b/scripts/verify-lexical-http-quickstarts.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SERVER_DLL="$ROOT/src/Vyral.Server/bin/Release/net10.0/Vyral.Server.dll" +if [[ ! -f "$SERVER_DLL" ]]; then + echo "Build Vyral.sln in Release mode before running this gate." >&2 + exit 2 +fi + +work_root="$(mktemp -d "${TMPDIR:-/tmp}/vyral-lexical-http-XXXXXX")" +server_pid="" +cleanup() { + if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi + rm -rf "$work_root" +} +trap cleanup EXIT + +port="$(python3 - <<'PY' +from __future__ import annotations + +import socket + +with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + print(listener.getsockname()[1]) +PY +)" +url="http://127.0.0.1:$port" + +ASPNETCORE_ENVIRONMENT=Development \ +DOTNET_ENVIRONMENT=Development \ +ASPNETCORE_URLS="$url" \ +DatabasePath="$work_root/vyral.sqlite" \ +ObjectsPath="$work_root/objects" \ +Providers__ArtifactDirectory="$work_root/provider-runs" \ +dotnet "$SERVER_DLL" >"$work_root/server.log" 2>&1 & +server_pid="$!" + +ready=false +for _ in $(seq 1 100); do + if curl --fail --silent --max-time 1 "$url/health" >/dev/null; then + ready=true + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + break + fi + sleep 0.1 +done +if [[ "$ready" != "true" ]]; then + cat "$work_root/server.log" >&2 + echo "The isolated Vyral server did not become ready." >&2 + exit 1 +fi + +( + cd "$ROOT" + VYRAL_URL="$url" python3 examples/python/rag_quickstart.py \ + >"$work_root/python.txt" + VYRAL_URL="$url" node examples/javascript/rag-quickstart.mjs \ + >"$work_root/javascript.txt" +) + +for output in "$work_root/python.txt" "$work_root/javascript.txt"; do + grep -q '^retrieval=lexical embeddings=unused$' "$output" + grep -q '^\[c1\] memory://quickstart/retention$' "$output" + grep -q '^contextTextHash=sha256:' "$output" + grep -q '^promptHash=sha256:' "$output" +done + +printf 'lexical-http-quickstarts=ok server=%s languages=python,javascript\n' "$url" diff --git a/scripts/verify-python-runtime.sh b/scripts/verify-python-runtime.sh index 2898d3f..31bdf17 100755 --- a/scripts/verify-python-runtime.sh +++ b/scripts/verify-python-runtime.sh @@ -72,6 +72,7 @@ required_package_files = { "vyral_runtime/_contracts/public-sdk-surface.json", "vyral_runtime/_contracts/vyral-public.schema.json", "vyral_runtime/_contracts/vyral.openapi.json", + "vyral_runtime/integrations/ripgrep.py", "vyral_runtime/_conformance/runtime/v1/manifest.json", "vyral_runtime/_conformance/runtime/v1/manifest.schema.json", "vyral_runtime/_conformance/runtime/v1/scenario.schema.json", @@ -118,6 +119,7 @@ with tarfile.open(sdist_path, "r:gz") as archive: "src/vyral_runtime/_contracts/public-sdk-surface.json", "src/vyral_runtime/_contracts/vyral-public.schema.json", "src/vyral_runtime/_contracts/vyral.openapi.json", + "src/vyral_runtime/integrations/ripgrep.py", "src/vyral_runtime/_conformance/runtime/v1/manifest.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/primitives-hashing.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/collection-snapshot-hash.json", @@ -201,6 +203,8 @@ replay = json.loads(replay_path.read_text(encoding="utf-8")) assert first["topology"] == "local-single-node" assert first["maturity"] == "prototype" assert first["fullLocalReady"] is False +assert first["retrieval"]["mode"] == "lexical" +assert first["embedding"]["used"] is False assert first["embedding"]["provider"] == "local-token-hash" assert first["embedding"]["requiresNetwork"] is False assert len(first["retrieval"]["citations"]) >= 1 From 275dada9ee308c09f3df01d5291fc849a5f519bf Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 03:52:02 -0400 Subject: [PATCH 2/8] chore(release): refresh public export evidence --- PUBLIC-EXPORT-MANIFEST.json | 57 ++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 27cc8af..878c743 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -1,5 +1,5 @@ { - "fileCount": 827, + "fileCount": 832, "files": [ { "mode": "644", @@ -54,7 +54,7 @@ { "mode": "644", "path": ".github/workflows/ci.yml", - "sha256": "80d2b432d425187dda1b08e377db5befec12ed5ac1b52876d9374de87246e6cd" + "sha256": "d8c34f6654ab50453c351e691f12e47a182a906a653803344f1744b61b1c9c64" }, { "mode": "644", @@ -104,7 +104,7 @@ { "mode": "644", "path": ".gitignore", - "sha256": "57ddda23948159577073baa5dbea4082b33c22f975b9b5669c65ef8810eaec3b" + "sha256": "7b15c20e8f94c53e6c50a55dc511928718af35a1cd0e3c9104c10d1ad43ef969" }, { "mode": "644", @@ -134,7 +134,7 @@ { "mode": "644", "path": "README.md", - "sha256": "77fcc61d84373d0cf29e1b9935105c7b104cb3f24f3cd3d6771a5e19606f3bfe" + "sha256": "2a6443fe2d34ae2d31c36ca706db9c8d55719fff8e3c9fbc48f8567eaffa546b" }, { "mode": "644", @@ -559,7 +559,7 @@ { "mode": "644", "path": "docs/README.md", - "sha256": "a4a4f2479130c950c0856f0b0d9d0e6858f26d42cedeaa46192eaf489617e8e3" + "sha256": "2a846342f0e616eaefcc46ba817ab7ab73b9f7611be92b85e7407e70ef974d46" }, { "mode": "644", @@ -579,13 +579,18 @@ { "mode": "644", "path": "docs/guides/consumer-handoff.md", - "sha256": "fb19dee1acb9d02f3f50fa79e4c17ec9313c90a76e4f547ebd88bb576f69ca84" + "sha256": "2de27f52d6ebc717ebe74a422fb9c028233aaa0c04ca7e8bbfdf71b3b596868c" }, { "mode": "644", "path": "docs/guides/extropic-execution.md", "sha256": "09e9c18f2f54f28da367906c197abbd192699d686387a147fdf114feff51f656" }, + { + "mode": "644", + "path": "docs/guides/source-native-retrieval.md", + "sha256": "e8cf1dfcf5bd815d2f320262d9854d8083a197eedbe8a91bfaf5505b02299ce9" + }, { "mode": "644", "path": "docs/maintainers/releasing.md", @@ -614,7 +619,7 @@ { "mode": "644", "path": "examples/README.md", - "sha256": "2163d0dd9b99a47e3f9dda60d4afc13cc6e80724dd7bfa3f2e2678afeeda94ae" + "sha256": "53ec5f86eb7e5deb8eeac8b12cbb365962b53ac4ec1c54bf094cfe42d8162bc4" }, { "mode": "644", @@ -624,7 +629,7 @@ { "mode": "644", "path": "examples/javascript/rag-quickstart.mjs", - "sha256": "3c603757a807432edc647d3945222b2098e235d0b4af6fe4854238ecf1a7711f" + "sha256": "a2f6dab248bdc892a8beaecec9cd0935fddb1bfe782f4c07503b192c2ee2fe00" }, { "mode": "644", @@ -644,7 +649,12 @@ { "mode": "644", "path": "examples/python/rag_quickstart.py", - "sha256": "9ccd3f757d1f90e773fd13cf719f75baf938b1663567915c72636f9247e1ee3b" + "sha256": "6210fcbcfaa8419b30576096cc48f82ff87a8c1de04e0acb3aa64d7edfdc7559" + }, + { + "mode": "644", + "path": "examples/python/source_native_search.py", + "sha256": "6078532d1b25733f3b0b9a4556cf5445c7e24f29b028e87389328c9dc7f6280b" }, { "mode": "644", @@ -674,7 +684,7 @@ { "mode": "644", "path": "runtimes/python/README.md", - "sha256": "f74eeb84a7bb500e7bfab63279a68737b0d0315d2e68fd2197b03a3996e407f2" + "sha256": "e1f79f657e531c49211524fa8405b21700f6f586753352af68cdd91e78e4785f" }, { "mode": "644", @@ -809,7 +819,7 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/_local_experience.py", - "sha256": "c00ff042ebb73e6518b9f2e87a50c61a4b1daf6512dd6fab472403f3d794689d" + "sha256": "c983078cb9addfb0141271f397512ef32657ac5e83afc938f1b5485ff996667d" }, { "mode": "644", @@ -1019,13 +1029,18 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/integrations/__init__.py", - "sha256": "92e6f7c95b560675cf838c4d1aeafa5c7490afce5f2a932c3d09ea53f9a65afe" + "sha256": "bf107710396b5386c6dab15accf39a9fcda991d66397577475f3064a908e9f10" }, { "mode": "644", "path": "runtimes/python/src/vyral_runtime/integrations/extropic.py", "sha256": "b2068cd5495d763c3f7831300afb144bbffa4e83f3cd0713cfd7522792050fbe" }, + { + "mode": "644", + "path": "runtimes/python/src/vyral_runtime/integrations/ripgrep.py", + "sha256": "6cc8f26459b80cc7b1d0c477273ed3de2acab4f72ff21843eebb23760c1ffae4" + }, { "mode": "644", "path": "runtimes/python/src/vyral_runtime/local/__init__.py", @@ -1229,7 +1244,7 @@ { "mode": "644", "path": "runtimes/python/tests/test_local_experience.py", - "sha256": "fe7948aa0501f626ca7b7485048d0ee3a22438f187c8857256ab5773037664c0" + "sha256": "1838232710d321167f0d27c1289e0ad21c160c78d60f42385376fdb7a55bfdb5" }, { "mode": "644", @@ -1281,6 +1296,11 @@ "path": "runtimes/python/tests/test_retrieval_evaluation.py", "sha256": "8c7e5ae6f0ab68478038c79c1e7b262b6edf4a21df7152a34baebeb212275229" }, + { + "mode": "644", + "path": "runtimes/python/tests/test_ripgrep_integration.py", + "sha256": "72383a2520106f05e9f474452f93f8f9620cbf0343418bc5f487d817c49d937b" + }, { "mode": "644", "path": "runtimes/python/tests/test_runtime.py", @@ -1444,7 +1464,7 @@ { "mode": "755", "path": "scripts/export-public-tree.py", - "sha256": "cda75ce6fc5864869f471ca3de0cd9172fdd3b37ae2bc850d6847f6818ccef5e" + "sha256": "4c51919eac13813eec8679e8b17c7580776ea306ce67d8d05fcdfae14cd097b6" }, { "mode": "755", @@ -1681,6 +1701,11 @@ "path": "scripts/verify-execution-smoke-container.sh", "sha256": "954a99316778d1316df8966d56ca064e60631d47a9f9410d2be3053b0c0e6b2f" }, + { + "mode": "755", + "path": "scripts/verify-lexical-http-quickstarts.sh", + "sha256": "4e7bc7d534aaef9f553a337f23c032d6915641b725a9a4b416c09e122d6b486f" + }, { "mode": "755", "path": "scripts/verify-markdown-links.py", @@ -1779,7 +1804,7 @@ { "mode": "755", "path": "scripts/verify-python-runtime.sh", - "sha256": "fbd9cd085eff6803debdad5bc52407a85d99fc5f2b9bae3d5b9de167f59b8e63" + "sha256": "7c8362e518ffbc810174972e9340f5d69cf9325211a33d42612227b2d2e13827" }, { "mode": "755", @@ -4139,5 +4164,5 @@ ], "schemaVersion": 1, "sourceDirty": false, - "treeSha256": "6ede7d4303711c80c351578da49700971ce01b18baa4729cf9566bc6e4561649" + "treeSha256": "8f310317079832bea59c83c253dcf26b2bebbc6e6c8963c2b1c2d8c0dd7e7733" } From 9931243b07d18f0c959a7182455fdb032cf15904 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:31:14 -0400 Subject: [PATCH 3/8] test(retrieval): require ripgrep admission evidence --- .github/workflows/ci.yml | 24 + PUBLIC-EXPORT-MANIFEST.json | 27 +- .../retrieval/fixtures/source-native-v1.json | 185 ++++++ .../src/vyral_runtime/integrations/ripgrep.py | 11 +- .../python/tests/test_ripgrep_integration.py | 1 + scripts/benchmark-ripgrep-retrieval.py | 598 ++++++++++++++++++ scripts/verify-ripgrep-retrieval-report.py | 369 +++++++++++ 7 files changed, 1208 insertions(+), 7 deletions(-) create mode 100644 benchmarks/retrieval/fixtures/source-native-v1.json create mode 100755 scripts/benchmark-ripgrep-retrieval.py create mode 100755 scripts/verify-ripgrep-retrieval-report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5763cf..280c7ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,30 @@ jobs: python -m mypy --config-file clients/python/pyproject.toml clients/python/tests/typecheck_consumer.py scripts/verify-python-runtime.sh + - name: Verify the source-native retrieval comparison + run: | + report="$(mktemp)" + tampered_report="$(mktemp)" + python3 scripts/benchmark-ripgrep-retrieval.py \ + --output "$report" \ + --noise-documents 60 \ + --iterations 3 \ + --require-admission + python3 scripts/verify-ripgrep-retrieval-report.py \ + "$report" \ + --minimum-noise 60 \ + --minimum-iterations 3 \ + --require-admission + jq '.quality.variants["ripgrep-fixed"].cases[0].metrics.recallAtK = 0' \ + "$report" > "$tampered_report" + if python3 scripts/verify-ripgrep-retrieval-report.py \ + "$tampered_report" \ + --minimum-noise 60 \ + --minimum-iterations 3; then + echo "The ripgrep comparison verifier accepted altered metrics." >&2 + exit 1 + fi + - name: Test Go worker client working-directory: clients/go run: go test ./... diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 878c743..9baad70 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -1,5 +1,5 @@ { - "fileCount": 832, + "fileCount": 835, "files": [ { "mode": "644", @@ -54,7 +54,7 @@ { "mode": "644", "path": ".github/workflows/ci.yml", - "sha256": "d8c34f6654ab50453c351e691f12e47a182a906a653803344f1744b61b1c9c64" + "sha256": "246b11dbc507bb36eec6491b5ae4569c574faa5400379c7222470b5fb4657818" }, { "mode": "644", @@ -171,6 +171,11 @@ "path": "benchmarks/canonical-store/mysql-8.0.46-local.json", "sha256": "9032bf64760c76820016af95192c1a7b60f38fad7380e1e8ef3bc88c35bcbb81" }, + { + "mode": "644", + "path": "benchmarks/retrieval/fixtures/source-native-v1.json", + "sha256": "abf12894b9720f59872a63f2e596bfa5b0fa268b1ebbd53a239561c5055618bb" + }, { "mode": "644", "path": "clients/go/README.md", @@ -1039,7 +1044,7 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/integrations/ripgrep.py", - "sha256": "6cc8f26459b80cc7b1d0c477273ed3de2acab4f72ff21843eebb23760c1ffae4" + "sha256": "d662ba9cded10b2dc33e910ae1275e012a8d1a20db0ee8623779c0b3a4018a80" }, { "mode": "644", @@ -1299,7 +1304,7 @@ { "mode": "644", "path": "runtimes/python/tests/test_ripgrep_integration.py", - "sha256": "72383a2520106f05e9f474452f93f8f9620cbf0343418bc5f487d817c49d937b" + "sha256": "9117dddae9441b7b1d21137d5533115e710554e04a36c7bcdd06c2c153d10bae" }, { "mode": "644", @@ -1451,6 +1456,11 @@ "path": "scripts/benchmark-python-runtime.py", "sha256": "08282add3792153654fbcb6211a974e94e95f693ab7c778f33e6c1e83c6085e2" }, + { + "mode": "755", + "path": "scripts/benchmark-ripgrep-retrieval.py", + "sha256": "a2ffcc98b2ef92697a2d0a879f6282395ba073ae65ba4776bb398156cb182f20" + }, { "mode": "755", "path": "scripts/download-benchmark-onnx-models.sh", @@ -1811,6 +1821,11 @@ "path": "scripts/verify-release-artifacts.sh", "sha256": "37656bb728e094285b4915f11044fea227cc1c3c34d1497ad8cbde425090d0a6" }, + { + "mode": "755", + "path": "scripts/verify-ripgrep-retrieval-report.py", + "sha256": "bd3ae53e0b90e2ba55c987d28e90e7fe31be5474b6c84a294eeaf2a8e4f507c7" + }, { "mode": "755", "path": "scripts/verify-runtime-conformance.py", @@ -4163,6 +4178,6 @@ } ], "schemaVersion": 1, - "sourceDirty": false, - "treeSha256": "8f310317079832bea59c83c253dcf26b2bebbc6e6c8963c2b1c2d8c0dd7e7733" + "sourceDirty": true, + "treeSha256": "cf37ea61c1d4998534ae5ad689342d0e66ce3b6e0bd8a6cb7476c01b437e60ec" } diff --git a/benchmarks/retrieval/fixtures/source-native-v1.json b/benchmarks/retrieval/fixtures/source-native-v1.json new file mode 100644 index 0000000..4be56dc --- /dev/null +++ b/benchmarks/retrieval/fixtures/source-native-v1.json @@ -0,0 +1,185 @@ +{ + "schemaVersion": "vyral.retrieval.source-native-fixture.v1", + "documents": [ + { + "path": "runtime/admission.py", + "content": "ADMISSION_RECEIPT_V1 = \"accepted work survives restart\"\n\ndef admit_work(worker_id: str) -> str:\n \"\"\"Commit the durable receipt before a worker executes the handler.\"\"\"\n return f\"receipt:{worker_id}\"\n" + }, + { + "path": "runtime/recovery.md", + "content": "# Recovery\n\nA persisted acceptance receipt lets queued work resume after a process restart. The dispatcher reuses the preserved run identity.\n" + }, + { + "path": "runtime/retry.md", + "content": "# Retry policy\n\nERR_RETRY_EXHAUSTED_503 marks a handler that exhausted exponential backoff with jitter. A retry is not a new admission.\n" + }, + { + "path": "runtime/worker-pool.md", + "content": "# Worker pool\n\nA worker may restart after a routine deployment. Pool capacity and shutdown grace periods are deployment concerns.\n" + }, + { + "path": "retrieval/source-native.md", + "content": "# Source-native retrieval\n\nUse source-native fixed-string search for exact identifiers in current local code and Markdown. It requires no copied index.\n" + }, + { + "path": "retrieval/semantic.md", + "content": "# Semantic retrieval\n\nVector retrieval helps when a question and its answer share meaning but not vocabulary. Embeddings trade freshness and operating cost for semantic recall.\n" + }, + { + "path": "retrieval/prefix.md", + "content": "# Legal retrieval fixture\n\nThe preliminary injunction deadline is recorded in the governed matter index.\n" + }, + { + "path": "retrieval/hard-negative.md", + "content": "# Restart checklist\n\nRestart the documentation preview worker after changing the theme. This page does not describe durable admission.\n" + }, + { + "path": "mcp/routing.md", + "content": "# Stateless MCP routing\n\nMcp-Session-Id is a request header used when session routing is required. A self-describing request can otherwise land on any gateway instance.\n" + }, + { + "path": "canonical/conflict.md", + "content": "# Canonical conflict\n\nERR_CANONICAL_CONFLICT_409 means a conditional write lost a canonical state revision race. Reload the current revision before retrying.\n" + }, + { + "path": "tenants/tenant-7f3a.md", + "content": "# Tenant fixture\n\ntenant-7f3a owns this governed record. Indexed retrieval can enforce its partition boundary before ranking.\n" + }, + { + "path": "operations/request-timeout.md", + "content": "# Request timeout\n\nA request timeout stops an upstream provider call after its declared wall-clock budget.\n" + }, + { + "path": "operations/socket-timeout.md", + "content": "# Socket timeout\n\nA socket timeout indicates that the transport did not make progress before its network deadline.\n" + }, + { + "path": "operations/retry-budget.md", + "content": "# Retry budget\n\nRetry limits protect a provider from an unbounded failure loop. The budget is separate from a request timeout.\n" + }, + { + "path": "governance/authorization.md", + "content": "# Authorization\n\nSearch authorization is evaluated before retrieval. Tenant filters and record policy remain part of the governed query boundary.\n" + }, + { + "path": "live/freshness.md", + "content": "# Freshness fixture\n\nThis file begins without the post-index freshness canary.\n" + } + ], + "qualityCases": [ + { + "id": "identifier-admission-receipt", + "group": "exact-literal", + "query": "ADMISSION_RECEIPT_V1", + "relevant": ["runtime/admission.py"] + }, + { + "id": "identifier-canonical-conflict", + "group": "exact-literal", + "query": "ERR_CANONICAL_CONFLICT_409", + "relevant": ["canonical/conflict.md"] + }, + { + "id": "identifier-retry-exhausted", + "group": "exact-literal", + "query": "ERR_RETRY_EXHAUSTED_503", + "relevant": ["runtime/retry.md"] + }, + { + "id": "identifier-tenant", + "group": "exact-literal", + "query": "tenant-7f3a", + "relevant": ["tenants/tenant-7f3a.md"] + }, + { + "id": "header-mcp-session", + "group": "exact-literal", + "query": "mcp-session-id", + "relevant": ["mcp/routing.md"] + }, + { + "id": "phrase-source-native", + "group": "exact-literal", + "query": "source-native fixed-string search", + "relevant": ["retrieval/source-native.md"] + }, + { + "id": "phrase-accepted-work", + "group": "exact-literal", + "query": "accepted work survives restart", + "relevant": ["runtime/admission.py"] + }, + { + "id": "phrase-injunction", + "group": "exact-literal", + "query": "preliminary injunction deadline", + "relevant": ["retrieval/prefix.md"] + }, + { + "id": "terms-reordered-admission", + "group": "term-retrieval", + "query": "restart accepted work", + "relevant": ["runtime/admission.py"] + }, + { + "id": "terms-separated-receipt", + "group": "term-retrieval", + "query": "receipt worker commit", + "relevant": ["runtime/admission.py"] + }, + { + "id": "natural-recovery", + "group": "term-retrieval", + "query": "queued work resume after restart", + "relevant": ["runtime/recovery.md"] + }, + { + "id": "natural-mcp-routing", + "group": "term-retrieval", + "query": "header routes MCP session", + "relevant": ["mcp/routing.md"] + }, + { + "id": "prefix-injunction", + "group": "prefix", + "query": "prelim injunc deadl", + "relevant": ["retrieval/prefix.md"] + }, + { + "id": "prefix-canonical-conflict", + "group": "prefix", + "query": "canon confl", + "relevant": ["canonical/conflict.md"] + }, + { + "id": "ambiguous-timeout", + "group": "ambiguous", + "query": "timeout", + "relevant": [ + "operations/request-timeout.md", + "operations/socket-timeout.md" + ] + }, + { + "id": "ambiguous-retry", + "group": "ambiguous", + "query": "retry", + "relevant": [ + "operations/retry-budget.md", + "runtime/retry.md" + ] + }, + { + "id": "negative-missing-identifier", + "group": "negative", + "query": "ERR_UNICORN_TRANSPORT_9917", + "relevant": [] + }, + { + "id": "negative-missing-phrase", + "group": "negative", + "query": "violet telescope orchard", + "relevant": [] + } + ] +} diff --git a/runtimes/python/src/vyral_runtime/integrations/ripgrep.py b/runtimes/python/src/vyral_runtime/integrations/ripgrep.py index 75fb4b4..995ec0b 100644 --- a/runtimes/python/src/vyral_runtime/integrations/ripgrep.py +++ b/runtimes/python/src/vyral_runtime/integrations/ripgrep.py @@ -238,7 +238,16 @@ def search( f"ripgrep exited without a usable result (code {return_code})" ) - raw_matches = _parse_match_events(output) + raw_matches = tuple( + sorted( + _parse_match_events(output), + key=lambda event: ( + event["path"], + event["line_number"], + event["byte_column"], + ), + ) + ) revisions: dict[str, str] = {} matches: list[RipgrepSourceMatch] = [] filtered_sensitive_paths = 0 diff --git a/runtimes/python/tests/test_ripgrep_integration.py b/runtimes/python/tests/test_ripgrep_integration.py index 957bdf4..edc9b82 100644 --- a/runtimes/python/tests/test_ripgrep_integration.py +++ b/runtimes/python/tests/test_ripgrep_integration.py @@ -76,6 +76,7 @@ def test_request_limit_and_case_policy_are_enforced(self) -> None: insensitive = adapter.search(RipgrepSearchRequest("needle", limit=1)) self.assertEqual(1, len(insensitive.matches)) + self.assertEqual("a.md", insensitive.matches[0].relative_path) self.assertTrue(insensitive.truncated) sensitive = adapter.search( diff --git a/scripts/benchmark-ripgrep-retrieval.py b/scripts/benchmark-ripgrep-retrieval.py new file mode 100755 index 0000000..39b5276 --- /dev/null +++ b/scripts/benchmark-ripgrep-retrieval.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +"""Compare bounded ripgrep search with Vyral's local lexical record index.""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import sqlite3 +import statistics +import subprocess +import sys +import tempfile +from time import perf_counter +from typing import Any, Callable, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parent.parent +PYTHON_RUNTIME = ROOT / "runtimes/python/src" +FIXTURE_PATH = ROOT / "benchmarks/retrieval/fixtures/source-native-v1.json" +COLLECTION = "source-native-comparison" +PARTITION = "fixture" +TOP_K = 5 +FRESHNESS_CANARY = "FRESHNESS_CANARY_7D91A3" +SAFETY_CANARY = "CREDENTIAL_CANARY_4E82B1" + +sys.path.insert(0, str(PYTHON_RUNTIME)) + +from vyral_runtime import ( # noqa: E402 + RecordCollectionPolicy, + SQLiteRecordStore, + VyralRecord, +) +from vyral_runtime.integrations.ripgrep import ( # noqa: E402 + RipgrepAdapterOptions, + RipgrepSearchAdapter, + RipgrepSearchRequest, +) + + +JSONObject = dict[str, Any] + + +def _timer(operation: Callable[[], Any]) -> tuple[Any, float]: + started = perf_counter() + value = operation() + return value, (perf_counter() - started) * 1000 + + +def _percentile(values: Sequence[float], fraction: float) -> float: + if not values: + raise ValueError("A percentile requires at least one measurement.") + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _latency_summary(values: Sequence[float]) -> JSONObject: + return { + "count": len(values), + "min": round(min(values), 3), + "p50": round(_percentile(values, 0.50), 3), + "p95": round(_percentile(values, 0.95), 3), + "max": round(max(values), 3), + "mean": round(statistics.fmean(values), 3), + } + + +def _load_fixture() -> JSONObject: + value = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + if value.get("schemaVersion") != "vyral.retrieval.source-native-fixture.v1": + raise RuntimeError("The source-native fixture has an unsupported schema version.") + if not isinstance(value.get("documents"), list) or not isinstance( + value.get("qualityCases"), list + ): + raise RuntimeError("The source-native fixture is incomplete.") + return value + + +def _write_corpus(root: Path, fixture: Mapping[str, Any], noise_count: int) -> list[str]: + paths: list[str] = [] + for document in fixture["documents"]: + relative = str(document["path"]) + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(str(document["content"]), encoding="utf-8") + paths.append(relative) + + randomizer = random.Random(20260811) + vocabulary = ( + "adapter", "artifact", "boundary", "checkpoint", "collection", "contract", + "dispatch", "evidence", "fixture", "gateway", "manifest", "provider", + "qualification", "record", "revision", "routing", "runtime", "snapshot", + "storage", "workflow", + ) + for index in range(noise_count): + words = randomizer.sample(vocabulary, 8) + relative = f"generated/noise-{index:05d}.md" + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "# Generated fixture\n\n" + f"Noise document {index:05d}: {' '.join(words)}. " + f"Unique marker NOISE_{index:05d}.\n", + encoding="utf-8", + ) + paths.append(relative) + + (root / ".env").write_text(f"TOKEN={SAFETY_CANARY}\n", encoding="utf-8") + (root / "secrets.md").write_text( + f"This sensitive fixture contains {SAFETY_CANARY}.\n", + encoding="utf-8", + ) + return paths + + +def _record(relative: str, corpus_root: Path) -> VyralRecord: + text = (corpus_root / relative).read_text(encoding="utf-8") + return VyralRecord( + id="source-" + hashlib.sha256(relative.encode("utf-8")).hexdigest()[:24], + partition_key=PARTITION, + type="source.document", + metadata={"path": relative, "extension": Path(relative).suffix}, + content={"text": text}, + sources=( + { + "id": relative, + "kind": "source-file", + "uri": f"vyral-source://fixture/{relative}", + "label": relative, + "span": None, + }, + ), + ) + + +def _quality_metrics(returned: Sequence[str], relevant: Sequence[str], k: int) -> JSONObject: + selected = list(dict.fromkeys(returned))[:k] + expected = set(relevant) + found = [path for path in selected if path in expected] + if expected: + reciprocal_rank = next( + (1.0 / (index + 1) for index, path in enumerate(selected) if path in expected), + 0.0, + ) + recall = len(set(found)) / len(expected) + else: + reciprocal_rank = None + recall = 1.0 if not selected else 0.0 + precision = len(found) / len(selected) if selected else (1.0 if not expected else 0.0) + return { + "returned": selected, + "relevant": list(relevant), + "hit": bool(found) if expected else not selected, + "recallAtK": round(recall, 6), + "precisionReturned": round(precision, 6), + "reciprocalRank": ( + round(reciprocal_rank, 6) if reciprocal_rank is not None else None + ), + "falsePositiveCount": len([path for path in selected if path not in expected]), + "exactSet": set(selected) == expected, + } + + +def _aggregate_quality(cases: Sequence[Mapping[str, Any]]) -> JSONObject: + positive = [case for case in cases if case["metrics"]["relevant"]] + return { + "caseCount": len(cases), + "positiveCaseCount": len(positive), + "meanRecallAtK": round( + statistics.fmean(case["metrics"]["recallAtK"] for case in cases), 6 + ), + "meanPrecisionReturned": round( + statistics.fmean( + case["metrics"]["precisionReturned"] for case in cases + ), + 6, + ), + "meanReciprocalRank": ( + round( + statistics.fmean( + case["metrics"]["reciprocalRank"] for case in positive + ), + 6, + ) + if positive + else None + ), + "exactSetRate": round( + statistics.fmean(1.0 if case["metrics"]["exactSet"] else 0.0 for case in cases), + 6, + ), + } + + +def _ripgrep_paths(adapter: RipgrepSearchAdapter, query: str) -> list[str]: + result = adapter.search(RipgrepSearchRequest(query=query, limit=TOP_K)) + return list(dict.fromkeys(match.relative_path for match in result.matches)) + + +def _vyral_paths( + store: SQLiteRecordStore, + query: str, + *, + match_mode: str, + prefix_matching: bool, +) -> list[str]: + matches = store.search_records( + COLLECTION, + { + "lexical": { + "query": query, + "fields": ["/content/text", "/metadata/path"], + "top": TOP_K, + "matchMode": match_mode, + "prefixMatching": prefix_matching, + "prefixMinChars": 3, + }, + "limit": TOP_K, + }, + ) + paths: list[str] = [] + for match in matches: + path = match.record.metadata.get("path") + if not isinstance(path, str): + raise RuntimeError("A benchmark record is missing its source path.") + paths.append(path) + return paths + + +def _variant_operations( + adapter: RipgrepSearchAdapter, + store: SQLiteRecordStore, +) -> dict[str, Callable[[str], list[str]]]: + return { + "ripgrep-fixed": lambda query: _ripgrep_paths(adapter, query), + "vyral-lexical-any": lambda query: _vyral_paths( + store, query, match_mode="any", prefix_matching=False + ), + "vyral-lexical-all": lambda query: _vyral_paths( + store, query, match_mode="all", prefix_matching=False + ), + "vyral-lexical-prefix": lambda query: _vyral_paths( + store, query, match_mode="all", prefix_matching=True + ), + } + + +def _run_quality( + fixture: Mapping[str, Any], + operations: Mapping[str, Callable[[str], list[str]]], +) -> JSONObject: + results: dict[str, list[JSONObject]] = {name: [] for name in operations} + grouped: dict[str, dict[str, list[JSONObject]]] = { + name: defaultdict(list) for name in operations + } + for raw_case in fixture["qualityCases"]: + case_id = str(raw_case["id"]) + group = str(raw_case["group"]) + query = str(raw_case["query"]) + relevant = [str(value) for value in raw_case["relevant"]] + for name, operation in operations.items(): + returned = operation(query) + result = { + "id": case_id, + "group": group, + "query": query, + "metrics": _quality_metrics(returned, relevant, TOP_K), + } + results[name].append(result) + grouped[name][group].append(result) + return { + "topK": TOP_K, + "variants": { + name: { + "aggregate": _aggregate_quality(cases), + "groups": { + group: _aggregate_quality(group_cases) + for group, group_cases in sorted(grouped[name].items()) + }, + "cases": cases, + } + for name, cases in results.items() + }, + } + + +def _run_latency( + fixture: Mapping[str, Any], + operations: Mapping[str, Callable[[str], list[str]]], + iterations: int, +) -> JSONObject: + queries = [str(case["query"]) for case in fixture["qualityCases"]] + positive_queries = [ + str(case["query"]) for case in fixture["qualityCases"] if case["relevant"] + ] + summaries: dict[str, JSONObject] = {} + for name, operation in operations.items(): + cold_query = positive_queries[0] + _, cold_ms = _timer(lambda: operation(cold_query)) + values: list[float] = [] + for _ in range(iterations): + for query in queries: + _, duration = _timer(lambda query=query: operation(query)) + values.append(duration) + summaries[name] = { + "coldFirstQueryMs": round(cold_ms, 3), + "warmAllCasesMs": _latency_summary(values), + } + return { + "iterations": iterations, + "queriesPerIteration": len(queries), + "variants": summaries, + } + + +def _run_freshness( + corpus_root: Path, + adapter: RipgrepSearchAdapter, + store: SQLiteRecordStore, +) -> JSONObject: + relative = "live/freshness.md" + path = corpus_root / relative + before_ripgrep = _ripgrep_paths(adapter, FRESHNESS_CANARY) + before_vyral = _vyral_paths( + store, FRESHNESS_CANARY, match_mode="all", prefix_matching=False + ) + path.write_text( + path.read_text(encoding="utf-8") + + f"\nThe committed source now contains {FRESHNESS_CANARY}.\n", + encoding="utf-8", + ) + rg_after, rg_after_ms = _timer(lambda: _ripgrep_paths(adapter, FRESHNESS_CANARY)) + stale_vyral, stale_vyral_ms = _timer( + lambda: _vyral_paths( + store, FRESHNESS_CANARY, match_mode="all", prefix_matching=False + ) + ) + _, refresh_ms = _timer(lambda: store.upsert_record(COLLECTION, _record(relative, corpus_root))) + refreshed_vyral, refreshed_vyral_ms = _timer( + lambda: _vyral_paths( + store, FRESHNESS_CANARY, match_mode="all", prefix_matching=False + ) + ) + return { + "path": relative, + "absentBeforeEdit": not before_ripgrep and not before_vyral, + "ripgrepVisibleWithoutIndexRefresh": relative in rg_after, + "ripgrepWriteToResultMs": round(rg_after_ms, 3), + "vyralStaleBeforeRecordRefresh": relative not in stale_vyral, + "vyralStaleQueryMs": round(stale_vyral_ms, 3), + "vyralRecordRefreshMs": round(refresh_ms, 3), + "vyralVisibleAfterRecordRefresh": relative in refreshed_vyral, + "vyralPostRefreshQueryMs": round(refreshed_vyral_ms, 3), + } + + +def _run_safety(adapter: RipgrepSearchAdapter, store: SQLiteRecordStore) -> JSONObject: + result = adapter.search(RipgrepSearchRequest(SAFETY_CANARY, limit=TOP_K)) + citation_sample = adapter.search( + RipgrepSearchRequest("ADMISSION_RECEIPT_V1", limit=TOP_K) + ) + vyral_paths = _vyral_paths( + store, SAFETY_CANARY, match_mode="all", prefix_matching=False + ) + serialized = json.dumps(result.to_dict(), sort_keys=True) + return { + "ripgrepReturnedPaths": [match.relative_path for match in result.matches], + "ripgrepFilteredSensitivePaths": result.filtered_sensitive_paths, + "ripgrepSensitiveCanaryExcluded": not result.matches, + "vyralSensitiveCanaryExcluded": not vyral_paths, + "absoluteRootDisclosed": str(adapter.root_path) in serialized, + "citationsRevisionBound": all( + match.source_uri.startswith("vyral-source://ripgrep/") + and "#L" in match.source_uri + and match.source_revision.startswith("sha256:") + and match.line_number > 0 + for match in citation_sample.matches + ), + "citationSampleCount": len(citation_sample.matches), + } + + +def _admission(report: Mapping[str, Any]) -> JSONObject: + variants = report["quality"]["variants"] + exact = variants["ripgrep-fixed"]["groups"]["exact-literal"] + terms_rg = variants["ripgrep-fixed"]["groups"]["term-retrieval"] + terms_vyral = variants["vyral-lexical-all"]["groups"]["term-retrieval"] + prefix_rg = variants["ripgrep-fixed"]["groups"]["prefix"] + prefix_vyral = variants["vyral-lexical-prefix"]["groups"]["prefix"] + latency = report["latencyMs"]["variants"]["ripgrep-fixed"]["warmAllCasesMs"] + freshness = report["freshness"] + safety = report["safety"] + criteria = { + "sourceTreeClean": not report["sourceDirty"], + "exactLiteralRecallIsPerfect": exact["meanRecallAtK"] == 1.0, + "exactLiteralPrecisionIsPerfect": exact["meanPrecisionReturned"] == 1.0, + "exactLiteralFirstRelevantIsPerfect": exact["meanReciprocalRank"] == 1.0, + "localP95AtOrBelow100Ms": latency["p95"] <= 100.0, + "editVisibleWithoutIndexRefresh": freshness["ripgrepVisibleWithoutIndexRefresh"], + "indexedRecordIsStaleUntilRefresh": freshness["vyralStaleBeforeRecordRefresh"], + "sensitiveCanaryExcluded": ( + safety["ripgrepSensitiveCanaryExcluded"] + and safety["vyralSensitiveCanaryExcluded"] + ), + "absoluteRootNotDisclosed": not safety["absoluteRootDisclosed"], + "revisionBoundLineCitations": ( + safety["citationSampleCount"] > 0 and safety["citationsRevisionBound"] + ), + "vyralWinsReorderedTerms": ( + terms_vyral["meanRecallAtK"] > terms_rg["meanRecallAtK"] + ), + "vyralWinsPrefixQueries": ( + prefix_vyral["meanRecallAtK"] > prefix_rg["meanRecallAtK"] + ), + } + admitted = all(criteria.values()) + return { + "decision": "admit" if admitted else "reject", + "criteria": criteria, + "identifiedUseCase": ( + "Bounded, read-only, zero-index lookup of exact literals in an authorized " + "local code or Markdown tree, with revision-bound line citations and " + "immediate visibility of source edits." + ), + "userPath": ( + "Construct RipgrepSearchAdapter once with an application-owned root and " + "static globs, then call search with a fixed-string query and bounded limit." + ), + "notFor": [ + "semantic or paraphrase retrieval", + "governed record filtering or tenant authorization", + "remote or non-text corpora", + "regex or caller-selected filesystem traversal", + ], + } + + +def _git_commit() -> str: + completed = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + check=True, + stdout=subprocess.PIPE, + text=True, + ) + return completed.stdout.strip() + + +def _git_dirty() -> bool: + completed = subprocess.run( + [ + "git", + "-C", + str(ROOT), + "status", + "--porcelain", + "--untracked-files=all", + ], + check=True, + stdout=subprocess.PIPE, + text=True, + ) + return bool(completed.stdout.strip()) + + +def _environment(adapter: RipgrepSearchAdapter) -> JSONObject: + return { + "platform": platform.system().lower(), + "architecture": platform.machine().lower(), + "logicalCpuCount": os.cpu_count(), + "pythonVersion": platform.python_version(), + "sqliteVersion": sqlite3.sqlite_version, + "ripgrepVersion": adapter.executable_version, + } + + +def run_benchmark(noise_documents: int, iterations: int) -> JSONObject: + fixture = _load_fixture() + with tempfile.TemporaryDirectory(prefix="vyral-ripgrep-comparison-") as temporary: + root = Path(temporary) + corpus_root = root / "corpus" + corpus_root.mkdir() + paths, corpus_ms = _timer(lambda: _write_corpus(corpus_root, fixture, noise_documents)) + + adapter, adapter_ms = _timer( + lambda: RipgrepSearchAdapter( + corpus_root, + RipgrepAdapterOptions( + include_globs=("*.py", "*.md", ".env"), + max_results=TOP_K, + timeout_seconds=10.0, + ), + ) + ) + store, store_ms = _timer(lambda: SQLiteRecordStore(root / "vyral.sqlite")) + store.create_collection( + RecordCollectionPolicy( + name=COLLECTION, + indexed_metadata=("/metadata/path", "/metadata/extension"), + ) + ) + ingest_result, ingest_ms = _timer( + lambda: store.upsert_records( + COLLECTION, + [_record(relative, corpus_root) for relative in paths], + ) + ) + if ingest_result.failed != 0 or ingest_result.succeeded != len(paths): + raise RuntimeError("The Vyral comparison corpus did not ingest completely.") + operations = _variant_operations(adapter, store) + report: JSONObject = { + "schemaVersion": "vyral.retrieval.ripgrep-comparison.v1", + "sourceCommit": _git_commit(), + "sourceDirty": _git_dirty(), + "generatedAtUtc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "environment": _environment(adapter), + "parameters": { + "fixture": FIXTURE_PATH.relative_to(ROOT).as_posix(), + "fixtureSha256": hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest(), + "authoredDocuments": len(fixture["documents"]), + "generatedNoiseDocuments": noise_documents, + "totalIndexedDocuments": len(paths), + "iterations": iterations, + "topK": TOP_K, + "randomSeed": 20260811, + }, + "setupMs": { + "writeCorpus": round(corpus_ms, 3), + "ripgrepAdapterInitialize": round(adapter_ms, 3), + "vyralStoreInitialize": round(store_ms, 3), + "vyralRecordIngest": round(ingest_ms, 3), + }, + "storageBytes": { + "sourceCorpus": sum((corpus_root / relative).stat().st_size for relative in paths), + "vyralDatabase": store.diagnostics().database_bytes, + }, + "quality": _run_quality(fixture, operations), + "latencyMs": _run_latency(fixture, operations, iterations), + "freshness": _run_freshness(corpus_root, adapter, store), + "safety": _run_safety(adapter, store), + } + report["admission"] = _admission(report) + return report + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--noise-documents", type=int, default=2_000) + parser.add_argument("--iterations", type=int, default=30) + parser.add_argument("--require-admission", action="store_true") + arguments = parser.parse_args() + if arguments.noise_documents < 0: + parser.error("--noise-documents must be non-negative") + if arguments.iterations <= 0: + parser.error("--iterations must be greater than zero") + return arguments + + +def main() -> int: + arguments = _arguments() + report = run_benchmark(arguments.noise_documents, arguments.iterations) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + admission = report["admission"] + quality = report["quality"]["variants"] + print( + "ripgrep-retrieval-comparison=" + f"{admission['decision']} " + f"documents={report['parameters']['totalIndexedDocuments']} " + f"rg-exact-recall={quality['ripgrep-fixed']['groups']['exact-literal']['meanRecallAtK']} " + f"vyral-term-recall={quality['vyral-lexical-all']['groups']['term-retrieval']['meanRecallAtK']} " + f"report={arguments.output}" + ) + if arguments.require_admission and admission["decision"] != "admit": + failed = [name for name, passed in admission["criteria"].items() if not passed] + print("ripgrep admission criteria failed: " + ", ".join(failed), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-ripgrep-retrieval-report.py b/scripts/verify-ripgrep-retrieval-report.py new file mode 100755 index 0000000..9d8f4e1 --- /dev/null +++ b/scripts/verify-ripgrep-retrieval-report.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Validate a retained Vyral-versus-ripgrep comparison receipt.""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys +from typing import Any, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parent.parent +COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +def _object(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be an object") + return value + + +def _number(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a number") + return float(value) + + +def _integer(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _boolean(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _variant(report: Mapping[str, Any], name: str) -> Mapping[str, Any]: + quality = _object(report.get("quality"), "quality") + variants = _object(quality.get("variants"), "quality.variants") + return _object(variants.get(name), f"quality.variants.{name}") + + +def _group( + report: Mapping[str, Any], variant: str, group: str +) -> Mapping[str, Any]: + groups = _object(_variant(report, variant).get("groups"), f"{variant}.groups") + return _object(groups.get(group), f"{variant}.groups.{group}") + + +def _recomputed_aggregate(cases: Sequence[Mapping[str, Any]]) -> dict[str, float | int | None]: + if not cases: + raise ValueError("A quality aggregate must contain cases") + metrics = [_object(case.get("metrics"), "case.metrics") for case in cases] + positive = [metric for metric in metrics if metric.get("relevant")] + return { + "caseCount": len(cases), + "positiveCaseCount": len(positive), + "meanRecallAtK": round( + sum(_number(metric.get("recallAtK"), "recallAtK") for metric in metrics) + / len(metrics), + 6, + ), + "meanPrecisionReturned": round( + sum( + _number(metric.get("precisionReturned"), "precisionReturned") + for metric in metrics + ) + / len(metrics), + 6, + ), + "meanReciprocalRank": ( + round( + sum( + _number(metric.get("reciprocalRank"), "reciprocalRank") + for metric in positive + ) + / len(positive), + 6, + ) + if positive + else None + ), + "exactSetRate": round( + sum(1.0 if _boolean(metric.get("exactSet"), "exactSet") else 0.0 for metric in metrics) + / len(metrics), + 6, + ), + } + + +def _string_array(value: Any, name: str) -> list[str]: + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ValueError(f"{name} must be an array of strings") + return value + + +def _recomputed_metrics(metrics: Mapping[str, Any], top_k: int) -> dict[str, Any]: + returned = list(dict.fromkeys(_string_array(metrics.get("returned"), "returned")))[:top_k] + relevant = _string_array(metrics.get("relevant"), "relevant") + expected = set(relevant) + found = [path for path in returned if path in expected] + if expected: + reciprocal_rank = next( + (1.0 / (index + 1) for index, path in enumerate(returned) if path in expected), + 0.0, + ) + recall = len(set(found)) / len(expected) + else: + reciprocal_rank = None + recall = 1.0 if not returned else 0.0 + precision = len(found) / len(returned) if returned else (1.0 if not expected else 0.0) + return { + "returned": returned, + "relevant": relevant, + "hit": bool(found) if expected else not returned, + "recallAtK": round(recall, 6), + "precisionReturned": round(precision, 6), + "reciprocalRank": ( + round(reciprocal_rank, 6) if reciprocal_rank is not None else None + ), + "falsePositiveCount": len([path for path in returned if path not in expected]), + "exactSet": set(returned) == expected, + } + + +def _verify_quality(report: Mapping[str, Any]) -> None: + quality = _object(report.get("quality"), "quality") + top_k = _integer(quality.get("topK"), "quality.topK") + fixture = json.loads( + (ROOT / "benchmarks/retrieval/fixtures/source-native-v1.json").read_text( + encoding="utf-8" + ) + ) + fixture_cases = { + case["id"]: { + "group": case["group"], + "query": case["query"], + "relevant": case["relevant"], + } + for case in fixture["qualityCases"] + } + for name in ( + "ripgrep-fixed", + "vyral-lexical-any", + "vyral-lexical-all", + "vyral-lexical-prefix", + ): + variant = _variant(report, name) + raw_cases = variant.get("cases") + if not isinstance(raw_cases, list): + raise ValueError(f"{name}.cases must be an array") + cases = [_object(case, f"{name}.case") for case in raw_cases] + if len(cases) != len(fixture_cases): + raise ValueError(f"{name} does not contain the complete labeled query set") + seen: set[str] = set() + for case in cases: + case_id = case.get("id") + if not isinstance(case_id, str) or case_id in seen or case_id not in fixture_cases: + raise ValueError(f"{name} contains an invalid or duplicate case id") + seen.add(case_id) + expected_case = fixture_cases[case_id] + if ( + case.get("group") != expected_case["group"] + or case.get("query") != expected_case["query"] + ): + raise ValueError(f"{name}.{case_id} does not match the labeled fixture") + metrics = _object(case.get("metrics"), f"{name}.{case_id}.metrics") + if metrics.get("relevant") != expected_case["relevant"]: + raise ValueError(f"{name}.{case_id} changes the relevance labels") + if dict(metrics) != _recomputed_metrics(metrics, top_k): + raise ValueError(f"{name}.{case_id} metrics do not match its results") + aggregate = _object(variant.get("aggregate"), f"{name}.aggregate") + if dict(aggregate) != _recomputed_aggregate(cases): + raise ValueError(f"{name} aggregate does not match its cases") + by_group: dict[str, list[Mapping[str, Any]]] = {} + for case in cases: + group = case.get("group") + if not isinstance(group, str) or not group: + raise ValueError(f"{name} case has an invalid group") + by_group.setdefault(group, []).append(case) + groups = _object(variant.get("groups"), f"{name}.groups") + if set(groups) != set(by_group): + raise ValueError(f"{name} group set does not match its cases") + for group, group_cases in by_group.items(): + if dict(_object(groups[group], f"{name}.{group}")) != _recomputed_aggregate( + group_cases + ): + raise ValueError(f"{name}.{group} does not match its cases") + + +def _expected_criteria(report: Mapping[str, Any]) -> dict[str, bool]: + exact = _group(report, "ripgrep-fixed", "exact-literal") + terms_rg = _group(report, "ripgrep-fixed", "term-retrieval") + terms_vyral = _group(report, "vyral-lexical-all", "term-retrieval") + prefix_rg = _group(report, "ripgrep-fixed", "prefix") + prefix_vyral = _group(report, "vyral-lexical-prefix", "prefix") + latency = _object( + _object( + _object(report.get("latencyMs"), "latencyMs").get("variants"), + "latencyMs.variants", + ).get("ripgrep-fixed"), + "latencyMs.variants.ripgrep-fixed", + ) + warm_latency = _object(latency.get("warmAllCasesMs"), "ripgrep warm latency") + freshness = _object(report.get("freshness"), "freshness") + safety = _object(report.get("safety"), "safety") + return { + "sourceTreeClean": not _boolean(report.get("sourceDirty"), "sourceDirty"), + "exactLiteralRecallIsPerfect": _number( + exact.get("meanRecallAtK"), "exact meanRecallAtK" + ) + == 1.0, + "exactLiteralPrecisionIsPerfect": _number( + exact.get("meanPrecisionReturned"), "exact meanPrecisionReturned" + ) + == 1.0, + "exactLiteralFirstRelevantIsPerfect": _number( + exact.get("meanReciprocalRank"), "exact meanReciprocalRank" + ) + == 1.0, + "localP95AtOrBelow100Ms": _number(warm_latency.get("p95"), "ripgrep p95") + <= 100.0, + "editVisibleWithoutIndexRefresh": _boolean( + freshness.get("ripgrepVisibleWithoutIndexRefresh"), + "ripgrep freshness", + ), + "indexedRecordIsStaleUntilRefresh": _boolean( + freshness.get("vyralStaleBeforeRecordRefresh"), + "Vyral stale-before-refresh", + ), + "sensitiveCanaryExcluded": _boolean( + safety.get("ripgrepSensitiveCanaryExcluded"), "ripgrep sensitive exclusion" + ) + and _boolean( + safety.get("vyralSensitiveCanaryExcluded"), "Vyral sensitive exclusion" + ), + "absoluteRootNotDisclosed": not _boolean( + safety.get("absoluteRootDisclosed"), "absoluteRootDisclosed" + ), + "revisionBoundLineCitations": ( + _integer(safety.get("citationSampleCount"), "citationSampleCount") > 0 + and _boolean(safety.get("citationsRevisionBound"), "citationsRevisionBound") + ), + "vyralWinsReorderedTerms": _number( + terms_vyral.get("meanRecallAtK"), "Vyral term recall" + ) + > _number(terms_rg.get("meanRecallAtK"), "ripgrep term recall"), + "vyralWinsPrefixQueries": _number( + prefix_vyral.get("meanRecallAtK"), "Vyral prefix recall" + ) + > _number(prefix_rg.get("meanRecallAtK"), "ripgrep prefix recall"), + } + + +def _verify_source_commit(value: Any) -> None: + if not isinstance(value, str) or not COMMIT_PATTERN.fullmatch(value): + raise ValueError("sourceCommit must be a full Git commit") + completed = subprocess.run( + ["git", "-C", str(ROOT), "merge-base", "--is-ancestor", value, "HEAD"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if completed.returncode != 0: + raise ValueError("sourceCommit is not an ancestor of the current checkout") + + +def _scan_private_paths(value: Any) -> None: + if isinstance(value, str): + if any(prefix in value for prefix in ("/home/", "/projects/", "C:\\Users\\")): + raise ValueError("report contains a host-specific absolute path") + elif isinstance(value, Mapping): + for key, child in value.items(): + _scan_private_paths(key) + _scan_private_paths(child) + elif isinstance(value, list): + for child in value: + _scan_private_paths(child) + + +def verify( + report: Mapping[str, Any], + *, + minimum_noise: int, + minimum_iterations: int, + require_admission: bool, +) -> None: + if report.get("schemaVersion") != "vyral.retrieval.ripgrep-comparison.v1": + raise ValueError("report schemaVersion is unsupported") + _verify_source_commit(report.get("sourceCommit")) + generated = report.get("generatedAtUtc") + if not isinstance(generated, str): + raise ValueError("generatedAtUtc must be a string") + datetime.fromisoformat(generated.replace("Z", "+00:00")) + parameters = _object(report.get("parameters"), "parameters") + fixture_path = parameters.get("fixture") + if fixture_path != "benchmarks/retrieval/fixtures/source-native-v1.json": + raise ValueError("report references an unexpected fixture") + fixture_sha256 = parameters.get("fixtureSha256") + expected_fixture_sha256 = hashlib.sha256( + (ROOT / fixture_path).read_bytes() + ).hexdigest() + if fixture_sha256 != expected_fixture_sha256: + raise ValueError("report fixture digest does not match the current fixture") + if _integer(parameters.get("generatedNoiseDocuments"), "generatedNoiseDocuments") < minimum_noise: + raise ValueError("report does not meet the minimum noise-document count") + if _integer(parameters.get("iterations"), "iterations") < minimum_iterations: + raise ValueError("report does not meet the minimum iteration count") + if _integer(parameters.get("authoredDocuments"), "authoredDocuments") < 16: + raise ValueError("report does not contain the full authored corpus") + if _integer(parameters.get("topK"), "topK") != 5: + raise ValueError("report topK must remain 5") + _verify_quality(report) + expected_criteria = _expected_criteria(report) + admission = _object(report.get("admission"), "admission") + criteria = _object(admission.get("criteria"), "admission.criteria") + if dict(criteria) != expected_criteria: + raise ValueError("admission criteria do not match the report evidence") + expected_decision = "admit" if all(expected_criteria.values()) else "reject" + if admission.get("decision") != expected_decision: + raise ValueError("admission decision does not match its criteria") + if require_admission and expected_decision != "admit": + failed = [name for name, passed in expected_criteria.items() if not passed] + raise ValueError("adapter admission failed: " + ", ".join(failed)) + _scan_private_paths(report) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("--minimum-noise", type=int, default=2_000) + parser.add_argument("--minimum-iterations", type=int, default=30) + parser.add_argument("--require-admission", action="store_true") + return parser.parse_args() + + +def main() -> int: + arguments = _arguments() + try: + raw = json.loads(arguments.report.read_text(encoding="utf-8")) + report = _object(raw, "report") + verify( + report, + minimum_noise=arguments.minimum_noise, + minimum_iterations=arguments.minimum_iterations, + require_admission=arguments.require_admission, + ) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as error: + print(f"ripgrep retrieval report invalid: {error}", file=sys.stderr) + return 1 + print( + "ripgrep-retrieval-report=ok " + f"decision={report['admission']['decision']} " + f"documents={report['parameters']['totalIndexedDocuments']} " + f"report={arguments.report}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2e9eb67ebb82c070ddac2abf97b243a54e69789c Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:34:38 -0400 Subject: [PATCH 4/8] docs(retrieval): retain ripgrep comparison evidence --- .github/workflows/ci.yml | 3 + PUBLIC-EXPORT-MANIFEST.json | 28 +- README.md | 2 +- benchmarks/retrieval/README.md | 76 + .../ripgrep-vs-vyral-local-2026-08-11.json | 1711 +++++++++++++++++ docs/guides/source-native-retrieval.md | 22 + examples/README.md | 9 +- runtimes/python/README.md | 9 +- scripts/export-public-tree.py | 5 + scripts/verify-release-artifacts.sh | 3 + 10 files changed, 1853 insertions(+), 15 deletions(-) create mode 100644 benchmarks/retrieval/README.md create mode 100644 benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280c7ca..e7757d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,9 @@ jobs: python3 scripts/test-audit-github-launch-controls.py python3 scripts/test-export-public-tree.py python3 scripts/test-python-runtime-platform-matrix.py + python3 scripts/verify-ripgrep-retrieval-report.py \ + benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json \ + --require-admission while IFS= read -r -d '' script; do bash -n "$script"; done < <(find scripts deploy -type f -name '*.sh' -print0) find conformance contracts qualification -type f -name '*.json' -print0 | xargs -0 jq empty diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 9baad70..a54a436 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -1,5 +1,5 @@ { - "fileCount": 835, + "fileCount": 837, "files": [ { "mode": "644", @@ -54,7 +54,7 @@ { "mode": "644", "path": ".github/workflows/ci.yml", - "sha256": "246b11dbc507bb36eec6491b5ae4569c574faa5400379c7222470b5fb4657818" + "sha256": "3d47c7a4fc46186d4ed091a4f6a89c3c9f0b368afe458a43554bc247043da8e8" }, { "mode": "644", @@ -134,7 +134,7 @@ { "mode": "644", "path": "README.md", - "sha256": "2a6443fe2d34ae2d31c36ca706db9c8d55719fff8e3c9fbc48f8567eaffa546b" + "sha256": "4181ac731be532f77e8044780902d57f925da784e169d147094ccc4a3311840e" }, { "mode": "644", @@ -171,11 +171,21 @@ "path": "benchmarks/canonical-store/mysql-8.0.46-local.json", "sha256": "9032bf64760c76820016af95192c1a7b60f38fad7380e1e8ef3bc88c35bcbb81" }, + { + "mode": "644", + "path": "benchmarks/retrieval/README.md", + "sha256": "2eb3d5e8bdb8518fa3ddadf29b4deb6d29d1fb465f1172a63c0648cc46669b94" + }, { "mode": "644", "path": "benchmarks/retrieval/fixtures/source-native-v1.json", "sha256": "abf12894b9720f59872a63f2e596bfa5b0fa268b1ebbd53a239561c5055618bb" }, + { + "mode": "644", + "path": "benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json", + "sha256": "20fd30ca51b43ca7ac8a36a95be3dff346fd94af39c3d6fdc57ad1d0c2c4c273" + }, { "mode": "644", "path": "clients/go/README.md", @@ -594,7 +604,7 @@ { "mode": "644", "path": "docs/guides/source-native-retrieval.md", - "sha256": "e8cf1dfcf5bd815d2f320262d9854d8083a197eedbe8a91bfaf5505b02299ce9" + "sha256": "3bc7161ed4b1d83ee3a8461968bf7bc681ca49a38b4535ec81763629ce0e941e" }, { "mode": "644", @@ -624,7 +634,7 @@ { "mode": "644", "path": "examples/README.md", - "sha256": "53ec5f86eb7e5deb8eeac8b12cbb365962b53ac4ec1c54bf094cfe42d8162bc4" + "sha256": "6ebe3db180f7c8c18bfa35fb433168cc6e4e8523848b39e0c20b15d7b570ef01" }, { "mode": "644", @@ -689,7 +699,7 @@ { "mode": "644", "path": "runtimes/python/README.md", - "sha256": "e1f79f657e531c49211524fa8405b21700f6f586753352af68cdd91e78e4785f" + "sha256": "f2ae828afaf0af80241b1acfc38f2f536d02d217fe129a6c08190707fe536b20" }, { "mode": "644", @@ -1474,7 +1484,7 @@ { "mode": "755", "path": "scripts/export-public-tree.py", - "sha256": "4c51919eac13813eec8679e8b17c7580776ea306ce67d8d05fcdfae14cd097b6" + "sha256": "3854825cf1d2c38c4a87633479d6b1a68f4209644430b78dcc0f487f70fc0ff5" }, { "mode": "755", @@ -1819,7 +1829,7 @@ { "mode": "755", "path": "scripts/verify-release-artifacts.sh", - "sha256": "37656bb728e094285b4915f11044fea227cc1c3c34d1497ad8cbde425090d0a6" + "sha256": "85c0f2f93ab818261d4c6d583eda7d6c0570dc13177c54bd2b0c5abae324c7a5" }, { "mode": "755", @@ -4179,5 +4189,5 @@ ], "schemaVersion": 1, "sourceDirty": true, - "treeSha256": "cf37ea61c1d4998534ae5ad689342d0e66ce3b6e0bd8a6cb7476c01b437e60ec" + "treeSha256": "fb75e25a335be7dd10e5ae4095ec6b8220a860a03cfbd4ebc4b1004eb76051da" } diff --git a/README.md b/README.md index ca51070..a7ff3a9 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ compatibility policy. | .NET local server, SQLite, retrieval, and RAG | Primary supported implementation | | Python and JavaScript HTTP clients | Implemented and tested against the public SDK catalog | | Python-native embedded runtime and REST/MCP host | Prototype; implemented and clean-artifact qualified on Python 3.10–3.12 across Linux, macOS, and Windows; independent security review remains | -| Source-native ripgrep integration | Experimental; bounded, read-only Python integration outside the stable wire/MCP contract | +| Source-native ripgrep integration | Experimental; retained local evidence supports bounded exact-literal search over authorized code/Markdown without a duplicate index; outside the stable wire/MCP contract | | Stateless MCP `2026-07-28` endpoint | Preview; opt-in and REST-backed | | Local SQLite and Azure Durable execution | Preview; locally conformant | | AWS, Google, Temporal, pgvector, and Cloudflare adapters | Preview or prototype; live qualification is opt-in | diff --git a/benchmarks/retrieval/README.md b/benchmarks/retrieval/README.md new file mode 100644 index 0000000..879ad8d --- /dev/null +++ b/benchmarks/retrieval/README.md @@ -0,0 +1,76 @@ +# Source-native retrieval comparison + +Vyral retains a local comparison between the experimental ripgrep adapter and +the Python runtime's SQLite FTS5 lexical retrieval. The purpose is narrow: prove +whether direct source search has a user-facing role that is not already served +better by an indexed Vyral collection. + +The retained receipt uses 16 labeled documents, 2,000 deterministic noise +documents, 18 queries, top 5 retrieval, and 30 timed iterations per query. It +compares fixed-string ripgrep with Vyral lexical `any`, `all`, and prefix +policies. The source tree was clean and the report identifies its exact fixture, +fixture digest, source commit, runtime versions, and machine shape. + +## Result + +The adapter is retained as an experimental integration for one identifiable +case: bounded exact-literal lookup in an authorized, rapidly changing local +code or Markdown tree when copying that tree into a retrieval index is not +justified. + +| Query group | ripgrep fixed recall@5 | Vyral lexical policy | Vyral recall@5 | +| --- | ---: | --- | ---: | +| Exact literals, 8 cases | 1.00 | `all` | 1.00 | +| Reordered or separated terms, 4 cases | 0.00 | `all` | 0.75 | +| Prefixes, 2 cases | 0.00 | `all` plus prefix matching | 1.00 | +| Ambiguous single terms, 2 cases | 1.00 | `all` | 1.00 | +| Queries expected to return nothing, 2 cases | 1.00 | `all` | 1.00 | + +On this machine, ripgrep's warm p50/p95 was 16.734/33.398 ms. Indexed Vyral +lexical `all` was 1.172/1.508 ms, and the prefix policy was 1.292/1.738 ms. +Vyral is decisively faster after indexing. The default lexical `any` policy had +a 291.765 ms p95 because common fixture terms produced broad candidate sets; +its term-retrieval recall was 1.00, but returned precision was 0.425. + +The tradeoff appears before the first query. The ripgrep adapter initialized in +2.668 ms and used the existing source tree directly. Mirroring 2,016 documents +into the local Vyral store took 29,158.337 ms and produced a 13,324,288-byte +database for a 288,995-byte fixture corpus. These ratios are fixture-specific; +the small generated documents emphasize per-record overhead and should not be +generalized to larger records. + +After a source edit, ripgrep returned the new canary without an index refresh +in 16.589 ms. Vyral correctly remained stale until the changed record was +upserted, then returned it in 1.851 ms. Sensitive-path canaries were excluded, +the absolute root was not disclosed, and the adapter emitted line citations +bound to source SHA-256 revisions. + +The evidence does not support using ripgrep for semantic or paraphrase +retrieval, prefix discovery, governed record filtering, tenant authorization, +remote corpora, or non-text sources. Those remain Vyral indexed retrieval or +adapter concerns. + +Results are from one Linux x86-64 machine and establish this bounded local use +case, not a universal performance ranking. See the +[full retained receipt](ripgrep-vs-vyral-local-2026-08-11.json) for every query, +returned path, metric, timing, and admission criterion. + +## Reproduce + +Run the full comparison from a clean checkout: + +```bash +python3 scripts/benchmark-ripgrep-retrieval.py \ + --output /tmp/ripgrep-vs-vyral.json \ + --noise-documents 2000 \ + --iterations 30 \ + --require-admission + +python3 scripts/verify-ripgrep-retrieval-report.py \ + /tmp/ripgrep-vs-vyral.json \ + --require-admission +``` + +CI also runs a smaller comparison and proves that its verifier rejects altered +quality metrics. A future result that fails any admission criterion changes the +decision to `reject`; the experimental adapter must then be revised or removed. diff --git a/benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json b/benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json new file mode 100644 index 0000000..3244dcd --- /dev/null +++ b/benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json @@ -0,0 +1,1711 @@ +{ + "admission": { + "criteria": { + "absoluteRootNotDisclosed": true, + "editVisibleWithoutIndexRefresh": true, + "exactLiteralFirstRelevantIsPerfect": true, + "exactLiteralPrecisionIsPerfect": true, + "exactLiteralRecallIsPerfect": true, + "indexedRecordIsStaleUntilRefresh": true, + "localP95AtOrBelow100Ms": true, + "revisionBoundLineCitations": true, + "sensitiveCanaryExcluded": true, + "sourceTreeClean": true, + "vyralWinsPrefixQueries": true, + "vyralWinsReorderedTerms": true + }, + "decision": "admit", + "identifiedUseCase": "Bounded, read-only, zero-index lookup of exact literals in an authorized local code or Markdown tree, with revision-bound line citations and immediate visibility of source edits.", + "notFor": [ + "semantic or paraphrase retrieval", + "governed record filtering or tenant authorization", + "remote or non-text corpora", + "regex or caller-selected filesystem traversal" + ], + "userPath": "Construct RipgrepSearchAdapter once with an application-owned root and static globs, then call search with a fixed-string query and bounded limit." + }, + "environment": { + "architecture": "x86_64", + "logicalCpuCount": 16, + "platform": "linux", + "pythonVersion": "3.12.3", + "ripgrepVersion": "ripgrep 15.2.0 (rev e89fff89ac)", + "sqliteVersion": "3.45.1" + }, + "freshness": { + "absentBeforeEdit": true, + "path": "live/freshness.md", + "ripgrepVisibleWithoutIndexRefresh": true, + "ripgrepWriteToResultMs": 16.589, + "vyralPostRefreshQueryMs": 1.851, + "vyralRecordRefreshMs": 17.377, + "vyralStaleBeforeRecordRefresh": true, + "vyralStaleQueryMs": 1.452, + "vyralVisibleAfterRecordRefresh": true + }, + "generatedAtUtc": "2026-08-11T08:31:53.999398Z", + "latencyMs": { + "iterations": 30, + "queriesPerIteration": 18, + "variants": { + "ripgrep-fixed": { + "coldFirstQueryMs": 17.152, + "warmAllCasesMs": { + "count": 540, + "max": 34.845, + "mean": 19.579, + "min": 16.212, + "p50": 16.734, + "p95": 33.398 + } + }, + "vyral-lexical-all": { + "coldFirstQueryMs": 1.591, + "warmAllCasesMs": { + "count": 540, + "max": 1.84, + "mean": 1.147, + "min": 0.699, + "p50": 1.172, + "p95": 1.508 + } + }, + "vyral-lexical-any": { + "coldFirstQueryMs": 2.231, + "warmAllCasesMs": { + "count": 540, + "max": 429.971, + "mean": 19.426, + "min": 0.701, + "p50": 1.492, + "p95": 291.765 + } + }, + "vyral-lexical-prefix": { + "coldFirstQueryMs": 1.117, + "warmAllCasesMs": { + "count": 540, + "max": 2.296, + "mean": 1.289, + "min": 0.744, + "p50": 1.292, + "p95": 1.738 + } + } + } + }, + "parameters": { + "authoredDocuments": 16, + "fixture": "benchmarks/retrieval/fixtures/source-native-v1.json", + "fixtureSha256": "abf12894b9720f59872a63f2e596bfa5b0fa268b1ebbd53a239561c5055618bb", + "generatedNoiseDocuments": 2000, + "iterations": 30, + "randomSeed": 20260811, + "topK": 5, + "totalIndexedDocuments": 2016 + }, + "quality": { + "topK": 5, + "variants": { + "ripgrep-fixed": { + "aggregate": { + "caseCount": 18, + "exactSetRate": 0.555556, + "meanPrecisionReturned": 0.62963, + "meanRecallAtK": 0.666667, + "meanReciprocalRank": 0.59375, + "positiveCaseCount": 16 + }, + "cases": [ + { + "group": "exact-literal", + "id": "identifier-admission-receipt", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "ADMISSION_RECEIPT_V1" + }, + { + "group": "exact-literal", + "id": "identifier-canonical-conflict", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [ + "canonical/conflict.md" + ] + }, + "query": "ERR_CANONICAL_CONFLICT_409" + }, + { + "group": "exact-literal", + "id": "identifier-retry-exhausted", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/retry.md" + ], + "returned": [ + "runtime/retry.md" + ] + }, + "query": "ERR_RETRY_EXHAUSTED_503" + }, + { + "group": "exact-literal", + "id": "identifier-tenant", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "tenants/tenant-7f3a.md" + ], + "returned": [ + "tenants/tenant-7f3a.md" + ] + }, + "query": "tenant-7f3a" + }, + { + "group": "exact-literal", + "id": "header-mcp-session", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [ + "mcp/routing.md" + ] + }, + "query": "mcp-session-id" + }, + { + "group": "exact-literal", + "id": "phrase-source-native", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/source-native.md" + ], + "returned": [ + "retrieval/source-native.md" + ] + }, + "query": "source-native fixed-string search" + }, + { + "group": "exact-literal", + "id": "phrase-accepted-work", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "accepted work survives restart" + }, + { + "group": "exact-literal", + "id": "phrase-injunction", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [ + "retrieval/prefix.md" + ] + }, + "query": "preliminary injunction deadline" + }, + { + "group": "term-retrieval", + "id": "terms-reordered-admission", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [] + }, + "query": "restart accepted work" + }, + { + "group": "term-retrieval", + "id": "terms-separated-receipt", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [] + }, + "query": "receipt worker commit" + }, + { + "group": "term-retrieval", + "id": "natural-recovery", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "runtime/recovery.md" + ], + "returned": [] + }, + "query": "queued work resume after restart" + }, + { + "group": "term-retrieval", + "id": "natural-mcp-routing", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [] + }, + "query": "header routes MCP session" + }, + { + "group": "prefix", + "id": "prefix-injunction", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [] + }, + "query": "prelim injunc deadl" + }, + { + "group": "prefix", + "id": "prefix-canonical-conflict", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [] + }, + "query": "canon confl" + }, + { + "group": "ambiguous", + "id": "ambiguous-timeout", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/request-timeout.md", + "operations/socket-timeout.md" + ], + "returned": [ + "operations/request-timeout.md", + "operations/retry-budget.md", + "operations/socket-timeout.md" + ] + }, + "query": "timeout" + }, + { + "group": "ambiguous", + "id": "ambiguous-retry", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 0.5, + "relevant": [ + "operations/retry-budget.md", + "runtime/retry.md" + ], + "returned": [ + "canonical/conflict.md", + "operations/retry-budget.md", + "runtime/retry.md" + ] + }, + "query": "retry" + }, + { + "group": "negative", + "id": "negative-missing-identifier", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "ERR_UNICORN_TRANSPORT_9917" + }, + { + "group": "negative", + "id": "negative-missing-phrase", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "violet telescope orchard" + } + ], + "groups": { + "ambiguous": { + "caseCount": 2, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.666667, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 0.75, + "positiveCaseCount": 2 + }, + "exact-literal": { + "caseCount": 8, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 8 + }, + "negative": { + "caseCount": 2, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": null, + "positiveCaseCount": 0 + }, + "prefix": { + "caseCount": 2, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.0, + "meanRecallAtK": 0.0, + "meanReciprocalRank": 0.0, + "positiveCaseCount": 2 + }, + "term-retrieval": { + "caseCount": 4, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.0, + "meanRecallAtK": 0.0, + "meanReciprocalRank": 0.0, + "positiveCaseCount": 4 + } + } + }, + "vyral-lexical-all": { + "aggregate": { + "caseCount": 18, + "exactSetRate": 0.777778, + "meanPrecisionReturned": 0.814815, + "meanRecallAtK": 0.833333, + "meanReciprocalRank": 0.8125, + "positiveCaseCount": 16 + }, + "cases": [ + { + "group": "exact-literal", + "id": "identifier-admission-receipt", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "ADMISSION_RECEIPT_V1" + }, + { + "group": "exact-literal", + "id": "identifier-canonical-conflict", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [ + "canonical/conflict.md" + ] + }, + "query": "ERR_CANONICAL_CONFLICT_409" + }, + { + "group": "exact-literal", + "id": "identifier-retry-exhausted", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/retry.md" + ], + "returned": [ + "runtime/retry.md" + ] + }, + "query": "ERR_RETRY_EXHAUSTED_503" + }, + { + "group": "exact-literal", + "id": "identifier-tenant", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "tenants/tenant-7f3a.md" + ], + "returned": [ + "tenants/tenant-7f3a.md" + ] + }, + "query": "tenant-7f3a" + }, + { + "group": "exact-literal", + "id": "header-mcp-session", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [ + "mcp/routing.md" + ] + }, + "query": "mcp-session-id" + }, + { + "group": "exact-literal", + "id": "phrase-source-native", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/source-native.md" + ], + "returned": [ + "retrieval/source-native.md" + ] + }, + "query": "source-native fixed-string search" + }, + { + "group": "exact-literal", + "id": "phrase-accepted-work", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "accepted work survives restart" + }, + { + "group": "exact-literal", + "id": "phrase-injunction", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [ + "retrieval/prefix.md" + ] + }, + "query": "preliminary injunction deadline" + }, + { + "group": "term-retrieval", + "id": "terms-reordered-admission", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "restart accepted work" + }, + { + "group": "term-retrieval", + "id": "terms-separated-receipt", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "receipt worker commit" + }, + { + "group": "term-retrieval", + "id": "natural-recovery", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/recovery.md" + ], + "returned": [ + "runtime/recovery.md" + ] + }, + "query": "queued work resume after restart" + }, + { + "group": "term-retrieval", + "id": "natural-mcp-routing", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [] + }, + "query": "header routes MCP session" + }, + { + "group": "prefix", + "id": "prefix-injunction", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [] + }, + "query": "prelim injunc deadl" + }, + { + "group": "prefix", + "id": "prefix-canonical-conflict", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [] + }, + "query": "canon confl" + }, + { + "group": "ambiguous", + "id": "ambiguous-timeout", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/request-timeout.md", + "operations/socket-timeout.md" + ], + "returned": [ + "operations/request-timeout.md", + "operations/socket-timeout.md", + "operations/retry-budget.md" + ] + }, + "query": "timeout" + }, + { + "group": "ambiguous", + "id": "ambiguous-retry", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/retry-budget.md", + "runtime/retry.md" + ], + "returned": [ + "operations/retry-budget.md", + "runtime/retry.md" + ] + }, + "query": "retry" + }, + { + "group": "negative", + "id": "negative-missing-identifier", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "ERR_UNICORN_TRANSPORT_9917" + }, + { + "group": "negative", + "id": "negative-missing-phrase", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "violet telescope orchard" + } + ], + "groups": { + "ambiguous": { + "caseCount": 2, + "exactSetRate": 0.5, + "meanPrecisionReturned": 0.833333, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 2 + }, + "exact-literal": { + "caseCount": 8, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 8 + }, + "negative": { + "caseCount": 2, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": null, + "positiveCaseCount": 0 + }, + "prefix": { + "caseCount": 2, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.0, + "meanRecallAtK": 0.0, + "meanReciprocalRank": 0.0, + "positiveCaseCount": 2 + }, + "term-retrieval": { + "caseCount": 4, + "exactSetRate": 0.75, + "meanPrecisionReturned": 0.75, + "meanRecallAtK": 0.75, + "meanReciprocalRank": 0.75, + "positiveCaseCount": 4 + } + } + }, + "vyral-lexical-any": { + "aggregate": { + "caseCount": 18, + "exactSetRate": 0.166667, + "meanPrecisionReturned": 0.427778, + "meanRecallAtK": 0.833333, + "meanReciprocalRank": 0.875, + "positiveCaseCount": 16 + }, + "cases": [ + { + "group": "exact-literal", + "id": "identifier-admission-receipt", + "metrics": { + "exactSet": false, + "falsePositiveCount": 3, + "hit": true, + "precisionReturned": 0.25, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py", + "runtime/recovery.md", + "retrieval/hard-negative.md", + "runtime/retry.md" + ] + }, + "query": "ADMISSION_RECEIPT_V1" + }, + { + "group": "exact-literal", + "id": "identifier-canonical-conflict", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.5, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [ + "canonical/conflict.md", + "runtime/retry.md" + ] + }, + "query": "ERR_CANONICAL_CONFLICT_409" + }, + { + "group": "exact-literal", + "id": "identifier-retry-exhausted", + "metrics": { + "exactSet": false, + "falsePositiveCount": 2, + "hit": true, + "precisionReturned": 0.333333, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/retry.md" + ], + "returned": [ + "runtime/retry.md", + "operations/retry-budget.md", + "canonical/conflict.md" + ] + }, + "query": "ERR_RETRY_EXHAUSTED_503" + }, + { + "group": "exact-literal", + "id": "identifier-tenant", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.5, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "tenants/tenant-7f3a.md" + ], + "returned": [ + "tenants/tenant-7f3a.md", + "governance/authorization.md" + ] + }, + "query": "tenant-7f3a" + }, + { + "group": "exact-literal", + "id": "header-mcp-session", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.5, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [ + "mcp/routing.md", + "runtime/admission.py" + ] + }, + "query": "mcp-session-id" + }, + { + "group": "exact-literal", + "id": "phrase-source-native", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.5, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/source-native.md" + ], + "returned": [ + "retrieval/source-native.md", + "governance/authorization.md" + ] + }, + "query": "source-native fixed-string search" + }, + { + "group": "exact-literal", + "id": "phrase-accepted-work", + "metrics": { + "exactSet": false, + "falsePositiveCount": 3, + "hit": true, + "precisionReturned": 0.25, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py", + "runtime/recovery.md", + "retrieval/hard-negative.md", + "runtime/worker-pool.md" + ] + }, + "query": "accepted work survives restart" + }, + { + "group": "exact-literal", + "id": "phrase-injunction", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.5, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [ + "retrieval/prefix.md", + "operations/socket-timeout.md" + ] + }, + "query": "preliminary injunction deadline" + }, + { + "group": "term-retrieval", + "id": "terms-reordered-admission", + "metrics": { + "exactSet": false, + "falsePositiveCount": 3, + "hit": true, + "precisionReturned": 0.25, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py", + "runtime/recovery.md", + "retrieval/hard-negative.md", + "runtime/worker-pool.md" + ] + }, + "query": "restart accepted work" + }, + { + "group": "term-retrieval", + "id": "terms-separated-receipt", + "metrics": { + "exactSet": false, + "falsePositiveCount": 3, + "hit": true, + "precisionReturned": 0.25, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py", + "runtime/worker-pool.md", + "runtime/recovery.md", + "retrieval/hard-negative.md" + ] + }, + "query": "receipt worker commit" + }, + { + "group": "term-retrieval", + "id": "natural-recovery", + "metrics": { + "exactSet": false, + "falsePositiveCount": 4, + "hit": true, + "precisionReturned": 0.2, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/recovery.md" + ], + "returned": [ + "runtime/recovery.md", + "runtime/admission.py", + "retrieval/hard-negative.md", + "runtime/worker-pool.md", + "operations/request-timeout.md" + ] + }, + "query": "queued work resume after restart" + }, + { + "group": "term-retrieval", + "id": "natural-mcp-routing", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [ + "mcp/routing.md" + ] + }, + "query": "header routes MCP session" + }, + { + "group": "prefix", + "id": "prefix-injunction", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [] + }, + "query": "prelim injunc deadl" + }, + { + "group": "prefix", + "id": "prefix-canonical-conflict", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [] + }, + "query": "canon confl" + }, + { + "group": "ambiguous", + "id": "ambiguous-timeout", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/request-timeout.md", + "operations/socket-timeout.md" + ], + "returned": [ + "operations/request-timeout.md", + "operations/socket-timeout.md", + "operations/retry-budget.md" + ] + }, + "query": "timeout" + }, + { + "group": "ambiguous", + "id": "ambiguous-retry", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/retry-budget.md", + "runtime/retry.md" + ], + "returned": [ + "operations/retry-budget.md", + "runtime/retry.md" + ] + }, + "query": "retry" + }, + { + "group": "negative", + "id": "negative-missing-identifier", + "metrics": { + "exactSet": false, + "falsePositiveCount": 3, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": null, + "relevant": [], + "returned": [ + "operations/socket-timeout.md", + "canonical/conflict.md", + "runtime/retry.md" + ] + }, + "query": "ERR_UNICORN_TRANSPORT_9917" + }, + { + "group": "negative", + "id": "negative-missing-phrase", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "violet telescope orchard" + } + ], + "groups": { + "ambiguous": { + "caseCount": 2, + "exactSetRate": 0.5, + "meanPrecisionReturned": 0.833333, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 2 + }, + "exact-literal": { + "caseCount": 8, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.416667, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 8 + }, + "negative": { + "caseCount": 2, + "exactSetRate": 0.5, + "meanPrecisionReturned": 0.5, + "meanRecallAtK": 0.5, + "meanReciprocalRank": null, + "positiveCaseCount": 0 + }, + "prefix": { + "caseCount": 2, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.0, + "meanRecallAtK": 0.0, + "meanReciprocalRank": 0.0, + "positiveCaseCount": 2 + }, + "term-retrieval": { + "caseCount": 4, + "exactSetRate": 0.25, + "meanPrecisionReturned": 0.425, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 4 + } + } + }, + "vyral-lexical-prefix": { + "aggregate": { + "caseCount": 18, + "exactSetRate": 0.833333, + "meanPrecisionReturned": 0.907407, + "meanRecallAtK": 0.944444, + "meanReciprocalRank": 0.9375, + "positiveCaseCount": 16 + }, + "cases": [ + { + "group": "exact-literal", + "id": "identifier-admission-receipt", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "ADMISSION_RECEIPT_V1" + }, + { + "group": "exact-literal", + "id": "identifier-canonical-conflict", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [ + "canonical/conflict.md" + ] + }, + "query": "ERR_CANONICAL_CONFLICT_409" + }, + { + "group": "exact-literal", + "id": "identifier-retry-exhausted", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/retry.md" + ], + "returned": [ + "runtime/retry.md" + ] + }, + "query": "ERR_RETRY_EXHAUSTED_503" + }, + { + "group": "exact-literal", + "id": "identifier-tenant", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "tenants/tenant-7f3a.md" + ], + "returned": [ + "tenants/tenant-7f3a.md" + ] + }, + "query": "tenant-7f3a" + }, + { + "group": "exact-literal", + "id": "header-mcp-session", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [ + "mcp/routing.md" + ] + }, + "query": "mcp-session-id" + }, + { + "group": "exact-literal", + "id": "phrase-source-native", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/source-native.md" + ], + "returned": [ + "retrieval/source-native.md" + ] + }, + "query": "source-native fixed-string search" + }, + { + "group": "exact-literal", + "id": "phrase-accepted-work", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "accepted work survives restart" + }, + { + "group": "exact-literal", + "id": "phrase-injunction", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [ + "retrieval/prefix.md" + ] + }, + "query": "preliminary injunction deadline" + }, + { + "group": "term-retrieval", + "id": "terms-reordered-admission", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "restart accepted work" + }, + { + "group": "term-retrieval", + "id": "terms-separated-receipt", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/admission.py" + ], + "returned": [ + "runtime/admission.py" + ] + }, + "query": "receipt worker commit" + }, + { + "group": "term-retrieval", + "id": "natural-recovery", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "runtime/recovery.md" + ], + "returned": [ + "runtime/recovery.md" + ] + }, + "query": "queued work resume after restart" + }, + { + "group": "term-retrieval", + "id": "natural-mcp-routing", + "metrics": { + "exactSet": false, + "falsePositiveCount": 0, + "hit": false, + "precisionReturned": 0.0, + "recallAtK": 0.0, + "reciprocalRank": 0.0, + "relevant": [ + "mcp/routing.md" + ], + "returned": [] + }, + "query": "header routes MCP session" + }, + { + "group": "prefix", + "id": "prefix-injunction", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "retrieval/prefix.md" + ], + "returned": [ + "retrieval/prefix.md" + ] + }, + "query": "prelim injunc deadl" + }, + { + "group": "prefix", + "id": "prefix-canonical-conflict", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "canonical/conflict.md" + ], + "returned": [ + "canonical/conflict.md" + ] + }, + "query": "canon confl" + }, + { + "group": "ambiguous", + "id": "ambiguous-timeout", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/request-timeout.md", + "operations/socket-timeout.md" + ], + "returned": [ + "operations/request-timeout.md", + "operations/socket-timeout.md", + "operations/retry-budget.md" + ] + }, + "query": "timeout" + }, + { + "group": "ambiguous", + "id": "ambiguous-retry", + "metrics": { + "exactSet": false, + "falsePositiveCount": 1, + "hit": true, + "precisionReturned": 0.666667, + "recallAtK": 1.0, + "reciprocalRank": 1.0, + "relevant": [ + "operations/retry-budget.md", + "runtime/retry.md" + ], + "returned": [ + "operations/retry-budget.md", + "runtime/retry.md", + "canonical/conflict.md" + ] + }, + "query": "retry" + }, + { + "group": "negative", + "id": "negative-missing-identifier", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "ERR_UNICORN_TRANSPORT_9917" + }, + { + "group": "negative", + "id": "negative-missing-phrase", + "metrics": { + "exactSet": true, + "falsePositiveCount": 0, + "hit": true, + "precisionReturned": 1.0, + "recallAtK": 1.0, + "reciprocalRank": null, + "relevant": [], + "returned": [] + }, + "query": "violet telescope orchard" + } + ], + "groups": { + "ambiguous": { + "caseCount": 2, + "exactSetRate": 0.0, + "meanPrecisionReturned": 0.666667, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 2 + }, + "exact-literal": { + "caseCount": 8, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 8 + }, + "negative": { + "caseCount": 2, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": null, + "positiveCaseCount": 0 + }, + "prefix": { + "caseCount": 2, + "exactSetRate": 1.0, + "meanPrecisionReturned": 1.0, + "meanRecallAtK": 1.0, + "meanReciprocalRank": 1.0, + "positiveCaseCount": 2 + }, + "term-retrieval": { + "caseCount": 4, + "exactSetRate": 0.75, + "meanPrecisionReturned": 0.75, + "meanRecallAtK": 0.75, + "meanReciprocalRank": 0.75, + "positiveCaseCount": 4 + } + } + } + } + }, + "safety": { + "absoluteRootDisclosed": false, + "citationSampleCount": 1, + "citationsRevisionBound": true, + "ripgrepFilteredSensitivePaths": 1, + "ripgrepReturnedPaths": [], + "ripgrepSensitiveCanaryExcluded": true, + "vyralSensitiveCanaryExcluded": true + }, + "schemaVersion": "vyral.retrieval.ripgrep-comparison.v1", + "setupMs": { + "ripgrepAdapterInitialize": 2.668, + "vyralRecordIngest": 29158.337, + "vyralStoreInitialize": 43.954, + "writeCorpus": 205.486 + }, + "sourceCommit": "9931243b07d18f0c959a7182455fdb032cf15904", + "sourceDirty": false, + "storageBytes": { + "sourceCorpus": 288995, + "vyralDatabase": 13324288 + } +} diff --git a/docs/guides/source-native-retrieval.md b/docs/guides/source-native-retrieval.md index 49f7889..dd458ea 100644 --- a/docs/guides/source-native-retrieval.md +++ b/docs/guides/source-native-retrieval.md @@ -83,6 +83,28 @@ The executable is resolved once when the adapter starts and must identify itself as ripgrep. Install `rg` separately and keep its version in evaluation receipts when results need to be reproduced. +### Retained comparison evidence + +A clean-tree local comparison over 2,016 documents and 18 labeled queries +identifies one bounded role for this integration. Ripgrep achieved 1.00 recall, +1.00 returned precision, and 1.00 MRR across eight exact-literal cases, returned +a committed edit without an index refresh, and produced revision-bound line +citations. Its warm p50/p95 was 16.734/33.398 ms. + +Vyral lexical `all` matched the exact-literal quality at a much lower indexed +p50/p95 of 1.172/1.508 ms and beat fixed-string ripgrep on reordered terms. +Vyral prefix retrieval achieved 1.00 recall on the two prefix cases where +ripgrep returned none. Building the local record index took 29.158 seconds; +ripgrep used the authorized source tree directly and initialized in 2.668 ms. + +The adapter therefore remains experimental for exact identifiers, error codes, +headers, and literal phrases in current local code or Markdown when an index +would be duplicate state. Prefer Vyral lexical retrieval once records need +filters, tenant boundaries, stable snapshots, term-order tolerance, prefixes, +or consistently lower query latency. The +[retained report](../../benchmarks/retrieval/README.md) describes the fixture, +limitations, complete metrics, and reproduction command. + ## Corpus comparison recipe Use a labeled fixture that resembles the real corpus. Keep the source revision, diff --git a/examples/README.md b/examples/README.md index a6b803f..b3f524b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -46,8 +46,9 @@ These scripts create isolated example collections by default. Override `VYRAL_CO ## Source-native retrieval -For a local code or Markdown corpus, test the bounded experimental ripgrep -integration before creating an index: +For an authorized local code or Markdown tree, use the bounded experimental +ripgrep integration to find exact identifiers, error codes, headers, or literal +phrases without first copying the tree into an index: ```bash python3 examples/python/source_native_search.py \ @@ -59,7 +60,9 @@ The example searches only allowlisted `*.py` and `*.md` paths by default and prints root-relative line citations with source revisions. It does not expose a REST or MCP tool. See the [source-native retrieval guide](../docs/guides/source-native-retrieval.md) for -the safety boundary and comparison recipe. +the safety boundary and retained comparison evidence. Use indexed Vyral +retrieval when queries need reordered terms, prefixes, record filters, tenant +boundaries, or consistently lower post-index latency. ## Execution Runtime Sample diff --git a/runtimes/python/README.md b/runtimes/python/README.md index 919632d..5808c7a 100644 --- a/runtimes/python/README.md +++ b/runtimes/python/README.md @@ -205,8 +205,13 @@ Python function must not silently create a different operation. for code, Markdown, and other safely accessible text sources. It uses a static root and glob allowlist, fixed-string queries over standard input, source revision citations, and strict resource limits. It is not part of the stable -wire contract and is not automatically exposed through REST or MCP. See the -[source-native retrieval guide](../../docs/guides/source-native-retrieval.md). +wire contract and is not automatically exposed through REST or MCP. Retained +comparison evidence supports it for exact identifiers and phrases in current +local sources when maintaining a duplicate index is not worthwhile. Use Vyral +lexical retrieval for reordered terms, prefixes, record filters, tenant +boundaries, snapshots, and lower post-index query latency. See the +[source-native retrieval guide](../../docs/guides/source-native-retrieval.md) +and [comparison receipt](../../benchmarks/retrieval/README.md). ### Experimental Extropic execution diff --git a/scripts/export-public-tree.py b/scripts/export-public-tree.py index afc7670..353b7a3 100755 --- a/scripts/export-public-tree.py +++ b/scripts/export-public-tree.py @@ -129,6 +129,9 @@ ".gitattributes", ".github/workflows/ci.yml", ".github/workflows/python-runtime-qualification.yml", + "benchmarks/retrieval/README.md", + "benchmarks/retrieval/fixtures/source-native-v1.json", + "benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json", "conformance/invariant.md", "deploy/canonical-cutover/cloudbuild-mysql.yaml", "deploy/canonical-cutover/cloudbuild-postgres.yaml", @@ -155,6 +158,7 @@ "runtimes/python/pyproject.toml", "runtimes/python/LICENSE", "scripts/benchmark-python-runtime.py", + "scripts/benchmark-ripgrep-retrieval.py", "scripts/audit-github-launch-controls.py", "scripts/export-public-tree.py", "scripts/mcp-wire-proxy.py", @@ -176,6 +180,7 @@ "scripts/verify-oci-image-identity.py", "scripts/verify-public-export.sh", "scripts/verify-publication-policy.py", + "scripts/verify-ripgrep-retrieval-report.py", "scripts/write-python-runtime-platform-receipt.py", } diff --git a/scripts/verify-release-artifacts.sh b/scripts/verify-release-artifacts.sh index 9c88ab5..a8a855f 100755 --- a/scripts/verify-release-artifacts.sh +++ b/scripts/verify-release-artifacts.sh @@ -110,6 +110,9 @@ scripts/test-validate-google-execution-live.sh scripts/test-validate-aws-opensearch-live.sh scripts/test-scan-release-history.sh scripts/test-validate-canonical-mysql-benchmark-report.sh +python3 scripts/verify-ripgrep-retrieval-report.py \ + benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json \ + --require-admission if [[ -n "${VYRAL_PUBLIC_HISTORY_DENYLIST_FILE:-}" || -n "${VYRAL_PUBLIC_HISTORY_DENYLIST:-}" ]]; then scripts/scan-release-history.sh From 0e738ac87070a9d86d6ef0d2f1feec0c1205fb55 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:37:55 -0400 Subject: [PATCH 5/8] test(retrieval): exercise the user-facing source search --- .github/workflows/ci.yml | 15 +++++++++++++++ PUBLIC-EXPORT-MANIFEST.json | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7757d1..0bcdb92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,21 @@ jobs: echo "The ripgrep comparison verifier accepted altered metrics." >&2 exit 1 fi + user_result="$(mktemp)" + python3 examples/python/source_native_search.py \ + RipgrepSearchAdapter \ + --root . \ + --limit 5 \ + --json > "$user_result" + jq -e ' + (.matches | length > 0) + and all( + .matches[]; + (.sourceUri | startswith("vyral-source://ripgrep/")) + and (.sourceUri | contains("#L")) + and (.sourceRevision | startswith("sha256:")) + ) + ' "$user_result" >/dev/null - name: Test Go worker client working-directory: clients/go diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index a54a436..13932ec 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -54,7 +54,7 @@ { "mode": "644", "path": ".github/workflows/ci.yml", - "sha256": "3d47c7a4fc46186d4ed091a4f6a89c3c9f0b368afe458a43554bc247043da8e8" + "sha256": "e17a1e95ed15e26be2fc1afdb0192c0fb8f7c9f3a649914b07066b038eba687e" }, { "mode": "644", @@ -4189,5 +4189,5 @@ ], "schemaVersion": 1, "sourceDirty": true, - "treeSha256": "fb75e25a335be7dd10e5ae4095ec6b8220a860a03cfbd4ebc4b1004eb76051da" + "treeSha256": "1ca53fa5cd63c8344929484e31a9bb096a5696c89f45ab8d59335ec3d5760b18" } From cf6679326ffbdd23b26063cd54f15915d986d19c Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:38:34 -0400 Subject: [PATCH 6/8] chore(release): seal retrieval evidence export --- PUBLIC-EXPORT-MANIFEST.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 13932ec..0a394af 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -4188,6 +4188,6 @@ } ], "schemaVersion": 1, - "sourceDirty": true, + "sourceDirty": false, "treeSha256": "1ca53fa5cd63c8344929484e31a9bb096a5696c89f45ab8d59335ec3d5760b18" } From 34ba39f1dcc3c44dc3f4f2340386587903568d64 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:42:10 -0400 Subject: [PATCH 7/8] test(retrieval): run comparison with release dependencies --- .github/workflows/ci.yml | 39 ----------------------------- scripts/verify-release-artifacts.sh | 38 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bcdb92..8f3a0e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,45 +95,6 @@ jobs: python -m mypy --config-file clients/python/pyproject.toml clients/python/tests/typecheck_consumer.py scripts/verify-python-runtime.sh - - name: Verify the source-native retrieval comparison - run: | - report="$(mktemp)" - tampered_report="$(mktemp)" - python3 scripts/benchmark-ripgrep-retrieval.py \ - --output "$report" \ - --noise-documents 60 \ - --iterations 3 \ - --require-admission - python3 scripts/verify-ripgrep-retrieval-report.py \ - "$report" \ - --minimum-noise 60 \ - --minimum-iterations 3 \ - --require-admission - jq '.quality.variants["ripgrep-fixed"].cases[0].metrics.recallAtK = 0' \ - "$report" > "$tampered_report" - if python3 scripts/verify-ripgrep-retrieval-report.py \ - "$tampered_report" \ - --minimum-noise 60 \ - --minimum-iterations 3; then - echo "The ripgrep comparison verifier accepted altered metrics." >&2 - exit 1 - fi - user_result="$(mktemp)" - python3 examples/python/source_native_search.py \ - RipgrepSearchAdapter \ - --root . \ - --limit 5 \ - --json > "$user_result" - jq -e ' - (.matches | length > 0) - and all( - .matches[]; - (.sourceUri | startswith("vyral-source://ripgrep/")) - and (.sourceUri | contains("#L")) - and (.sourceRevision | startswith("sha256:")) - ) - ' "$user_result" >/dev/null - - name: Test Go worker client working-directory: clients/go run: go test ./... diff --git a/scripts/verify-release-artifacts.sh b/scripts/verify-release-artifacts.sh index a8a855f..bfb5a82 100755 --- a/scripts/verify-release-artifacts.sh +++ b/scripts/verify-release-artifacts.sh @@ -114,6 +114,44 @@ python3 scripts/verify-ripgrep-retrieval-report.py \ benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json \ --require-admission +ripgrep_comparison="$ARTIFACT_ROOT/ripgrep-comparison.json" +ripgrep_tampered="$ARTIFACT_ROOT/ripgrep-comparison-tampered.json" +ripgrep_user_result="$ARTIFACT_ROOT/ripgrep-user-path.json" +python3 scripts/benchmark-ripgrep-retrieval.py \ + --output "$ripgrep_comparison" \ + --noise-documents 60 \ + --iterations 3 \ + --require-admission +python3 scripts/verify-ripgrep-retrieval-report.py \ + "$ripgrep_comparison" \ + --minimum-noise 60 \ + --minimum-iterations 3 \ + --require-admission +jq '.quality.variants["ripgrep-fixed"].cases[0].metrics.recallAtK = 0' \ + "$ripgrep_comparison" > "$ripgrep_tampered" +if python3 scripts/verify-ripgrep-retrieval-report.py \ + "$ripgrep_tampered" \ + --minimum-noise 60 \ + --minimum-iterations 3; then + echo "The ripgrep comparison verifier accepted altered metrics." >&2 + exit 1 +fi +python3 examples/python/source_native_search.py \ + RipgrepSearchAdapter \ + --root . \ + --limit 5 \ + --json > "$ripgrep_user_result" +jq -e ' + (.matches | length > 0) + and all( + .matches[]; + (.sourceUri | startswith("vyral-source://ripgrep/")) + and (.sourceUri | contains("#L")) + and (.sourceRevision | startswith("sha256:")) + ) +' "$ripgrep_user_result" >/dev/null +printf 'ripgrep-retrieval-admission-gate=ok\n' + if [[ -n "${VYRAL_PUBLIC_HISTORY_DENYLIST_FILE:-}" || -n "${VYRAL_PUBLIC_HISTORY_DENYLIST:-}" ]]; then scripts/scan-release-history.sh fi From 0ed109b4f13830a945d85f66714a81e4ab2cfafe Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Tue, 11 Aug 2026 04:42:18 -0400 Subject: [PATCH 8/8] chore(release): refresh comparison gate evidence --- PUBLIC-EXPORT-MANIFEST.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index 0a394af..12ffddd 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -54,7 +54,7 @@ { "mode": "644", "path": ".github/workflows/ci.yml", - "sha256": "e17a1e95ed15e26be2fc1afdb0192c0fb8f7c9f3a649914b07066b038eba687e" + "sha256": "f2ee6e2e92d2cd36fe5fc65928be57e2c77620c062c75ce7e867320937a66036" }, { "mode": "644", @@ -1829,7 +1829,7 @@ { "mode": "755", "path": "scripts/verify-release-artifacts.sh", - "sha256": "85c0f2f93ab818261d4c6d583eda7d6c0570dc13177c54bd2b0c5abae324c7a5" + "sha256": "b84d3aa20a74f49ebce0f34b18c79099335dea9f6c9808eb9182febfc65cc92e" }, { "mode": "755", @@ -4189,5 +4189,5 @@ ], "schemaVersion": 1, "sourceDirty": false, - "treeSha256": "1ca53fa5cd63c8344929484e31a9bb096a5696c89f45ab8d59335ec3d5760b18" + "treeSha256": "e6b7959543b600c6eed8e33b420b1cecd373519e2a212af9f81f7817bc9f5058" }