test(retrieval): add lexical controls and ripgrep comparator

.github/workflows/ci.yml

@@ -41,6 +41,9 @@ jobs:
       - name: Run deterministic tests
         run: dotnet test Vyral.sln --configuration Release --no-build --logger 'console;verbosity=minimal'
 
+      - name: Verify zero-embedding HTTP quickstarts
+        run: scripts/verify-lexical-http-quickstarts.sh
+
       - name: Prove core contract TFMs
         run: |
           dotnet build src/Vyral.Abstractions/Vyral.Abstractions.csproj --configuration Release --framework net8.0 --no-restore
@@ -126,6 +129,9 @@ jobs:
           python3 scripts/test-audit-github-launch-controls.py
           python3 scripts/test-export-public-tree.py
           python3 scripts/test-python-runtime-platform-matrix.py
+          python3 scripts/verify-ripgrep-retrieval-report.py \
+            benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json \
+            --require-admission
           while IFS= read -r -d '' script; do bash -n "$script"; done < <(find scripts deploy -type f -name '*.sh' -print0)
           find conformance contracts qualification -type f -name '*.json' -print0 | xargs -0 jq empty
 

.gitignore

@@ -44,6 +44,7 @@ docs/contributing/*
 docs/guides/*
 !docs/guides/consumer-handoff.md
 !docs/guides/extropic-execution.md
+!docs/guides/source-native-retrieval.md
 !docs/maintainers/
 docs/maintainers/*
 !docs/maintainers/releasing.md

PUBLIC-EXPORT-MANIFEST.json

@@ -1,5 +1,5 @@
 {
-  "fileCount": 827,
+  "fileCount": 837,
   "files": [
     {
       "mode": "644",
@@ -54,7 +54,7 @@
     {
       "mode": "644",
       "path": ".github/workflows/ci.yml",
-      "sha256": "80d2b432d425187dda1b08e377db5befec12ed5ac1b52876d9374de87246e6cd"
+      "sha256": "f2ee6e2e92d2cd36fe5fc65928be57e2c77620c062c75ce7e867320937a66036"
     },
     {
       "mode": "644",
@@ -104,7 +104,7 @@
     {
       "mode": "644",
       "path": ".gitignore",
-      "sha256": "57ddda23948159577073baa5dbea4082b33c22f975b9b5669c65ef8810eaec3b"
+      "sha256": "7b15c20e8f94c53e6c50a55dc511928718af35a1cd0e3c9104c10d1ad43ef969"
     },
     {
       "mode": "644",
@@ -134,7 +134,7 @@
     {
       "mode": "644",
       "path": "README.md",
-      "sha256": "77fcc61d84373d0cf29e1b9935105c7b104cb3f24f3cd3d6771a5e19606f3bfe"
+      "sha256": "4181ac731be532f77e8044780902d57f925da784e169d147094ccc4a3311840e"
     },
     {
       "mode": "644",
@@ -171,6 +171,21 @@
       "path": "benchmarks/canonical-store/mysql-8.0.46-local.json",
       "sha256": "9032bf64760c76820016af95192c1a7b60f38fad7380e1e8ef3bc88c35bcbb81"
     },
+    {
+      "mode": "644",
+      "path": "benchmarks/retrieval/README.md",
+      "sha256": "2eb3d5e8bdb8518fa3ddadf29b4deb6d29d1fb465f1172a63c0648cc46669b94"
+    },
+    {
+      "mode": "644",
+      "path": "benchmarks/retrieval/fixtures/source-native-v1.json",
+      "sha256": "abf12894b9720f59872a63f2e596bfa5b0fa268b1ebbd53a239561c5055618bb"
+    },
+    {
+      "mode": "644",
+      "path": "benchmarks/retrieval/ripgrep-vs-vyral-local-2026-08-11.json",
+      "sha256": "20fd30ca51b43ca7ac8a36a95be3dff346fd94af39c3d6fdc57ad1d0c2c4c273"
+    },
     {
       "mode": "644",
       "path": "clients/go/README.md",
@@ -559,7 +574,7 @@
     {
       "mode": "644",
       "path": "docs/README.md",
-      "sha256": "a4a4f2479130c950c0856f0b0d9d0e6858f26d42cedeaa46192eaf489617e8e3"
+      "sha256": "2a846342f0e616eaefcc46ba817ab7ab73b9f7611be92b85e7407e70ef974d46"
     },
     {
       "mode": "644",
@@ -579,13 +594,18 @@
     {
       "mode": "644",
       "path": "docs/guides/consumer-handoff.md",
-      "sha256": "fb19dee1acb9d02f3f50fa79e4c17ec9313c90a76e4f547ebd88bb576f69ca84"
+      "sha256": "2de27f52d6ebc717ebe74a422fb9c028233aaa0c04ca7e8bbfdf71b3b596868c"
     },
     {
       "mode": "644",
       "path": "docs/guides/extropic-execution.md",
       "sha256": "09e9c18f2f54f28da367906c197abbd192699d686387a147fdf114feff51f656"
     },
+    {
+      "mode": "644",
+      "path": "docs/guides/source-native-retrieval.md",
+      "sha256": "3bc7161ed4b1d83ee3a8461968bf7bc681ca49a38b4535ec81763629ce0e941e"
+    },
     {
       "mode": "644",
       "path": "docs/maintainers/releasing.md",
@@ -614,7 +634,7 @@
     {
       "mode": "644",
       "path": "examples/README.md",
-      "sha256": "2163d0dd9b99a47e3f9dda60d4afc13cc6e80724dd7bfa3f2e2678afeeda94ae"
+      "sha256": "6ebe3db180f7c8c18bfa35fb433168cc6e4e8523848b39e0c20b15d7b570ef01"
     },
     {
       "mode": "644",
@@ -624,7 +644,7 @@
     {
       "mode": "644",
       "path": "examples/javascript/rag-quickstart.mjs",
-      "sha256": "3c603757a807432edc647d3945222b2098e235d0b4af6fe4854238ecf1a7711f"
+      "sha256": "a2f6dab248bdc892a8beaecec9cd0935fddb1bfe782f4c07503b192c2ee2fe00"
     },
     {
       "mode": "644",
@@ -644,7 +664,12 @@
     {
       "mode": "644",
       "path": "examples/python/rag_quickstart.py",
-      "sha256": "9ccd3f757d1f90e773fd13cf719f75baf938b1663567915c72636f9247e1ee3b"
+      "sha256": "6210fcbcfaa8419b30576096cc48f82ff87a8c1de04e0acb3aa64d7edfdc7559"
+    },
+    {
+      "mode": "644",
+      "path": "examples/python/source_native_search.py",
+      "sha256": "6078532d1b25733f3b0b9a4556cf5445c7e24f29b028e87389328c9dc7f6280b"
     },
     {
       "mode": "644",
@@ -674,7 +699,7 @@
     {
       "mode": "644",
       "path": "runtimes/python/README.md",
-      "sha256": "f74eeb84a7bb500e7bfab63279a68737b0d0315d2e68fd2197b03a3996e407f2"
+      "sha256": "f2ae828afaf0af80241b1acfc38f2f536d02d217fe129a6c08190707fe536b20"
     },
     {
       "mode": "644",
@@ -809,7 +834,7 @@
     {
       "mode": "644",
       "path": "runtimes/python/src/vyral_runtime/_local_experience.py",
-      "sha256": "c00ff042ebb73e6518b9f2e87a50c61a4b1daf6512dd6fab472403f3d794689d"
+      "sha256": "c983078cb9addfb0141271f397512ef32657ac5e83afc938f1b5485ff996667d"
     },
     {
       "mode": "644",
@@ -1019,13 +1044,18 @@
     {
       "mode": "644",
       "path": "runtimes/python/src/vyral_runtime/integrations/__init__.py",
-      "sha256": "92e6f7c95b560675cf838c4d1aeafa5c7490afce5f2a932c3d09ea53f9a65afe"
+      "sha256": "bf107710396b5386c6dab15accf39a9fcda991d66397577475f3064a908e9f10"
     },
     {
       "mode": "644",
       "path": "runtimes/python/src/vyral_runtime/integrations/extropic.py",
       "sha256": "b2068cd5495d763c3f7831300afb144bbffa4e83f3cd0713cfd7522792050fbe"
     },
+    {
+      "mode": "644",
+      "path": "runtimes/python/src/vyral_runtime/integrations/ripgrep.py",
+      "sha256": "d662ba9cded10b2dc33e910ae1275e012a8d1a20db0ee8623779c0b3a4018a80"
+    },
     {
       "mode": "644",
       "path": "runtimes/python/src/vyral_runtime/local/__init__.py",
@@ -1229,7 +1259,7 @@
     {
       "mode": "644",
       "path": "runtimes/python/tests/test_local_experience.py",
-      "sha256": "fe7948aa0501f626ca7b7485048d0ee3a22438f187c8857256ab5773037664c0"
+      "sha256": "1838232710d321167f0d27c1289e0ad21c160c78d60f42385376fdb7a55bfdb5"
     },
     {
       "mode": "644",
@@ -1281,6 +1311,11 @@
       "path": "runtimes/python/tests/test_retrieval_evaluation.py",
       "sha256": "8c7e5ae6f0ab68478038c79c1e7b262b6edf4a21df7152a34baebeb212275229"
     },
+    {
+      "mode": "644",
+      "path": "runtimes/python/tests/test_ripgrep_integration.py",
+      "sha256": "9117dddae9441b7b1d21137d5533115e710554e04a36c7bcdd06c2c153d10bae"
+    },
     {
       "mode": "644",
       "path": "runtimes/python/tests/test_runtime.py",
@@ -1431,6 +1466,11 @@
       "path": "scripts/benchmark-python-runtime.py",
       "sha256": "08282add3792153654fbcb6211a974e94e95f693ab7c778f33e6c1e83c6085e2"
     },
+    {
+      "mode": "755",
+      "path": "scripts/benchmark-ripgrep-retrieval.py",
+      "sha256": "a2ffcc98b2ef92697a2d0a879f6282395ba073ae65ba4776bb398156cb182f20"
+    },
     {
       "mode": "755",
       "path": "scripts/download-benchmark-onnx-models.sh",
@@ -1444,7 +1484,7 @@
     {
       "mode": "755",
       "path": "scripts/export-public-tree.py",
-      "sha256": "cda75ce6fc5864869f471ca3de0cd9172fdd3b37ae2bc850d6847f6818ccef5e"
+      "sha256": "3854825cf1d2c38c4a87633479d6b1a68f4209644430b78dcc0f487f70fc0ff5"
     },
     {
       "mode": "755",
@@ -1681,6 +1721,11 @@
       "path": "scripts/verify-execution-smoke-container.sh",
       "sha256": "954a99316778d1316df8966d56ca064e60631d47a9f9410d2be3053b0c0e6b2f"
     },
+    {
+      "mode": "755",
+      "path": "scripts/verify-lexical-http-quickstarts.sh",
+      "sha256": "4e7bc7d534aaef9f553a337f23c032d6915641b725a9a4b416c09e122d6b486f"
+    },
     {
       "mode": "755",
       "path": "scripts/verify-markdown-links.py",
@@ -1779,12 +1824,17 @@
     {
       "mode": "755",
       "path": "scripts/verify-python-runtime.sh",
-      "sha256": "fbd9cd085eff6803debdad5bc52407a85d99fc5f2b9bae3d5b9de167f59b8e63"
+      "sha256": "7c8362e518ffbc810174972e9340f5d69cf9325211a33d42612227b2d2e13827"
     },
     {
       "mode": "755",
       "path": "scripts/verify-release-artifacts.sh",
-      "sha256": "37656bb728e094285b4915f11044fea227cc1c3c34d1497ad8cbde425090d0a6"
+      "sha256": "b84d3aa20a74f49ebce0f34b18c79099335dea9f6c9808eb9182febfc65cc92e"
+    },
+    {
+      "mode": "755",
+      "path": "scripts/verify-ripgrep-retrieval-report.py",
+      "sha256": "bd3ae53e0b90e2ba55c987d28e90e7fe31be5474b6c84a294eeaf2a8e4f507c7"
     },
     {
       "mode": "755",
@@ -4139,5 +4189,5 @@
   ],
   "schemaVersion": 1,
   "sourceDirty": false,
-  "treeSha256": "6ede7d4303711c80c351578da49700971ce01b18baa4729cf9566bc6e4561649"
+  "treeSha256": "e6b7959543b600c6eed8e33b420b1cecd373519e2a212af9f81f7817bc9f5058"
 }

README.md

@@ -15,6 +15,7 @@
 
 Vyral is a local-first contract layer and runtime for applications that need
 **records and objects**, **lexical/vector/hybrid retrieval and RAG**,
+**source-native retrieval experiments**,
 **durable jobs**, and **provider-backed AI runs** without binding application
 code to a single cloud, vector store, orchestration engine, or model vendor.
 
@@ -33,9 +34,9 @@ From a source checkout, run:
 
 That is the complete local path. It needs only Python 3.10 or newer—no package
 installation, .NET SDK, container runtime, cloud account, external service, or
-model download. It runs retrieval with citations, admits durable work, closes
-and reopens the runtime, and completes the preserved run. State remains visible
-under `./.vyral/quickstart`.
+model download. It runs zero-embedding lexical retrieval with citations, admits
+durable work, closes and reopens the runtime, and completes the preserved run.
+State remains visible under `./.vyral/quickstart`.
 
 Create a small editable application when you are ready to build:
 
@@ -54,6 +55,7 @@ output, reset, and server instructions are in the
 | You need… | Start here |
 | --- | --- |
 | Local RAG / retrieval substrate | [Quick start](#quick-start) |
+| Source-native, lexical, vector, or hybrid choice | [Retrieval guide](docs/guides/source-native-retrieval.md) |
 | Embedded Python runtime, no server or .NET | [Python runtime](runtimes/python) |
 | HTTP clients (Python / JavaScript) | [Client packages](#client-packages) |
 | Go external-worker client | [Go client](clients/go) |
@@ -72,7 +74,7 @@ Vyral keeps the capability model application-owned:
 
 - **Records** — ids, partition keys, metadata, content, sources, vectors, and etags
 - **Objects** — large immutable payloads and manifests outside the record index
-- **Retrieval** — filters, lexical, vector, hybrid, profiles, and evaluation
+- **Retrieval** — filters, lexical, vector, hybrid, profiles, evaluation, and experimental source-native search
 - **RAG** — deterministic chunk plans, dry-run/commit, manifests, context, and citations
 - **Provider runs** — one envelope for local, CLI, and ONNX-backed AI capabilities
 - **Execution** — durable runs and plugins with local and provider-backed adapters
@@ -81,6 +83,13 @@ Vyral keeps the capability model application-owned:
 Use Vyral when you want realistic local development and a rehearsed path to
 re-home storage, retrieval, or execution without rewriting the product surface.
 
+For local, current, structured sources that an application can safely access,
+searching the source directly is often the best first retrieval path: exact,
+immediately fresh, and free of index synchronization. Use indexed lexical
+retrieval when you need a governed record boundary, vectors when labeled tests
+show semantic recall is worth the added index, and hybrid or reranking only
+when the same evaluation demonstrates an improvement.
+
 ## Status
 
 Vyral is an Apache-2.0, pre-1.0 project in active development. The local paths
@@ -93,6 +102,7 @@ compatibility policy.
 | .NET local server, SQLite, retrieval, and RAG | Primary supported implementation |
 | Python and JavaScript HTTP clients | Implemented and tested against the public SDK catalog |
 | Python-native embedded runtime and REST/MCP host | Prototype; implemented and clean-artifact qualified on Python 3.10–3.12 across Linux, macOS, and Windows; independent security review remains |
+| Source-native ripgrep integration | Experimental; retained local evidence supports bounded exact-literal search over authorized code/Markdown without a duplicate index; outside the stable wire/MCP contract |
 | Stateless MCP `2026-07-28` endpoint | Preview; opt-in and REST-backed |
 | Local SQLite and Azure Durable execution | Preview; locally conformant |
 | AWS, Google, Temporal, pgvector, and Cloudflare adapters | Preview or prototype; live qualification is opt-in |
@@ -111,8 +121,8 @@ scripts/start-local-server.sh
 ```
 
 It listens on `http://127.0.0.1:5220` and writes local state beneath `.vyral/`.
-Run either HTTP client example to create a collection, ingest three documents,
-retrieve context, and print citations:
+Run either HTTP client example to create a collection, store three documents,
+retrieve lexical context without embeddings, and print citations:
 
 ```bash
 python3 examples/python/rag_quickstart.py