diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256 index 8498d23e..a9bdd327 100644 --- a/.claude-plugin/skill-assets.sha256 +++ b/.claude-plugin/skill-assets.sha256 @@ -1,6 +1,6 @@ b20f1f0f7263b044f880341ddfde3b346b517581321044196c851d7384042563 .claude-plugin/marketplace.json ef872abd9089782b25c8c42a0d5c1582ca1a8930dedb564d1609951f2d263ce3 .claude-plugin/plugin.json -696fe737e83a8d073c8dac77704ada7261332ede2527b4c4d426b1e657e034da skills/engraphis-memory/SKILL.md +018364f63e2181d83ba8d9532c50a41bd30e65eaa8358a721538c297dd063084 skills/engraphis-memory/SKILL.md 7ee71fb5ff9bd2b02f50b3ee8dc62f390a0e1bcd849a55739c4a376ac03d9784 skills/engraphis-memory/references/CONVENTIONS.md 8aafd2daba872be38ec8d42377e886d795d8941bf7c6a39795937ffc1d1f0d88 skills/engraphis-memory/references/SCOPING.md -4c1478453237643e7b4ee2ab4484b9fea8fd759f19f9a5fbf9eeda216d1d6f1a skills/engraphis-memory/references/TOOLS.md +6b0bbb97db4bfa4b1682f9f195bd823f05a2950384ea5c5b261b446b9461d1f1 skills/engraphis-memory/references/TOOLS.md diff --git a/.env.example b/.env.example index ce5808c0..121f1cd9 100644 --- a/.env.example +++ b/.env.example @@ -255,6 +255,10 @@ ENGRAPHIS_LLM_API_KEY=sk-your-key-here # ENGRAPHIS_CHUNK_TOKENS=512 # ENGRAPHIS_CHUNK_MAX=2048 # ENGRAPHIS_CHUNK_OVERLAP=64 +# Optional reader-tokenizer parity for chunk sizes (requires transformers). +# Pin the revision when the resulting memories support reproducible evidence. +# ENGRAPHIS_CHUNK_TOKENIZER_MODEL=Qwen/Qwen3.5-9B +# ENGRAPHIS_CHUNK_TOKENIZER_REVISION= # ENGRAPHIS_LOOP_INTERVAL=300 # ENGRAPHIS_LOOP_TOP_K=10 # ENGRAPHIS_RERANK_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 diff --git a/AGENTS.md b/AGENTS.md index bd7b30e1..74d70567 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,7 @@ task is ambiguous, decide which side it belongs to *before* editing. ```bash # ── Install ────────────────────────────────────────────────────────────────── pip install numpy pytest # v2 core + tests, fully OFFLINE (this is what CI does) -pip install -e ".[dev]" # full stack: FastAPI server, ST embeddings, ruff +pip install -e ".[all,dev]" # full stack: FastAPI server, ST embeddings, ruff cp .env.example .env # only needed for the v1 server / LLM features # ── Quality gate (offline, no API key — KEEP THIS GREEN; mirrors .github/workflows/ci.yml) ── diff --git a/BENCHMARKS.md b/BENCHMARKS.md index f0503571..16fa0f22 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -37,7 +37,43 @@ and stated everywhere the numbers appear (`eval/external.py`). disabled so repeated measurements do not mutate their corpus. It reports p50/p95/p99 latency, retrieval quality, and packed context tokens in one JSON-safe schema. `--filler-memories` provides deterministic corpus scaling, and every report records the runtime, architecture, - embedder, vector backend, corpus size, warmups, and iteration count. + embedder, vector backend, corpus size, warmups, and iteration count. `--candidate-k` and + `--retrieval-profile` make adaptive-depth/routing experiments executable instead of changing + production defaults from an unmeasured hunch. +- **Workload context economy**: `eval/context_economy.py` compares three executable strategies + across every question in a workload: uncapped full-history replay, a contiguous recency window + at the same hard budget, and shipped Engraphis hybrid recall + packing. It reports evidence and + answer-token quality, cumulative reader-context tokens, a conservative total that charges one + complete source-token pass to indexing, and the query-count break-even point. The default is + deterministic/offline; `--embed-model` enables a real retrieval model, while + `--format locomo|longmemeval` reuses the established external loaders. + +The workload benchmark is also allowed to say “this workload is too small for a memory layer.” +On the 44-memory / 26-question CodeMem regression fixture, every case already fits inside a +64-token recency window. Full-history and recency therefore use the same 1,180 cumulative reader +tokens at perfect evidence/answer-token quality, while Engraphis uses 1,375–1,377 reader tokens +plus a conservative 631-token indexing pass. That is an honest no-break-even boundary result: +the benefit being measured begins when history is long or reused enough to outweigh retrieval +framing and indexing. + +The complementary real-model LoCoMo workload diagnostic covers 10 conversations and 1,986 +questions with `all-MiniLM-L6-v2`, `k=10`, a 512-token reader budget, and conflict resolution +disabled. Engraphis used **891,857** cumulative reader-context tokens versus **49,915,394** for +uncapped full history, **98.2133% lower**. Charging one complete 246,539-token corpus pass to +indexing produces a conservative Engraphis total of **1,138,396**, still **97.7193% lower**, with +a calculated break-even at query 10. The quality tradeoff is explicit: + +| LoCoMo workload method | Retrieval recall | Hit rate | Answer-token recall | Mean reader context | +|---|---:|---:|---:|---:| +| Engraphis hybrid recall | **0.600457** | **0.657417** | **0.679614** | **449.07** tokens | +| Same-budget recency window | 0.011289 | 0.012614 | 0.339941 | 487.87 tokens | +| Uncapped full history | 0.996997 | 0.997477 | 0.917247 | 25,133.63 tokens | + +This diagnostic supports a precise statement: Engraphis recovered much more useful evidence than +a same-budget recency window while using a small fraction of full-history context. It does not +support “same quality as full history,” provider-billing, or end-to-end answer-accuracy claims. +The embedding model revision was not pinned in that run, so rerun it with an immutable revision +before treating the numbers as canonical release evidence. ### Reproduce @@ -49,6 +85,10 @@ python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 python -m eval.harness --dataset eval/datasets/graph_multihop.jsonl --k 5 python -m eval.ablation python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 \ + --candidate-k 25 --candidate-depth adaptive --retrieval-profile auto --iterations 10 +python -m eval.context_economy --dataset eval/datasets/codemem.jsonl \ + --token-budget 512 --k 5 python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 \ --iterations 5 --filler-memories 1000 # Canonical latency/resource protocol: requires >=1,000 queries and five processes. @@ -57,6 +97,8 @@ python -m eval.performance --dataset fixed-1000-plus.jsonl --acceptance-matrix - # Real retrieval numbers (downloads all-MiniLM-L6-v2) python -m eval.external --dataset longmemeval_s.json --format longmemeval --k 10 python -m eval.external --dataset locomo10.json --format locomo --k 10 +python -m eval.context_economy --dataset locomo10.json --format locomo \ + --embed-model sentence-transformers/all-MiniLM-L6-v2 --token-budget 512 --k 10 --no-resolve ``` ## What we do NOT yet claim @@ -65,6 +107,9 @@ python -m eval.external --dataset locomo10.json --format locomo --k 10 - **No hosted-service latency comparison.** The in-repo p50/p95/p99 benchmark covers the local reference pipeline and records its environment; unlike environments are not compared. - **No neutral third-party ranking.** We have not run an external eval platform. +- **No provider bill estimate.** Context-economy counts reader evidence under its named counter. + It excludes system/tool prompts, questions, completions, prompt caching, provider pricing, + compute, and storage. Its indexing-inclusive total is a conservative text-volume proxy. Every publishable run should emit the `engraphis-benchmark/v2` envelope: dataset/config hashes, per-question records, explicit exclusions, fixed-budget context curves, and deterministic @@ -99,6 +144,12 @@ tokens; that single official point must not be presented as a five-point curve. cases. Retrieval-only abstention/no-evidence records remain visible in the artifact's `exclusions`; they are not counted as evidence-retrieval scores. +Official LongMemEval-V2 output can be converted into a public-safe QA artifact with +`python -m eval.longmemeval_v2_evidence`. The exporter keeps the official QA score, fixed-reader +context token count, latency, model revisions, source digests, repository state, and artifact +checksum. It removes raw questions, answers, prompts, reader output, and retrieved context before +the artifact can be written. See [`eval/EVIDENCE.md`](eval/EVIDENCE.md) for the exact command. + ### LongMemEval-V2 memory-module adapter `eval.longmemeval_v2.EngraphisLongMemEvalV2Memory` follows the official @@ -125,11 +176,15 @@ tokens. Packed sources are returned as separate context items, preserving the la evidence prefix instead of dropping one oversized monolithic item. The adapter does not download benchmark data or call the reader/evaluator; the official harness owns those steps. -## Next steps for external publishable numbers +## External evidence status and remaining executions -1. **Add a QA layer to `eval/external.py`.** Optional answering model + judge on top of the - existing retrieval pipeline, so the official datasets can report end-to-end accuracy while - reusing the retrieval harness underneath. +1. **Run the official LongMemEval-V2 reader and evaluator.** The adapter, pinned runner, and + redacted evidence exporter are implemented. The exact upstream commit boots in an isolated + Python 3.11 environment and the wrapper reaches the official harness CLI. Dataset revision + `f152293e235517d504809563c833d7190b8c713b` publishes 7,120,369,667 bytes before the pinned + Qwen reader and embedding model assets. A full official run therefore still requires those + resources, sufficient compute, and evaluator configuration; no canonical QA score is claimed + until that run completes. 2. **Publish production-backend latency.** Run `eval/performance.py` with the real embedder and sqlite-vec/backend configuration on a fixed machine class and corpus scale. 3. **Run the fixed-budget curve on the complete official datasets.** The v2 harness now measures @@ -138,6 +193,95 @@ benchmark data or call the reader/evaluator; the official harness owns those ste after complete official runs produce immutable artifacts for every point. 4. **Run an external evaluation platform** once (1)–(3) exist. +Do not make all evidence lanes variants of explicit factual recall. Executable offline adapters +now cover: + +- [MemoryAgentBench](https://github.com/HUST-AI-HYZ/MemoryAgentBench): incremental multi-turn + learning, long-range understanding, and conflict/consolidation inputs. +- [LoCoMo-Plus](https://github.com/xjtuleeyf/Locomo-Plus): an old implicit constraint must affect + a later response even when the later cue does not restate the remembered fact. +- [Mem2ActBench](https://github.com/Cantaloupe-M/Mem2ActBench): memory must select a tool and + ground its arguments, not merely return a passage. The current adapter measures retrieval and + expected tool-argument context coverage, not generated tool-call success. + +```bash +python -m eval.agent_benchmarks --dataset memoryagentbench.json \ + --format memoryagentbench +python -m eval.agent_benchmarks --dataset locomo_plus.json \ + --format locomo_plus +python -m eval.agent_benchmarks --dataset qa_dataset.jsonl \ + --conversations toolmem_conversation.jsonl --format mem2actbench \ + --artifact artifacts/mem2actbench.json +``` + +Use `--artifact` on any of these commands to write a redacted, immutable evidence envelope plus +an adjacent SHA256 file. The ordinary console/`--json` report is private run material and may +contain source questions for debugging. + +### Upstream-data diagnostic baseline (2026-07-30) + +These runs use the dependency-free deterministic embedder on upstream data. They validate the +adapters and expose product gaps; they are noncanonical diagnostics, not leaderboard or marketing +claims. The artifact validator accepted every completed envelope. + +| Upstream source | Executed scope | Result and boundary | +|---|---|---| +| LoCoMo-Plus commit `059f4e3d38f7f1f96765e8e2cb7de3097551bffb` | All 401 Cognitive cases, 40,270 source memories | Recall@10 **0.1259**, hit@10 **0.1272**, MRR@10 **0.0744**, answer-token context coverage **0.5095**. This is cue-evidence retrieval, not answer-judge accuracy. The low retrieval score is useful negative evidence: implicit-constraint recall remains a real product gap. | +| MemoryAgentBench commit `455306dcabc3842526eb83cd4e225e5d486c5c5d`, official Hugging Face `Accurate_Retrieval` first row | 100 questions | Recall@10 **0.5100**, hit@10 **0.8600**, answer-token context coverage **0.8500**. Gold evidence was derived only where an accepted answer occurred in a source chunk. | +| The same source, `Conflict_Resolution` first row | 100 questions | Recall@10 **0.4600**, hit@10 **0.6400**, answer-token context coverage **0.6800**. This plain-context export measures retrieval, not structured temporal invalidation. | +| The same source, `Long_Range_Understanding` first row | 1 question | Answer-token context coverage **0.2658**. The export supplied no evidence IDs and no accepted answer occurred verbatim in a source chunk, so retrieval was deliberately left unscored rather than reported as a false perfect score. | +| The same source, `Test_Time_Learning` first row | One 5.88 MB context | The no-resolution ingest did not complete within a five-minute local smoke ceiling. This is a measured large-ingest throughput gap, not a failed quality score; batch embedding and transaction work should precede a complete split run. | +| Mem2ActBench upstream smoke | 2 public rows | Recall@10, hit@10, MRR@10, and NDCG@10 **1.0000**; expected tool-call JSON token coverage **0.5714**. This is retrieval/context coverage, not generated action success. | + +The MemoryAgentBench loader accepts both its aligned public JSON export and the Hugging Face +dataset-server `rows[].row` envelope. Rows without gold evidence remain useful for answer-token +coverage, but are excluded from retrieval aggregates and counted separately as +`retrieval_scored_questions`. + +For paired code-agent runs, execute the same tasks with the same model, tools, machine, and +deterministic success oracle under `full_history` and `engraphis`. Then analyze the content-free +run records with: + +```bash +python -m eval.code_agent_ab --full-history full-history.jsonl \ + --engraphis engraphis.jsonl --output paired-report.json +``` + +The analyzer rejects unmatched task IDs and different success oracles, then reports paired +bootstrap intervals for task success, input/output/tool tokens, retries, latency, and optional +cost. Its aggregate output does not echo task IDs or oracle commands. It does not launch an agent +or invent a task-success oracle. + +## Optimization experiments to run before changing defaults + +1. **Budget-aware packing**: compare full source, safe summary, sentence-aligned safe summary + excerpt, and raw-source excerpt at fixed budgets. Gate on support/answer retention and + qualifier preservation, not token count alone. +2. **Adaptive retrieval work**: `--candidate-depth adaptive` is now an opt-in performance + experiment. It keeps wider graph/code pools and reduces routine lexical/balanced pools while + reporting the requested and actual depth. Sample and CodeMem kept every offline quality metric + at 1.0 with balanced depth reduced from 50 to 15; CodeMem plus 1,000 fillers reduced local + median recall latency from 20.666 ms to 18.991 ms in a 260-recall comparison, an 8.1% + reduction. These are machine-specific regression results, not production latency claims. Keep + the default fixed until complete external categories meet predeclared quality margins. +3. **Packing-pressure consolidation**: prioritize memory families that are frequently recalled, + repeatedly omitted, or costly per useful token. Count write/index/storage cost as well as later + reader-context savings. +4. **Tokenizer-aware ingestion**: implemented behind the chunk extractor. The dependency-free + default remains `engraphis.chars4.v1`; an explicitly configured Hugging Face reader tokenizer + enforces prose chunk and overlap budgets and records its identity in chunk metadata. Continue + measuring tokens-to-evidence, recall, and storage/index growth together before recommending a + model-specific default. +5. **Bulk ingestion**: add batch embedding plus a transaction-aware vector upsert path, then rerun + the 5.88 MB MemoryAgentBench Test-Time Learning row. Gate this on identical stored-memory, + provenance, graph-link, and temporal-resolution outcomes, not throughput alone. +6. **Scoped caches**: benchmark query embeddings and repeat-recall results keyed by workspace, + repo, time anchors, profile, and corpus version. Test invalidation correctness before claiming + latency gains. +7. **Privacy-safe real usage**: use `engraphis_context_savings` to let each workspace inspect + aggregate source/context/saved tokens already present in content-free receipts. Keep unlike + token counters separate and require a valid receipt chain before treating totals as auditable. + ## Evaluation question The predeclared question is whether the full vector + lexical/BM25 + sparse PPR graph + calibrated diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac6925f..2d8688af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,43 @@ All notable changes to Engraphis are documented here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/); versions use SemVer. +## [Unreleased] + +### Added + +- `engraphis_context_savings` aggregates validated, content-free recall receipts by workspace, + repo, operation, and token-counter identity. The same read-only view is available through the + service, Inspector, v2/read-only APIs, and dashboard receipt panel. +- Recall supports an explicit adaptive candidate-depth experiment while retaining the historical + fixed depth by default. Performance reports record requested and actual candidate depths. +- Chunk ingestion can enforce budgets with an injected or explicitly configured Hugging Face + tokenizer and records the counter identity, target, and overlap in each chunk's metadata. +- Offline adapters now cover MemoryAgentBench, LoCoMo-Plus, and Mem2ActBench. A paired code-agent + analyzer compares full-history and Engraphis runs using identical tasks and success oracles. +- Public benchmark evidence can carry source hashes, repository state, environment and model + provenance, secret-redacted commands, content digests, and adjacent immutable SHA-256 files. + +### Changed + +- Context-economy evaluation now compares uncapped full history, a same-budget recency window, + and shipped hybrid recall while charging an explicit one-time indexing token proxy. +- Official LongMemEval-V2 output has a dedicated redacted evidence exporter that retains the + official QA/token/latency measures without publishing prompts, answers, model output, or + retrieved context. +- Folder-sync dry runs no longer create a remote directory or persist a local device identity. + +### Fixed + +- Sync rejects malformed scope/repo combinations and every peer-driven visibility change for an + existing memory, including malformed legacy rows. Scope promotion or repair remains a local, + explicit governance operation. +- Workspace consolidation excludes session-private memories and partitions digests and entity + profiles by their exact visibility owner, preventing cross-repo or cross-scope summaries. +- Tokenizer-aware chunk overlap can no longer exceed the configured prose budget or emit a + duplicate overlap-only record before an oversized paragraph. Invalid token counters fail + closed instead of silently producing mis-sized chunks. +- The new evidence guide is included in wheel and source distributions. + ## [1.2.2] - 2026-07-30 ### Fixed diff --git a/MANIFEST.in b/MANIFEST.in index 7ceee97b..ad7fe66f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,6 +15,7 @@ include docker-entrypoint.sh Dockerfile docker-compose.yml include railway.json recursive-include eval *.py include eval/BASELINES.md +include eval/EVIDENCE.md recursive-include eval/configs *.json recursive-include eval/datasets *.jsonl recursive-include tests *.py diff --git a/README.md b/README.md index f7358671..8f73b50c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,18 @@ https://discord.com/invite/Wfr2ejBmY Knowledge Graph · run engraphis-dashboard to see it live

+--- + +> Update regularly for the latest fixes and improvements. +> +> **Open-core boundary:** this repository contains the free local engine, dashboard, MCP server, +> and customer-side clients. Hosted sync, analytics, automation, and team services run on the +> official hosted service; their server implementations are not distributed here. + +> **Support continued Engraphis development with Pro.** [Start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro&trial=pro#billing) +> or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing). + + ## What Engraphis gives an agent An agent should not have to reconstruct a project from scattered chat history on every task. @@ -28,10 +40,8 @@ that supports the current question; and returns a bounded, attributable context Store durable project knowledge · retrieve supporting evidence · give the agent only what it needs

-The flow is the essential path. See [measured context savings](#measured-quality-and-token-efficiency) -for reproducible fixture-level evidence of less returned content at the same tested retrieval -scores, without billing or latency claims. The sections below cover the dashboard, code graph, -local installation, governance controls, and hosted services in detail. +The flow is the essential path. See [measured token and context savings](#measured-token-and-context-savings) +for the short version of how much less history an agent has to carry. | Agent need | What Engraphis changes | |---|---| @@ -58,17 +68,6 @@ Run `python -m eval.chunking_eval` and `python -m eval.grounded` to reproduce th the former measures evidence retrieval and context size, while the latter measures the answer-versus-abstain decision. ---- - -> Open-source users: update regularly for the latest fixes and improvements. -> -> **Open-core boundary:** this repository contains the free local engine, dashboard, MCP server, -> and customer-side clients. Hosted sync, analytics, automation, and team services run on the -> official hosted service; their server implementations are not distributed here. - -> **Support continued Engraphis development with Pro.** [Start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro&trial=pro#billing) -> or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing). - ## Full Engraphis install: pip install "engraphis[all]" Engraphis-Dashboard opens `http://127.0.0.1:8700`. Local memory needs no cloud account, @@ -131,35 +130,18 @@ chunking. The activity view records outcomes, never keys, prompts, or raw provid --- -## Measured quality and token efficiency - -Engraphis ships typed and scoped memory, bi-temporal history, grounded recall, hybrid -vector/lexical/graph retrieval, deterministic context packing, and MCP-native agent tools. -The current deterministic offline regression fixtures reproduce these quality results: - -| Fixture | Reproduced result | -|---|---| -| CodeMem retrieval: 44 memories, 26 questions | **Recall@5 1.000**, hit@5 1.000, answer-token recall 1.000 | -| Grounded-answer decisions: 10 cases | **10/10 correct**: 5/5 answerable questions cited evidence and 5/5 off-topic questions abstained | - -### Proof at a glance - -| **73.0% less retrieved context** | **3.8× smaller evidence record** | **55.38% smaller MCP response** | -|---|---| -| **808.8 → 218.4** tokens per question | **162.2 → 42.4** tokens to supporting evidence | **17,172 → 7,663** serialized tokens | -| Same Recall@5 **1.000** in the long-document fixture | Same 18 fixture questions returned an evidence-holding memory | Same CodeMem retrieval scores across 260 timed recalls | - -**What it means:** agents carry less irrelevant history, leaving more room for the current task -and cited evidence. These controlled, deterministic fixtures measure context, not model billing, -task time, customer savings, or external-benchmark performance. +## Measured token and context savings

- Normalized chart: Engraphis retains 27.0 percent of retrieved content, 26.1 percent of the evidence-holding record, and 44.6 percent of the compact MCP response in separate controlled fixtures + Dark chart showing Engraphis using 98.21 percent less long-history context, 73.0 percent less retrieved content per question, 73.9 percent fewer tokens in the smallest useful memory, a 55.38 percent smaller memory response, and 47.8 percent less repeated-memory context after consolidation
- Each row uses a separate 100% baseline. The measurements have different counting boundaries and are not additive. + Less repeated history means more room for the task, tools, and useful evidence.

-#### A controlled before-and-after example +
+See benchmark details and reproduce the results + +### Controlled before-and-after example | Retrieval mode | Mean returned memory content | Recall@5 | |---|---:|---:| @@ -177,9 +159,11 @@ boundary. | What is counted | Comparison | Measured reduction | Quality held constant | |---|---|---|---| +| Cumulative reader context across a 1,986-question LoCoMo diagnostic | Full-history replay: **49,915,394** tokens → Engraphis: **891,857** tokens | **49,023,537 fewer context tokens** (**98.2133% lower**) | Focused retrieval used far less context; uncapped full history retained higher retrieval recall | | Retrieved top-5 memory content, averaged per question | Whole documents: **808.8** tokens → structure-aware chunks: **218.4** tokens | **590.4 fewer tokens per question** (**73.0% lower**, about **3.7× smaller**) | Recall@5 **1.000** in both modes across 6 documents and 18 questions | | Smallest returned memory that contains the reference evidence | Whole documents: **162.2** tokens → chunks: **42.4** tokens | **119.8 fewer tokens to evidence** (**73.9% lower**, about **3.8× smaller**) | The same 18 questions had a returned evidence-holding memory in both modes | | Serialized MCP recall response across 260 timed CodeMem recalls | Full result: **17,172** `engraphis.regex.v1` tokens → compact result: **7,663** tokens | **9,509 response tokens avoided** (**55.38% lower**) | Recall@5, hit@5, and answer-token recall all **1.000** | +| Repeated-memory consolidation fixture | 12 related episodic memories: **230** tokens → one digest: **120** tokens | **110 tokens removed from the active digest** (**47.8% lower**) | Original memories remain available for provenance and audit | | Packed prompt-context usage in the same CodeMem performance fixture | Hard budget: **1,500** tokens; observed mean: **87.73**; observed maximum: **106** | A hard cap prevents a recall from exceeding its configured context budget | This is usage accounting, not a before/after savings comparison | The compact MCP response avoids duplicating full memory bodies when the packed context and source @@ -209,6 +193,8 @@ normalized-character estimator. Chunking measures retrieved memory content, whil measures serialized MCP response size. See [`BENCHMARKS.md`](BENCHMARKS.md) for definitions, limitations, canonical external-evaluation requirements, and the no-unsupported-claims policy. +
+ --- ## Install @@ -289,7 +275,7 @@ claude mcp add engraphis -- engraphis-mcp cmd mcp add engraphis -- engraphis-mcp # Command Code CLI ``` -Your agent now has 30 tools: remember, recall context (plus full, grounded, and proactive recall), +Your agent now has 31 tools: remember, recall context (plus full, grounded, and proactive recall), proactive context, grounded answer alias, why, timeline, forget, pin, correct, promote, ingest, consolidate, index_repo, search/code path/impact/export, privacy receipts, PostgreSQL schema ingestion, link, @@ -426,7 +412,7 @@ to support the project and add hosted services. | | Free (available now) | Pro: $10/mo or $100/yr | Team: $20/seat/mo or $200/seat/yr | |---|---|---|---| | Dashboard WebUI (with built-in inspector) | ✓ | ✓ | ✓ | -| Memory engine + 30 MCP tools | ✓ | ✓ | ✓ | +| Memory engine + 31 MCP tools | ✓ | ✓ | ✓ | | Version-chain diffs, offline knowledge graph | ✓ | ✓ | ✓ | | Manual local consolidation (dry-run by default) | ✓ | ✓ | ✓ | | Local workspace export (JSON: memories, sessions, audit) | ✓ | ✓ | ✓ | @@ -466,6 +452,7 @@ to support the project and add hosted services. | Code | `engraphis_code_impact` | Rank changed files by symbols, dependents, communities, memories, and hotspots | | Code | `engraphis_export_code_graph` | Portable graph JSON + Markdown + HTML report | | Audit | `engraphis_receipts` | List content-free hashed operation receipts | +| Audit | `engraphis_context_savings` | Sum privacy-safe context usage by workspace/repo and token-counter identity | | Audit | `engraphis_verify_receipts` | Verify the receipt chain, local tail anchor, and optional externally saved head/count | | Audit | `engraphis_export_receipts` | Export the shareable receipt-only audit bundle | | Governance | `engraphis_forget` | Retire a memory: bi-temporal close, never deleted; every request is audited | @@ -496,9 +483,12 @@ repository is supplied. The operation-receipt chain is deliberately content-free. It records bounded operation metadata and chained hashes, while excluding raw memory/query text, workspace names, memory IDs, and actor identities from exported receipt payloads. Use `engraphis_receipts`, -`engraphis_verify_receipts`, and `engraphis_export_receipts` to inspect the chain or compare it -with a previously saved head/count anchor. A separately maintained local count/head anchor and -persistent integrity marker make interior edits, reordering, and tail truncation detectable. +`engraphis_context_savings`, `engraphis_verify_receipts`, and `engraphis_export_receipts` to +inspect the chain, aggregate retrieved-source versus packed-context tokens, or compare it with a +previously saved head/count anchor. Savings stay separated by token-counter identity and are +reported with chain validity; they are packing measurements, not provider bills. A separately +maintained local count/head anchor and persistent integrity marker make interior edits, reordering, +and tail truncation detectable. See [the v3 architecture document](docs/ARCHITECTURE_V3.md) for the data flow and [SECURITY.md](SECURITY.md) for the trust boundaries. @@ -613,7 +603,11 @@ Drag-and-drop or server-side import, access-controlled and bounded: a big context-reduction win on long docs. Works across all three ingest paths (dashboard upload, `import_folder`, and `engraphis_ingest`). Measure the payoff with the bundled eval: `python -m eval.chunking_eval --dataset eval/datasets/longdoc.jsonl --k 5` - (whole-file vs. chunked, same recall pipeline, offline). + (whole-file vs. chunked, same recall pipeline, offline). The dependency-free default + uses the named `engraphis.chars4.v1` estimate. Set + `ENGRAPHIS_CHUNK_TOKENIZER_MODEL` and, for reproducible runs, + `ENGRAPHIS_CHUNK_TOKENIZER_REVISION` to size prose chunks with the actual reader + tokenizer; the chosen counter identity is preserved in each chunk's metadata. - **Structured LLM extraction**: `ENGRAPHIS_EXTRACTOR=llm_structured` validates typed facts, entities, relations, and keywords before storage. Its preserved entity/relation metadata feeds the knowledge graph automatically. A successful dashboard connection test @@ -673,6 +667,8 @@ All via environment (or `.env`): | `ENGRAPHIS_DB_KEY` | Not set | Encrypt the database at rest (SQLCipher). Or use `ENGRAPHIS_DB_KEY_FILE` | | `ENGRAPHIS_EMBED_MODEL` | `sentence-transformers/all-MiniLM-L6-v2` | sentence-transformers model | | `ENGRAPHIS_EXTRACTOR` | `none` | `none` = verbatim; `chunk` = offline structure-aware chunks; `llm` = free-form LLM facts; `llm_structured` = schema-validated facts + graph metadata | +| `ENGRAPHIS_CHUNK_TOKENIZER_MODEL` | Not set | Optional Hugging Face tokenizer used to enforce chunk budgets with the downstream reader's real tokenization; requires the optional `transformers` package | +| `ENGRAPHIS_CHUNK_TOKENIZER_REVISION` | Not set | Optional immutable tokenizer/model revision recorded in the chunk-counter identity; pin this for reproducible benchmark artifacts | | `ENGRAPHIS_GRAPH_EXTRACTOR` | `regex` | `regex` = offline heuristic NER; `none` = disable heuristic text extraction (validated `llm_structured` metadata still feeds the graph) | | `ENGRAPHIS_RETENTION_SUPERVISOR` | `none` | `none` = deterministic only; `llm` = sends a bounded excerpt to the configured provider for advisory ephemeral/normal/critical classification | | `ENGRAPHIS_WHISPER_MODEL` | Not set | Enables local faster-whisper audio/video transcription | @@ -708,7 +704,7 @@ engraphis/ │ ├── core/ # v2 engine: interfaces, store, recall, scoring, schema, sync │ ├── backends/ # pluggable embedder / vector index / reranker / codegraph / sync transports / encryption │ ├── service.py # validated MemoryService facade -│ ├── mcp_server.py # MCP server: 30 tools +│ ├── mcp_server.py # MCP server: 31 tools │ ├── dashboard_app.py # dashboard WebUI (FastAPI) │ ├── dashboard_assets/ # primary Ledger interface + graph engine │ ├── classic_assets/ # selectable full operator dashboard backup @@ -748,8 +744,11 @@ ruff check . ``` Numbers, not assertions: the offline harness is a **correctness floor** (deterministic embedder). -LoCoMo / LongMemEval adapters and the pinned LongMemEval-V2 reader profile are available for -approved official evaluation runs: see +LoCoMo, LongMemEval, MemoryAgentBench, LoCoMo-Plus, and Mem2ActBench adapters are available, +along with a pinned LongMemEval-V2 reader profile, redacted evidence exporter, and paired +full-history versus Engraphis code-agent analyzer. External adapters measure only the layer they +declare; retrieval or tool-argument context coverage is not presented as end-to-end answer, +action, or task success. Reproduction commands and remaining official-run requirements are in [`BENCHMARKS.md`](BENCHMARKS.md). --- diff --git a/docker-compose.yml b/docker-compose.yml index a023d8eb..a095a8e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,8 +54,10 @@ services: ENGRAPHIS_HOST: 0.0.0.0 # This process binds all container interfaces. Even though the published host port is # loopback-only, require an explicit bearer so a changed port mapping cannot silently - # expose the legacy v1 API. Compose fails before creating the container if it is absent. - ENGRAPHIS_API_TOKEN: ${ENGRAPHIS_API_TOKEN:?Set ENGRAPHIS_API_TOKEN before starting the api profile} + # expose the legacy v1 API. Keep interpolation optional so the inactive profile does + # not break a fresh ``docker compose up``; engraphis-server fails closed at startup + # when this profile is actually launched without a token. + ENGRAPHIS_API_TOKEN: ${ENGRAPHIS_API_TOKEN:-} # The v1 server uses a DIFFERENT, incompatible memory schema from the v2 dashboard, # so it MUST NOT share the dashboard's engraphis.db (doing so corrupts both). Give it # its own file on the shared volume. diff --git a/docs/ARCHITECTURE_V3.md b/docs/ARCHITECTURE_V3.md index 69c523ce..96ece2fc 100644 --- a/docs/ARCHITECTURE_V3.md +++ b/docs/ARCHITECTURE_V3.md @@ -7,7 +7,7 @@ retention-supervision, and privacy-receipt additions introduced with schema vers flowchart LR Agent["Agent / host LLM"] --> Intent["remember · link · recall_context (compact) · recall"] CLI["engraphis-graph CLI"] --> Service["MemoryService"] - MCP["30 MCP tools"] --> Service + MCP["31 MCP tools"] --> Service HTTP["Dashboard + read-only graph HTTP"] --> Service Import["Local resources / PostgreSQL catalog"] --> Extractors["Optional local extractors"] Extractors --> Service diff --git a/docs/KILO_CODE_INTEGRATION.md b/docs/KILO_CODE_INTEGRATION.md index df371b8d..170aa02e 100644 --- a/docs/KILO_CODE_INTEGRATION.md +++ b/docs/KILO_CODE_INTEGRATION.md @@ -10,9 +10,9 @@ This manual is written for someone who wants the full technical picture: what En There are two separate questions hiding inside "connect Kilo Code to Engraphis," and they are usually where people talk past each other: -1. **Transport layer: "get the pipes connected."** This is: install the Engraphis MCP server, tell Kilo Code how to launch it, confirm the tools show up. It's a plumbing task. When it's done, Kilo Code can *see* 30 `engraphis_*` tools. Success here is binary: either the tools appear or they don't. +1. **Transport layer: "get the pipes connected."** This is: install the Engraphis MCP server, tell Kilo Code how to launch it, confirm the tools show up. It's a plumbing task. When it's done, Kilo Code can *see* 31 `engraphis_*` tools. Success here is binary: either the tools appear or they don't. -2. **Orchestration layer: "use the memory well."** This is: *when* should the agent remember vs. recall, how should memories be scoped (`workspace → repo → session`), which of the 30 tools answers which question, and how to keep the store clean over time. This is where the actual value is, and it's a discipline, not a config. +2. **Orchestration layer: "use the memory well."** This is: *when* should the agent remember vs. recall, how should memories be scoped (`workspace → repo → session`), which of the 31 tools answers which question, and how to keep the store clean over time. This is where the actual value is, and it's a discipline, not a config. You need both. A perfect config with no discipline gives you an agent that has memory tools and never uses them correctly. Good discipline with a broken config gives you an agent that wants to remember and can't. **Section 3 is the transport layer. Sections 4–6 are the orchestration layer.** Do them in order. @@ -40,7 +40,7 @@ Everything runs on your machine. The whole store is a single SQLite file. Local You interact with Engraphis through three surfaces, all backed by the *same* engine (`MemoryService`), so they can never drift apart: - **The dashboard WebUI** (`engraphis-dashboard`, `http://127.0.0.1:8700`): a visual product to see, search, and curate memory. -- **The MCP server** (`engraphis-mcp`): the 30 tools your coding agent calls. **This is the surface Kilo Code uses.** +- **The MCP server** (`engraphis-mcp`): the 31 tools your coding agent calls. **This is the surface Kilo Code uses.** - **The Python library** (`from engraphis.service import MemoryService`): for direct programmatic use. ### 2.1 The five ideas that make it more than a vector store @@ -183,7 +183,7 @@ You can also click **Approve Always** on any tool at runtime to write the same r --- -## 4. The 30 tools: the orchestration surface +## 4. The 31 tools: the orchestration surface Once connected, Kilo Code sees these. Do **not** assume only `remember`/`recall` exist. The value is in the rest. This is the full surface, grouped by what question each one answers. @@ -208,6 +208,7 @@ Once connected, Kilo Code sees these. Do **not** assume only `remember`/`recall` | Code | `engraphis_code_impact` | Rank commit/PR impact by dependents, communities, memories, and hotspots. | | Code | `engraphis_export_code_graph` | Portable graph JSON + Markdown + self-contained HTML. | | **Audit** | `engraphis_receipts` | List content-free hashed operation receipts. | +| Audit | `engraphis_context_savings` | Cumulative packed-context savings from receipts, separated by token-counter identity. | | Audit | `engraphis_verify_receipts` | Verify the tamper-evident receipt chain. | | Audit | `engraphis_export_receipts` | Export a privacy-safe receipt-only audit bundle. | | **Governance** | `engraphis_forget` | Retire a memory: bi-temporal close, never a hard delete; every request is audited. | diff --git a/docs/dashboard-button-qa.md b/docs/dashboard-button-qa.md index 5a75c582..249e0487 100644 --- a/docs/dashboard-button-qa.md +++ b/docs/dashboard-button-qa.md @@ -41,10 +41,11 @@ Classic navigation/mobile-nav controls. ## Environment notes -- One parallel lane could not start against the repository's default database because that - existing database is schema version 5 while this checkout supports schema version 4. - This is an environment/data compatibility issue, not a dashboard button failure. The - isolated schema-4 fixture started and exercised the UI successfully. +- At the time of this manual pass, one parallel lane could not start against the repository's + default database because the checkout then supported schema version 4 while that existing + database was schema version 5. This historical environment/data compatibility issue was not a + dashboard button failure. The isolated schema-4 fixture started and exercised the UI + successfully. - The browser harness did not expose programmatic download events for the PNG/JSON export anchors, but the dashboard status confirmed both exports completed. No application console errors were observed during the manual pass. diff --git a/docs/images/context-efficiency.png b/docs/images/context-efficiency.png index 0041fc91..bd6543f4 100644 Binary files a/docs/images/context-efficiency.png and b/docs/images/context-efficiency.png differ diff --git a/docs/images/context-efficiency.svg b/docs/images/context-efficiency.svg index deb1e88b..1587eca0 100644 --- a/docs/images/context-efficiency.svg +++ b/docs/images/context-efficiency.svg @@ -1,45 +1,106 @@ - - Engraphis context efficiency in deterministic offline fixtures - Three normalized comparisons show retained token payload after using Engraphis: 27.0 percent for retrieved content, 26.1 percent for an evidence-holding record, and 44.6 percent for a compact MCP response. Every label has a separate line above its corresponding bar. Each row has a separate baseline. + + Engraphis measured token and context savings + A dark-mode chart with five measured comparisons. Engraphis used 98.21 percent less context over a long-history workload, 73.0 percent less retrieved context per question, 73.9 percent fewer tokens in the smallest useful memory, returned a 55.38 percent smaller memory-tool response, and reduced a repeated-memory cluster by 47.8 percent through consolidation. Supporting measurements show 53 times more evidence than recency-only retrieval at the same budget, 97.72 percent less total context after including the complete indexing pass with break-even by question 10, and an observed maximum of 106 context tokens under a 1500-token cap. - + + + + + + + + + + + - - - Less history in context; the same fixture-level retrieval quality - Each row is normalized to its own 100% baseline. The measures count different boundaries and are not additive. - baseline payload (100%) - Engraphis payload retained + + + + + Give your agent more room to think + Measured savings across long histories, retrieval, responses, and memory cleanup. + + + Without Engraphis + + With Engraphis + Each row has its own baseline - - Retrieved memory content - Long-document fixture · top-5 memory content - Baseline · whole documents · 808.8 tokens - - Engraphis · structure-aware chunks · 218.4 tokens · 73.0% lower - + + Long project history sent to the model + LoCoMo diagnostic · 10 conversations · 1,986 questions + Focused context; full-history recall was higher + Replay everything · 49,915,394 tokens + + Engraphis · 891,857 tokens + + 98.21% less - - Smallest evidence-holding record - Long-document fixture · evidence returned for the same 18 questions - Baseline · whole documents · 162.2 tokens - - Engraphis · structure-aware chunks · 42.4 tokens · 73.9% lower - + + Retrieved memory content per question + Long-document test · 18 questions · Recall@5 1.000 + Whole documents · 808.8 tokens + + Focused chunks · 218.4 tokens + + 73.0% less - - Serialized MCP recall response - CodeMem fixture · retrieval scores unchanged across 260 timed recalls - Baseline · full response · 17,172 tokens - - Engraphis · compact response · 7,663 tokens · 55.38% lower - + + Smallest useful memory returned + The same 18 questions found supporting evidence + Whole document · 162.2 tokens + + Useful chunk · 42.4 tokens + + 73.9% less + + + + + Complete memory-tool response + CodeMem test · 260 recalls + Retrieval scores unchanged + Full response · 17,172 tokens + + Compact response · 7,663 tokens + + 55.38% less + + + + + Repeated memories after consolidation + 12 related events condensed into one digest + Originals remain auditable + Repeated memories · 230 tokens + + Consolidated digest · 120 tokens + + 47.8% less + + + + + SAME 512-TOKEN BUDGET + 53× more evidence + than recency-only retrieval + + + INCLUDING INDEXING + 97.72% less total + paid back by question 10 + + + HARD CONTEXT CAP: 1,500 + 87.7 average · 106 max + observed context tokens in CodeMem - Controlled deterministic offline regression fixtures, not model-billing, task-time, customer-cost, or external benchmark claims. + Separate measured tests; percentages are not additive. Token/context measurements, not provider billing or customer-cost claims. Full methods: BENCHMARKS.md diff --git a/engraphis/backends/extractor.py b/engraphis/backends/extractor.py index 39845302..1b800b13 100644 --- a/engraphis/backends/extractor.py +++ b/engraphis/backends/extractor.py @@ -28,6 +28,7 @@ import json import os import re +from collections.abc import Callable from typing import Any, Optional, Type from engraphis.core.interfaces import ExtractedFact, MemoryType, LLM @@ -53,6 +54,7 @@ class ValidationError(Exception): # type: ignore CHUNK_TARGET_TOKENS = 256 # target tokens per prose chunk CHUNK_OVERLAP_TOKENS = 32 # sentence-level overlap carried between adjacent chunks CHUNK_MAX = 200 # hard cap on chunks per document (amplification guard) +DEFAULT_CHUNK_TOKEN_COUNTER = "engraphis.chars4.v1" _HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*$") _FENCE_RE = re.compile(r"^(```+|~~~+)") @@ -354,10 +356,22 @@ class ChunkingExtractor: def __init__(self, *, target_tokens: int = CHUNK_TARGET_TOKENS, overlap_tokens: int = CHUNK_OVERLAP_TOKENS, - max_chunks: int = CHUNK_MAX) -> None: + max_chunks: int = CHUNK_MAX, + token_counter: Optional[Callable[[str], int]] = None, + token_counter_identity: Optional[str] = None) -> None: self.target_tokens = max(16, int(target_tokens)) self.overlap_tokens = max(0, min(int(overlap_tokens), self.target_tokens // 2)) self.max_chunks = max(1, int(max_chunks)) + self._count = token_counter or estimate_tokens + self.token_counter_identity = ( + token_counter_identity + or getattr(self._count, "identity", None) + or ( + DEFAULT_CHUNK_TOKEN_COUNTER + if self._count is estimate_tokens + else getattr(self._count, "__name__", type(self._count).__name__) + ) + ) def extract(self, text: str, *, context: str = "") -> list[ExtractedFact]: text = text or "" @@ -370,14 +384,28 @@ def extract(self, text: str, *, context: str = "") -> list[ExtractedFact]: continue leaf = heading_path.split(" > ")[-1] if heading_path else "" title = _defang(leaf or _first_line(content), 1_000) - facts.append(ExtractedFact(content=content, title=title[:200], - keywords=_keywords(content))) + facts.append(ExtractedFact( + content=content, + title=title[:200], + keywords=_keywords(content), + metadata={ + "chunking": { + "target_tokens": self.target_tokens, + "overlap_tokens": self.overlap_tokens, + "token_counter": self.token_counter_identity, + }, + }, + )) if len(facts) >= self.max_chunks: break # Never lose the write: an all-whitespace/degenerate parse falls back to the # whole text, exactly like PassthroughExtractor. return facts or [ExtractedFact(content=_defang(text, 100_000))] + def count_tokens(self, text: str) -> int: + """Expose the exact configured counter for eval and composition boundaries.""" + return self._tokens(text) + # ── internals ──────────────────────────────────────────────────────────── def _chunks(self, text: str) -> list[tuple[str, str]]: out: list[tuple[str, str]] = [] @@ -449,22 +477,27 @@ def _pack(self, body: str) -> list[str]: paras = [p.strip() for p in _PARA_SPLIT_RE.split(body) if p.strip()] chunks: list[str] = [] cur: list[str] = [] - cur_tokens = 0 for para in paras: - ptokens = estimate_tokens(para) - if cur and cur_tokens + ptokens > self.target_tokens: + ptokens = self._tokens(para) + proposed = "\n\n".join([*cur, para]) + if cur and self._tokens(proposed) > self.target_tokens: joined = "\n\n".join(cur) chunks.append(joined) tail = self._overlap_tail(joined) cur = [tail] if tail else [] - cur_tokens = estimate_tokens(tail) if tail else 0 if ptokens > self.target_tokens: + # A tail here is overlap from the chunk just emitted above. + # The oversized paragraph is split independently; emitting the + # tail alone would create a duplicate evidence-only memory. + cur = [] for group in self._split_paragraph(para): chunks.append(group) - cur, cur_tokens = [], 0 continue + if cur and self._tokens("\n\n".join([*cur, para])) > self.target_tokens: + # Overlap is best-effort. It must never make the next otherwise + # admissible paragraph violate the configured reader budget. + cur = [] cur.append(para) - cur_tokens += ptokens if cur: chunks.append("\n\n".join(cur)) return chunks @@ -474,24 +507,23 @@ def _split_paragraph(self, para: str) -> list[str]: sentences = [s for s in _SENTENCE_RE.split(para.strip()) if s] groups: list[str] = [] cur: list[str] = [] - cur_tokens = 0 for sent in sentences: - stokens = estimate_tokens(sent) + stokens = self._tokens(sent) if stokens > self.target_tokens: if cur: joined = " ".join(cur) groups.append(joined) - cur, cur_tokens = [], 0 + cur = [] groups.extend(self._split_oversized_sentence(sent)) continue - if cur and cur_tokens + stokens > self.target_tokens: + if cur and self._tokens(" ".join([*cur, sent])) > self.target_tokens: joined = " ".join(cur) groups.append(joined) tail = self._overlap_tail(joined) cur = [tail] if tail else [] - cur_tokens = estimate_tokens(tail) if tail else 0 + if cur and self._tokens(" ".join([*cur, sent])) > self.target_tokens: + cur = [] cur.append(sent) - cur_tokens += stokens if cur: groups.append(" ".join(cur)) return groups @@ -503,16 +535,13 @@ def _split_oversized_sentence(self, sentence: str) -> list[str]: memory limit cannot be stored whole, so split near whitespace (or hard-cut a single giant token) instead of silently truncating it in ``_defang``. """ - max_chars = max(64, self.target_tokens * 4) remaining = sentence.strip() parts: list[str] = [] while remaining: - if len(remaining) <= max_chars: + if self._tokens(remaining) <= self.target_tokens: parts.append(remaining) break - cut = remaining.rfind(" ", max_chars // 2, max_chars + 1) - if cut < 0: - cut = max_chars + cut = self._largest_fitting_prefix(remaining) part = remaining[:cut].strip() if part: parts.append(part) @@ -525,16 +554,56 @@ def _overlap_tail(self, text: str) -> str: return "" sentences = [s for s in _SENTENCE_RE.split(text.strip()) if s] tail: list[str] = [] - tokens = 0 for sent in reversed(sentences): - if tail and tokens + estimate_tokens(sent) > self.overlap_tokens: + proposed = " ".join([sent, *tail]) + if tail and self._tokens(proposed) > self.overlap_tokens: break tail.insert(0, sent) - tokens += estimate_tokens(sent) - if tokens >= self.overlap_tokens: + if self._tokens(" ".join(tail)) >= self.overlap_tokens: break return " ".join(tail).strip() + def _tokens(self, text: str) -> int: + """Count with the configured reader counter and reject invalid adapters.""" + value = self._count(text or "") + if type(value) is not int: + raise TypeError("chunk token counter must return a non-negative integer") + if value < 0: + raise ValueError("chunk token counter must return a non-negative integer") + return value + + def _largest_fitting_prefix(self, text: str) -> int: + """Find a whitespace-aligned prefix within the declared token budget. + + Tokenizers need not expose token offsets. A bounded binary search keeps the + chunker backend-agnostic; the final verification loop handles merge-sensitive + tokenizers whose count is not perfectly monotonic at every character boundary. + """ + low, high = 1, len(text) + best = 0 + while low <= high: + middle = (low + high) // 2 + if self._tokens(text[:middle]) <= self.target_tokens: + best = middle + low = middle + 1 + else: + high = middle - 1 + if best <= 0: + if self._tokens(text[:1]) <= self.target_tokens: + return 1 + raise ValueError( + "chunk token counter cannot fit one character within target_tokens" + ) + whitespace = text.rfind(" ", max(0, best // 2), best + 1) + cut = whitespace if whitespace > 0 else best + while cut > 0 and self._tokens(text[:cut].strip()) > self.target_tokens: + cut -= 1 + if cut <= 0: + raise ValueError( + "chunk token counter cannot fit one character within target_tokens" + ) + return cut + def _first_line(text: str) -> str: for line in text.splitlines(): @@ -574,22 +643,65 @@ def _loads_lenient(raw: str) -> dict: return {} -def get_extractor(kind: str = "none", llm: Any = None): +def _load_chunk_token_counter( + model: str, revision: Optional[str] = None, +) -> tuple[Callable[[str], int], str]: + """Load an explicitly configured Hugging Face tokenizer at the backend edge.""" + try: + from transformers import AutoTokenizer + except ImportError as exc: # pragma: no cover - optional dependency + raise RuntimeError( + "ENGRAPHIS_CHUNK_TOKENIZER_MODEL requires the optional transformers package" + ) from exc + kwargs: dict[str, Any] = {"trust_remote_code": False} + if revision: + kwargs["revision"] = revision + tokenizer = AutoTokenizer.from_pretrained(model, **kwargs) + + def count(text: str) -> int: + return len(tokenizer.encode(text or "", add_special_tokens=False)) + + identity = f"hf:{model}@{revision or 'unversioned'}" + count.identity = identity # type: ignore[attr-defined] + return count, identity + + +def get_extractor( + kind: str = "none", + llm: Any = None, + *, + token_counter: Optional[Callable[[str], int]] = None, + token_counter_identity: Optional[str] = None, +): """Factory mirroring ``get_embedder``/``get_vector_index``: config in, backend out. ``kind='chunk'`` returns the deterministic, offline ``ChunkingExtractor`` (knobs from - ``ENGRAPHIS_CHUNK_TOKENS``/``_OVERLAP``/``_MAX``). ``kind='llm'`` with no ``llm`` - builds the v1 multi-provider ``LLMClient`` from settings (heavy import gated here, - never in ``core/``). ``kind='llm_structured'`` returns a schema-validated extractor - with entity/relation extraction. Anything else — including an LLM kind with no usable + ``ENGRAPHIS_CHUNK_TOKENS``/``_OVERLAP``/``_MAX``). A caller can inject the reader's + token counter, or explicitly configure ``ENGRAPHIS_CHUNK_TOKENIZER_MODEL`` and an + optional immutable ``ENGRAPHIS_CHUNK_TOKENIZER_REVISION``. Heavy tokenizer imports + stay behind this backend factory and the default remains dependency-free. + ``kind='llm'`` with no ``llm`` builds the v1 multi-provider ``LLMClient`` from + settings. ``kind='llm_structured'`` returns a schema-validated extractor with + entity/relation extraction. Anything else — including an LLM kind with no usable client — returns the offline passthrough. """ kind = (kind or "none").lower() if kind == "chunk": + if token_counter is None: + tokenizer_model = os.environ.get("ENGRAPHIS_CHUNK_TOKENIZER_MODEL", "").strip() + tokenizer_revision = os.environ.get( + "ENGRAPHIS_CHUNK_TOKENIZER_REVISION", "" + ).strip() + if tokenizer_model: + token_counter, token_counter_identity = _load_chunk_token_counter( + tokenizer_model, tokenizer_revision or None, + ) return ChunkingExtractor( target_tokens=_env_int("ENGRAPHIS_CHUNK_TOKENS", CHUNK_TARGET_TOKENS), overlap_tokens=_env_int("ENGRAPHIS_CHUNK_OVERLAP", CHUNK_OVERLAP_TOKENS), max_chunks=_env_int("ENGRAPHIS_CHUNK_MAX", CHUNK_MAX), + token_counter=token_counter, + token_counter_identity=token_counter_identity, ) if kind == "llm_structured": if llm is None: diff --git a/engraphis/backends/sync_folder.py b/engraphis/backends/sync_folder.py index 9ca95f79..2978d33b 100644 --- a/engraphis/backends/sync_folder.py +++ b/engraphis/backends/sync_folder.py @@ -50,13 +50,16 @@ def _safe_name(name: object) -> str: class FolderTransport: """A ``SyncTransport`` backed by a shared filesystem directory. - ``root`` is created if missing. Only ``*.json`` files are treated as bundles, so - dropping a README or other files in the folder is harmless. + ``root`` is created if missing unless ``create`` is false. The latter is for + dry-run callers: a missing remote then behaves as an empty transport rather than + being created by an operation advertised as read-only. Only ``*.json`` files are + treated as bundles, so dropping a README or other files in the folder is harmless. """ - def __init__(self, root: str) -> None: + def __init__(self, root: str, *, create: bool = True) -> None: self.root = Path(root) - self.root.mkdir(parents=True, exist_ok=True) + if create: + self.root.mkdir(parents=True, exist_ok=True) def push(self, name: str, data: bytes) -> None: """Atomically write ``data`` to ``root/`` (temp + fsync + os.replace). @@ -175,7 +178,8 @@ def get_transport(kind: str = "folder", **kw): """Factory mirroring ``get_embedder``/``get_vector_index`` — select a transport by name so swapping the folder backend for the managed relay is a config change. - - ``folder`` (default): shared-directory sync. Requires ``root=``. + - ``folder`` (default): shared-directory sync. Requires ``root=``; + pass ``create=False`` for a read-only probe of a possibly missing folder. - ``relay``: the managed Cloud Sync transport (``EncryptedRelayTransport``). Requires ``base_url=`` and ``workspace_id=`` (use the workspace *name*, so every authorized device on the account shares one namespace); @@ -192,7 +196,7 @@ def get_transport(kind: str = "folder", **kw): root = kw.get("root") if not root: raise ValueError("folder transport requires root=") - return FolderTransport(root) + return FolderTransport(root, create=bool(kw.get("create", True))) if kind == "relay": base_url = kw.get("base_url") workspace_id = kw.get("workspace_id") diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js index a54d4f3d..fbd23d73 100644 --- a/engraphis/classic_assets/dashboard.js +++ b/engraphis/classic_assets/dashboard.js @@ -251,11 +251,10 @@ function renderAnalytics(a,isPortfolio){const t=a.totals||{},f=a.decay_forecast| function managedConsentHtml(feature){const automation=/automation/i.test(feature),featureKey=`managed_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const primary=hostedCta('pro',featureKey),annual=primary.kind==='account'?'':{label:'Annual Pro option',href:hostedPlanUrl('pro',false,'annual',`${featureKey}_annual`),kind:'subscribe'},actions=`${ctaLinkHtml(primary,'btn btn-primary',featureKey)}${annual.href?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':licAccessState()==='lapsed'?'Your subscription needs billing attention. Update billing to restore hosted insights and maintenance.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Subscribe to Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} function managedConsentRequired(error){return error&&error.status===409&&error.detail&&error.detail.code==='consent_required'} const CLOUD_SYNC_PRIVACY_COPY='Cloud Sync encrypts eligible shared-workspace changes end-to-end before they leave this device. Engraphis Cloud cannot read their contents; secret and session-scoped memories stay local.'; -const MANAGED_COMPUTE_PRIVACY_COPY='Engraphis Cloud must read the bounded snapshot you submit to produce results. It travels over HTTPS but is not end-to-end encrypted; secret and session-scoped memories stay local.'; const EXTERNAL_LLM_PRIVACY_COPY='Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; -async function confirmCloudTransfer(title,summary,submit,privacyCopy){return confirmAction(title,summary+'\n\nPrivacy: '+(privacyCopy||MANAGED_COMPUTE_PRIVACY_COPY),submit||'Continue')} +async function confirmCloudTransfer(title,summary,submit,privacyCopy){return confirmAction(title,summary+(privacyCopy?'\n\nPrivacy: '+privacyCopy:''),submit||'Continue')} const managedConsentHtmlBase=managedConsentHtml; -managedConsentHtml=function(feature){const privacyCopy=/cloud sync/i.test(feature)?CLOUD_SYNC_PRIVACY_COPY:MANAGED_COMPUTE_PRIVACY_COPY;return managedConsentHtmlBase(feature).replace('',`
Privacy, by design. ${esc(privacyCopy)}
`)}; +managedConsentHtml=function(feature){return managedConsentHtmlBase(feature)}; /* Only an unconfigured local installation may turn a 401 into trial signup. A revoked or expired Cloud session is also a 401, but ``trial.available`` is false there and it must remain a reconnect error instead of offering a trial the control plane rejects. */ @@ -408,7 +407,7 @@ async function doTimeline(){ /* audit */ async function loadAudit(){const el=document.getElementById('audit-body');el.innerHTML='';try{const d=await api('/audit?workspace='+encodeURIComponent(WS||'')+'&limit=200');const rows=d.entries||d.audit||[];if(!rows.length){el.innerHTML='
No governance actions recorded.
';return}el.innerHTML='
'+rows.map(r=>`
${esc(r.action||r.op||r.kind||'edit')}${esc(r.memory_id||r.target||r.detail||'')}${esc(r.actor||'')}${r.ts||r.at?fmtRel(r.ts||r.at):''}
`).join('')+'
'}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} -async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{const d=await api('/receipts?workspace='+encodeURIComponent(WS||'')+'&limit=500');const v=await api('/receipts/verify?workspace='+encodeURIComponent(WS||''));const rows=d.entries||[];el.innerHTML=`
Receipt chain ${v.valid?'verified':'invalid'}
${v.count||0} receipts · head ${esc((v.head||'').slice(0,24))}
`+(rows.length?'
'+rows.map(r=>`
${esc(r.operation||'operation')}${esc((r.hash||'').slice(0,20))} · ${esc(r.status||'ok')} · ${r.target_count||0} target(s)${r.ts_ms?fmtRel(r.ts_ms/1000):''}
`).join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} +async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{const q='workspace='+encodeURIComponent(WS||'');const [d,v,s]=await Promise.all([api('/receipts?'+q+'&limit=500'),api('/receipts/verify?'+q),api('/context-savings?'+q)]);const rows=d.entries||[],counters=s.by_token_counter||[];const savings=counters.map(x=>`
${esc(x.token_counter||'unknown')}${x.context_tokens||0} packed / ${x.source_tokens||0} retrieved-source tokens; ${x.saved_tokens||0} not injected (${((x.savings_ratio||0)*100).toFixed(1)}%)
`).join('');const savingCard=`
Packed context efficiency
${s.savings_receipt_count||0} packed recalls; this measures retrieved source versus injected context, grouped by token counter.
${savings||'
No complete context-usage receipts yet.
'}
`;el.innerHTML=savingCard+`
Receipt chain ${v.valid?'verified':'invalid'}
${v.count||0} receipts · head ${esc((v.head||'').slice(0,24))}
`+(rows.length?'
'+rows.map(r=>`
${esc(r.operation||'operation')}${esc((r.hash||'').slice(0,20))} · ${esc(r.status||'ok')} · ${r.target_count||0} target(s)${r.ts_ms?fmtRel(r.ts_ms/1000):''}
`).join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} async function downloadReceipts(){try{const d=await api('/receipts/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-receipts-'+(WS||'workspace')+'.json';a.click();URL.revokeObjectURL(a.href);toast('Privacy-safe receipts exported','ok')}catch(e){toast(e.message,'err')}} /* consolidate */ @@ -1160,7 +1159,7 @@ function loadGraphEngine(){ if(GRAPH_ENGINE_LOADING)return GRAPH_ENGINE_LOADING; GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{ const script=document.createElement('script'); - script.src='/v2-assets/engraphis-graph.js?v=20260728-reference-materials'; + script.src='/v2-assets/engraphis-graph.js?v=20260730-drag-stability'; /* A 200 that never registers the global is a corrupt/truncated asset, not a success — resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */ script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()}; diff --git a/engraphis/config.py b/engraphis/config.py index 5b8e1376..214f175e 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -615,8 +615,9 @@ class Settings: # Fact extraction on the v2 write path: "none" (default — store text as given), # "chunk" (deterministic, offline structure-aware chunking — knobs - # ENGRAPHIS_CHUNK_TOKENS/_OVERLAP/_MAX), or "llm" (distill raw text into discrete - # facts via the configured LLM before storing). + # ENGRAPHIS_CHUNK_TOKENS/_OVERLAP/_MAX and optional pinned + # ENGRAPHIS_CHUNK_TOKENIZER_MODEL/_REVISION), or "llm" (distill raw text into + # discrete facts via the configured LLM before storing). extractor: str = field(default_factory=lambda: _env("ENGRAPHIS_EXTRACTOR", "none").lower()) llm_provider: str = field( diff --git a/engraphis/core/consolidate.py b/engraphis/core/consolidate.py index 796cf57e..1796289e 100644 --- a/engraphis/core/consolidate.py +++ b/engraphis/core/consolidate.py @@ -61,6 +61,9 @@ TRANSIENT_TYPES = [MemoryType.WORKING, MemoryType.EPISODIC] # Types the optional local profile pass rolls up. DURABLE_TYPES = [MemoryType.EPISODIC, MemoryType.SEMANTIC] +# Session memories are private to the active task. A workspace/repo maintenance sweep has no +# session write context, so it must neither distill nor archive them. +MAINTENANCE_SCOPES = [Scope.REPO, Scope.WORKSPACE, Scope.USER] _DIGEST_SYSTEM_PROMPT = ( "You consolidate recurring episodic agent memories into one durable semantic fact. " @@ -123,13 +126,21 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, raise ValueError("supersede_sources requires structured=True") store = engine.store now = time.time() if now is None else now - flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id) + flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id, + scopes=MAINTENANCE_SCOPES) episodic = store.list_memories( _replace(flt, mtypes=[MemoryType.EPISODIC]), limit=DISTILL_SCAN_LIMIT) - clusters = _cluster_by_subject( - episodic, threshold=subject_jaccard, store=store, flt=flt, - ) + # A digest inherits its owner from its first source. Cluster only records that have + # the exact same owner, otherwise a workspace sweep could write one repo's digest with + # another repo's content (or mix scope visibility). + clusters = [ + cluster + for owner_memories in _partition_by_visibility_owner(episodic) + for cluster in _cluster_by_subject( + owner_memories, threshold=subject_jaccard, store=store, flt=flt, + ) + ] report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, "clusters_found": 0, "digests_created": [], "archived": [], @@ -248,6 +259,19 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, # ── internals ───────────────────────────────────────────────────────────────── +def _visibility_owner(memory: MemoryRecord) -> tuple[str, Optional[str], Optional[str]]: + """Exact visibility identity a derived memory is allowed to inherit.""" + return (Scope(memory.scope).value, memory.repo_id, memory.session_id) + + +def _partition_by_visibility_owner(memories: list[MemoryRecord]) -> list[list[MemoryRecord]]: + """Keep source sets from distinct scope/repo/session owners disjoint.""" + partitions: dict[tuple[str, Optional[str], Optional[str]], list[MemoryRecord]] = {} + for memory in memories: + partitions.setdefault(_visibility_owner(memory), []).append(memory) + return list(partitions.values()) + + def _cluster_by_subject( memories: list[MemoryRecord], *, threshold: float, store=None, flt: Optional[SearchFilter] = None, @@ -738,7 +762,8 @@ def consolidate_profiles(engine, *, workspace_id: str, repo_id: Optional[str] = """ store = engine.store now = time.time() if now is None else now - flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id) + flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id, + scopes=MAINTENANCE_SCOPES) report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, "entities_considered": 0, "profiles_created": [], "skipped_existing": 0} @@ -752,26 +777,27 @@ def consolidate_profiles(engine, *, workspace_id: str, repo_id: Optional[str] = if len(name) < PROFILE_MIN_NAME_LEN: continue pattern = _entity_pattern(name) - sources = [m for m in live if pattern.search(f"{m.title} {m.content}")] - if len(sources) < min_mentions: - continue - report["entities_considered"] += 1 - if any(_in_profile(store, m.id) for m in sources): - report["skipped_existing"] += 1 - continue - content = _build_profile_content(name, ent.ntype, sources, llm=llm) - t_before = sum(_mem_tokens(m) for m in sources) - t_after = estimate_tokens(content) - p_before += t_before - p_after += t_after - entry = {"entity": name, "etype": ent.ntype, "mentions": len(sources), - **_compaction(t_before, t_after, len(sources))} - if dry_run: - entry["would_profile"] = [m.id for m in sources] - else: - entry["id"] = _write_profile(engine, name, ent.ntype, sources, - content=content, now=now) - report["profiles_created"].append(entry) + matching = [m for m in live if pattern.search(f"{m.title} {m.content}")] + for sources in _partition_by_visibility_owner(matching): + if len(sources) < min_mentions: + continue + report["entities_considered"] += 1 + if any(_in_profile(store, m.id) for m in sources): + report["skipped_existing"] += 1 + continue + content = _build_profile_content(name, ent.ntype, sources, llm=llm) + t_before = sum(_mem_tokens(m) for m in sources) + t_after = estimate_tokens(content) + p_before += t_before + p_after += t_after + entry = {"entity": name, "etype": ent.ntype, "mentions": len(sources), + **_compaction(t_before, t_after, len(sources))} + if dry_run: + entry["would_profile"] = [m.id for m in sources] + else: + entry["id"] = _write_profile(engine, name, ent.ntype, sources, + content=content, now=now) + report["profiles_created"].append(entry) report["compaction"] = _compaction(p_before, p_after, len(report["profiles_created"])) return report diff --git a/engraphis/core/context.py b/engraphis/core/context.py index 04c20722..15b26f86 100644 --- a/engraphis/core/context.py +++ b/engraphis/core/context.py @@ -217,6 +217,17 @@ def _excerpt( if summary and self._summary_is_useful(summary, full, query_terms): if self._count(summary) <= max_tokens: return summary, summary != full, "summary" + # A summary can still be more evidence-dense than the source even + # when it does not fit in full. Prefer a sentence-aligned subset + # only when it retains the same safeguards required for replacing + # the source at all: query evidence and every source qualifier. + summary_excerpt = self._sentence_excerpt( + summary, query_terms, max_tokens + ) + if summary_excerpt and self._summary_is_useful( + summary_excerpt, full, query_terms + ): + return summary_excerpt, True, "summary_excerpt" if full and self._count(full) <= max_tokens: return full, False, ( diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index cf7fb818..9047c6ee 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -970,7 +970,8 @@ def recall(self, query: str, *, workspace_id: Optional[str] = None, mtypes: Optional[list] = None, as_of: Optional[float] = None, valid_at: Optional[float] = None, known_at: Optional[float] = None, k: int = 8, token_budget: Optional[int] = None, - retrieval_profile: str = "balanced", diagnostics: bool = False, + retrieval_profile: str = "balanced", candidate_depth: str = "fixed", + diagnostics: bool = False, reinforce: bool = False) -> RecallResult: flt = self._recall_filter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, @@ -983,6 +984,7 @@ def recall(self, query: str, *, workspace_id: Optional[str] = None, return self.recall_engine.recall( query, flt, k=k, reinforce=bool(reinforce) and not flt.historical, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, diagnostics=diagnostics, ) @@ -993,7 +995,8 @@ def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, valid_at: Optional[float] = None, known_at: Optional[float] = None, k: int = 8, llm=None, min_support: Optional[float] = None, token_budget: Optional[int] = None, - retrieval_profile: str = "balanced", diagnostics: bool = False, + retrieval_profile: str = "balanced", candidate_depth: str = "fixed", + diagnostics: bool = False, max_citations: int = 5, reinforce: bool = True): """Recall, then answer *strictly from* what was recalled — with citations and an explicit abstain when the evidence is too weak (``core.grounded``). Offline and @@ -1015,7 +1018,8 @@ def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, # irrelevant nearest-neighbours an off-topic query happened to surface. result = self.recall_engine.recall( query, flt, k=k, reinforce=False, token_budget=token_budget, - retrieval_profile=retrieval_profile, diagnostics=diagnostics, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + diagnostics=diagnostics, ) floor = _grounded.GROUNDED_SUPPORT_FLOOR if min_support is None else min_support answer = _grounded.build_grounded_answer(query, result, self.embedder, llm=llm, diff --git a/engraphis/core/grounded.py b/engraphis/core/grounded.py index 179c3b59..6e8397f4 100644 --- a/engraphis/core/grounded.py +++ b/engraphis/core/grounded.py @@ -81,6 +81,10 @@ class GroundedAnswer: known_at: Optional[float] = None historical: bool = False retrieval_profile: str = "balanced" + candidate_depth: str = "fixed" + candidate_k_requested: int = 50 + candidate_k_used: int = 50 + candidate_depth_reason: str = "fixed requested depth" retrieval_trace: Optional[list[dict]] = None def to_dict(self) -> dict: @@ -98,6 +102,10 @@ def to_dict(self) -> dict: "known_at": self.known_at, "historical": self.historical, "retrieval_profile": self.retrieval_profile, + "candidate_depth": self.candidate_depth, + "candidate_k_requested": self.candidate_k_requested, + "candidate_k_used": self.candidate_k_used, + "candidate_depth_reason": self.candidate_depth_reason, } if self.retrieval_trace is not None: payload["retrieval_trace"] = self.retrieval_trace @@ -287,6 +295,10 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, "known_at": result.known_at, "historical": result.historical, "retrieval_profile": result.retrieval_profile, + "candidate_depth": result.candidate_depth_mode, + "candidate_k_requested": result.candidate_k_requested, + "candidate_k_used": result.candidate_k_used, + "candidate_depth_reason": result.candidate_depth_reason, "retrieval_trace": result.retrieval_trace, } recall_metadata["usage"]["answer_tokens"] = 0 diff --git a/engraphis/core/interfaces.py b/engraphis/core/interfaces.py index 847161e6..fd1e26e6 100644 --- a/engraphis/core/interfaces.py +++ b/engraphis/core/interfaces.py @@ -299,6 +299,13 @@ class RetrievalPolicy(Protocol): def profile(self, query: str) -> str: ... +@runtime_checkable +class CandidateDepthPolicy(Protocol): + """Select a bounded per-arm candidate depth for one recall request.""" + def candidate_depth(self, query: str, *, k: int, ceiling: int, + profile: str, mode: str) -> tuple[int, str]: ... + + @runtime_checkable class LLM(Protocol): """External or local model for synthesis and structured extraction (§8.2).""" diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py index 77bce3f9..012fb3ae 100644 --- a/engraphis/core/recall.py +++ b/engraphis/core/recall.py @@ -24,6 +24,7 @@ Candidate, ContextPacker, ContextUsage, + CandidateDepthPolicy, MemoryRecord, PackedChunk, Reranker, @@ -31,6 +32,7 @@ SearchFilter, ) from engraphis.core.retrieval_policy import ( + CANDIDATE_DEPTH_MODES, DeterministicRetrievalPolicy, ProfileConfig, RETRIEVAL_PROFILES, @@ -50,6 +52,10 @@ class RecallResult: known_at: Optional[float] = None historical: bool = False retrieval_profile: str = "balanced" + candidate_depth_mode: str = "fixed" + candidate_k_requested: int = 50 + candidate_k_used: int = 50 + candidate_depth_reason: str = "fixed requested depth" retrieval_trace: Optional[list[dict[str, Any]]] = None token_counter: Optional[Callable[[str], int]] = field(default=None, repr=False) @@ -59,7 +65,8 @@ def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Rera *, weights: Optional[dict] = None, recency_tau_days: float = 30.0, token_budget: int = 1500, graph_mode: str = "ppr", context_packer: Optional[ContextPacker] = None, - retrieval_policy: Optional[RetrievalPolicy] = None) -> None: + retrieval_policy: Optional[RetrievalPolicy] = None, + candidate_depth_policy: Optional[CandidateDepthPolicy] = None) -> None: self.store = store self.embedder = embedder self.index = vector_index @@ -69,6 +76,7 @@ def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Rera self.token_budget = token_budget self.context_packer = context_packer or DeterministicContextPacker() self.retrieval_policy = retrieval_policy or DeterministicRetrievalPolicy() + self.candidate_depth_policy = candidate_depth_policy or DeterministicRetrievalPolicy() # "ppr" (default) = Personalized PageRank over entities+links (multi-hop); # "1hop" = the Phase-1 entity expansion, kept for fallback and ablation. self.graph_mode = graph_mode @@ -77,6 +85,7 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, candidate_k: int = 50, reinforce: bool = False, token_budget: Optional[int] = None, retrieval_profile: str = "balanced", + candidate_depth: str = "fixed", diagnostics: bool = False, arm_config: Optional[ProfileConfig] = None) -> RecallResult: flt = flt or SearchFilter() @@ -105,6 +114,19 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, if requested_profile == "auto" else requested_profile ) + requested_depth_mode = str(candidate_depth or "fixed").strip().casefold() + if requested_depth_mode not in CANDIDATE_DEPTH_MODES: + choices = ", ".join(sorted(CANDIDATE_DEPTH_MODES)) + raise ValueError(f"candidate_depth must be one of: {choices}") + requested_candidate_k = max(1, int(candidate_k)) + candidate_k, candidate_depth_reason = self.candidate_depth_policy.candidate_depth( + query, + k=max(1, int(k)), + ceiling=requested_candidate_k, + profile=selected_profile, + mode=requested_depth_mode, + ) + candidate_k = max(1, min(requested_candidate_k, int(candidate_k))) # ``arm_config`` is a composition-time override for controlled offline # ablations. Normal callers still use only named RetrievalPolicy profiles, # so benchmark labels do not expand the public routing contract. @@ -150,6 +172,10 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, known_at=flt.known_at, historical=requested_historical, retrieval_profile=selected_profile, + candidate_depth_mode=requested_depth_mode, + candidate_k_requested=requested_candidate_k, + candidate_k_used=candidate_k, + candidate_depth_reason=candidate_depth_reason, retrieval_trace=[] if diagnostics else None, token_counter=getattr(self.context_packer, "count_tokens", None), ) @@ -296,6 +322,10 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, known_at=flt.known_at, historical=requested_historical, retrieval_profile=selected_profile, + candidate_depth_mode=requested_depth_mode, + candidate_k_requested=requested_candidate_k, + candidate_k_used=candidate_k, + candidate_depth_reason=candidate_depth_reason, retrieval_trace=trace, token_counter=getattr(self.context_packer, "count_tokens", None), ) diff --git a/engraphis/core/retrieval_policy.py b/engraphis/core/retrieval_policy.py index dbf93750..90a3cdf6 100644 --- a/engraphis/core/retrieval_policy.py +++ b/engraphis/core/retrieval_policy.py @@ -12,6 +12,7 @@ RETRIEVAL_PROFILES = frozenset({"balanced", "auto", "lexical", "graph", "code"}) +CANDIDATE_DEPTH_MODES = frozenset({"fixed", "adaptive"}) _CODE_RE = re.compile( r"(?:\w+[./\\])+\w+|::|->|\b(?:class|def|function|import|module)\b|" @@ -90,3 +91,41 @@ def resolve(self, requested: str, query: str) -> ProfileConfig: raise ValueError(f"retrieval_profile must be one of: {choices}") selected = self.profile(query) if normalized == "auto" else normalized return profile_config(selected) + + def candidate_depth( + self, + query: str, + *, + k: int, + ceiling: int, + profile: str, + mode: str, + ) -> tuple[int, str]: + """Return a deterministic bounded candidate depth and its explanation. + + ``fixed`` preserves the historical depth exactly. The explicit ``adaptive`` + mode reduces routine lexical/balanced recalls but deliberately retains a + wider pool when the selected profile depends on graph traversal or code + bridges. It is a per-arm cap, not a result-count change. + """ + del query # The selected profile already captures the stable query signals. + limit = max(1, int(ceiling)) + requested_mode = str(mode or "fixed").strip().casefold() + if requested_mode not in CANDIDATE_DEPTH_MODES: + choices = ", ".join(sorted(CANDIDATE_DEPTH_MODES)) + raise ValueError(f"candidate_depth must be one of: {choices}") + if requested_mode == "fixed": + return limit, "fixed requested depth" + + # Lower bounds are intentionally tied to output k. The graph/code floors + # remain larger because their useful evidence may enter through a bridge + # that is not top-ranked by the first retrieval arm. + floors = { + "lexical": max(8, k * 2), + "balanced": max(12, k * 3), + "graph": max(30, k * 6), + "code": max(30, k * 6), + } + selected = str(profile or "balanced").strip().casefold() + depth = min(limit, floors.get(selected, max(12, k * 3))) + return depth, f"adaptive {selected} floor" diff --git a/engraphis/core/store.py b/engraphis/core/store.py index ffd377ae..a64dcd47 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -208,6 +208,7 @@ def _edge_support_confidence(provenance: Any, source_kind: str) -> float: }, "layer": {"temporal", "entity", "causal", "semantic"}, "retrieval_profile": {"balanced", "auto", "lexical", "graph", "code"}, + "candidate_depth": {"fixed", "adaptive"}, "response_mode": {"full", "compact"}, } @@ -220,7 +221,8 @@ def _receipt_metadata(metadata: dict) -> dict: "files_scanned", "files_indexed", "files_removed", "symbols", "edges", "entities", "relations", "tables", "dry_run", "error_count", "entities_added", "relations_added", - "retrieval_profile", "response_mode", "historical", "token_usage", + "retrieval_profile", "candidate_depth", "candidate_k_requested", + "candidate_k_used", "response_mode", "historical", "token_usage", } def content_free_label(key: str, value: str) -> str: normalized = value.strip().casefold().replace(" ", "_") @@ -279,8 +281,9 @@ def content_free_label(key: str, value: str) -> str: "result_count", "grounded", "citations", "relation", "layer", "graph_layers", "files_scanned", "files_indexed", "files_removed", "symbols", "edges", "entities", "relations", "tables", "dry_run", "error_count", - "entities_added", "relations_added", "retrieval_profile", "response_mode", - "historical", "token_usage", + "entities_added", "relations_added", "retrieval_profile", "candidate_depth", + "candidate_k_requested", "candidate_k_used", "response_mode", "historical", + "token_usage", } _PUBLIC_RECEIPT_OPERATIONS = { "remember", "recall", "promote", "link", "index_repo", @@ -293,6 +296,13 @@ def content_free_label(key: str, value: str) -> str: } +def _receipt_scope_digest(workspace_id: str, repo_id: Optional[str]) -> str: + """Return the signed scope binding for an operation receipt.""" + return hashlib.sha256( + f"{workspace_id}\0{repo_id or ''}".encode("utf-8") + ).hexdigest()[:24] + + def _redacted_receipt_value(value: Any) -> str: raw = value if isinstance(value, str) else str(value or "") return "redacted_sha256:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() @@ -4009,9 +4019,7 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", transaction_started = True ts = now_ts() receipt_id = ids.new_id("receipt") - scope_digest = hashlib.sha256( - f"{workspace_id}\0{repo_id}".encode("utf-8") - ).hexdigest()[:24] + scope_digest = _receipt_scope_digest(workspace_id, repo_id) actor_digest = hashlib.sha256(actor.encode("utf-8")).hexdigest()[:16] anchor = self.conn.execute( "SELECT receipt_count, head_hash, integrity_error " @@ -4145,6 +4153,132 @@ def list_receipts(self, *, workspace_id: str, limit: int = 100) -> list[dict]: ).fetchall() return [_public_receipt_row(dict(row)) for row in rows] + def context_savings(self, *, workspace_id: str, repo_id: Optional[str] = None) -> dict: + """Aggregate validated, content-free context usage from scoped receipts. + + Token counts are kept separate by counter identity: a tokenizer change must not turn + into a misleading cumulative total. Invalid, missing, and incomplete receipts remain + visible only as counts; their payload is never reflected into this summary. The + workspace-wide receipt-chain validity is returned alongside any repo-scoped aggregate + so callers can distinguish useful local accounting from evidence eligible for audit. + """ + verification = self.verify_receipts(workspace_id=workspace_id) + where = "workspace_id=?" + params: list[str] = [workspace_id] + if repo_id is not None: + where += " AND repo_id=?" + params.append(repo_id) + rows = self.conn.execute( + "SELECT id, repo_id, payload, prev_hash, receipt_hash FROM operation_receipts WHERE " + where, + params, + ).fetchall() + totals = { + "receipt_count": len(rows), + "usage_receipt_count": 0, + "savings_receipt_count": 0, + "invalid_receipt_count": 0, + "incomplete_usage_receipt_count": 0, + } + buckets: dict[str, dict] = {} + + def bucket(counter: str) -> dict: + return buckets.setdefault(counter, { + "token_counter": counter, + "receipt_count": 0, + "source_tokens": 0, + "context_tokens": 0, + "saved_tokens": 0, + "budget_tokens": 0, + "packed_count": 0, + "omitted_count": 0, + "_operations": {}, + }) + + def add(target: dict, usage: dict, operation: str) -> None: + target["receipt_count"] += 1 + for key in ( + "source_tokens", "context_tokens", "saved_tokens", "budget_tokens", + "packed_count", "omitted_count", + ): + value = usage.get(key) + if type(value) in (int, float) and value >= 0: + target[key] += value + operation_totals = target["_operations"].setdefault(operation, { + "operation": operation, + "receipt_count": 0, + "source_tokens": 0, + "context_tokens": 0, + "saved_tokens": 0, + "budget_tokens": 0, + "packed_count": 0, + "omitted_count": 0, + }) + operation_totals["receipt_count"] += 1 + for key in ( + "source_tokens", "context_tokens", "saved_tokens", "budget_tokens", + "packed_count", "omitted_count", + ): + value = usage.get(key) + if type(value) in (int, float) and value >= 0: + operation_totals[key] += value + + def finished(target: dict) -> dict: + operations = target.pop("_operations") + target["savings_ratio"] = ( + target["saved_tokens"] / target["source_tokens"] + if target["source_tokens"] else 0.0 + ) + target["by_operation"] = [ + {**value, "savings_ratio": ( + value["saved_tokens"] / value["source_tokens"] + if value["source_tokens"] else 0.0 + )} + for _, value in sorted(operations.items()) + ] + return target + + for raw_row in rows: + receipt = _public_receipt_row(dict(raw_row)) + if ( + receipt.get("invalid_payload") + or receipt.get("scope_digest") + != _receipt_scope_digest(workspace_id, raw_row["repo_id"]) + ): + totals["invalid_receipt_count"] += 1 + continue + metadata = receipt.get("metadata") + usage = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(usage, dict): + continue + totals["usage_receipt_count"] += 1 + required = ("source_tokens", "context_tokens", "saved_tokens") + if not all( + type(usage.get(key)) in (int, float) and usage[key] >= 0 + for key in required + ): + totals["incomplete_usage_receipt_count"] += 1 + continue + expected_saved = max( + 0.0, float(usage["source_tokens"]) - float(usage["context_tokens"]) + ) + if not math.isclose( + float(usage["saved_tokens"]), expected_saved, rel_tol=0.0, abs_tol=1e-9 + ): + totals["incomplete_usage_receipt_count"] += 1 + continue + totals["savings_receipt_count"] += 1 + add( + bucket(str(usage.get("token_counter") or "unknown")), + usage, + str(receipt["operation"]), + ) + return { + **totals, + "receipt_chain_valid": bool(verification["valid"]), + "receipt_chain_error_count": len(verification["errors"]), + "by_token_counter": [finished(value) for _, value in sorted(buckets.items())], + } + def verify_receipts(self, *, workspace_id: str, expected_head: str = "", expected_count: Optional[int] = None) -> dict: chain = self._receipt_chain_state(workspace_id) diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 7a08bd6b..1d23c156 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -79,6 +79,7 @@ MAX_REPO_NAME_CHARS = 200 TS_FUTURE_SKEW = 2 * 86400 # tolerate 2 days of cross-device clock skew, no more _VALID_SENSITIVITY = ("normal", "sensitive", "secret") +_VALID_SCOPES = frozenset(scope.value for scope in Scope) # Strip C0/C1 control + ANSI-escape bytes (keep \t\n\r) — the same defense the rest of # the ingest surface applies (service.py) against hidden-instruction / terminal-injection @@ -668,10 +669,31 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, report["rejected"] += 1 return rec.session_id = None + remote_repo_id = d.get("repo_id") + raw_scope = d.get("scope") + if raw_scope is None: + # Sync v1 allowed callers to omit scope. Preserve that compatibility while + # canonicalizing the row: a repo pointer means repo scope; otherwise the row + # belongs to the workspace. Never persist the old invalid repo-without-owner + # default produced by ``_scope(None)``. + rec.scope = Scope.REPO if remote_repo_id is not None else Scope.WORKSPACE + elif not isinstance(raw_scope, str) or raw_scope not in _VALID_SCOPES: + report["rejected"] += 1 + return + # Scope pointers are an untrusted trust-boundary input, not merely metadata. + # A repo-scoped row must name one of the bundle's repos; workspace/user rows + # must not carry a repo pointer. Accepting an invalid combination and then + # re-homing it would turn a repo-owned row into an ancestor-visible global row. + if rec.scope == Scope.REPO: + if not isinstance(remote_repo_id, str) or not remote_repo_id: + report["rejected"] += 1 + return + elif remote_repo_id is not None: + report["rejected"] += 1 + return # Re-home into local scope, and tag provenance with the origin device so a # synced-in memory stays auditable ("why is this known?" — AGENTS.md §3.6). rec.workspace_id = local_ws - remote_repo_id = d.get("repo_id") if remote_repo_id: if remote_repo_id not in repo_remap: report["rejected"] += 1 @@ -707,6 +729,18 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, # overwrite the local private row with a non-session scope either. report["rejected"] += 1 return + if (existing is not None + and (existing.scope != rec.scope or existing.repo_id != rec.repo_id)): + # ``merge_record`` deliberately keeps scope pointers local. Letting the + # descriptive LWW winner change ``scope`` while retaining the existing local + # pointer would therefore create an impossible row (for example a + # workspace-scoped memory still attached to a repo), and could make a + # repo-owned fact ancestor-visible. Scope promotion is a local, explicit + # operation; a sync peer may merge a record only at its existing visibility. + # This also fails closed for malformed legacy rows: repairing an orphaned + # scope is a local migration decision, never authority delegated to a peer. + report["rejected"] += 1 + return if existing is not None: # Sync v1 bundles predate durable claim identity. Omission means # "unknown to this peer", not an instruction to erase local keys. diff --git a/engraphis/dashboard_assets/engraphis-graph.js b/engraphis/dashboard_assets/engraphis-graph.js index aa56d250..096aa4bc 100644 --- a/engraphis/dashboard_assets/engraphis-graph.js +++ b/engraphis/dashboard_assets/engraphis-graph.js @@ -78,6 +78,13 @@ const FULL_FORCE_NODE_LIMIT = LARGE_NODE_LIMIT; const FULL_FORCE_LINK_LIMIT = LARGE_LINK_LIMIT; + /* `zoomToFit()` derives its bounds from force-graph's default node geometry rather than + our custom canvas radius. A compact, nearly-linear graph can therefore produce a 10×+ + fit zoom even though its rendered nodes already fill the canvas. At that scale a normal + drag maps to a tiny world-space movement and reheating makes the rest of the layout look + like it is racing away. Keep auto-fit useful without letting its scale become unstable. */ + const MAX_AUTO_FIT_ZOOM = 4; + /* The classic renderer's *dense* signal (`GPERF.dense`, `links>1500` in dashboard.js). Past it the classic path turns off the two per-edge costs that scale with the link count and buy nothing at that density: link curvature (a quadratic bezier per relation instead of a @@ -876,6 +883,20 @@ const fg = ForceGraph()(el); const api = {}; + function autoFit(duration, padding) { + const bbox = fg.getGraphBbox && fg.getGraphBbox(); + const width = el.clientWidth, height = el.clientHeight; + if (!bbox || !bbox.x || !bbox.y || !Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return; + const xSpan = bbox.x[1] - bbox.x[0], ySpan = bbox.y[1] - bbox.y[0]; + if (!Number.isFinite(xSpan) || !Number.isFinite(ySpan)) return; + const zoom = Math.min(MAX_AUTO_FIT_ZOOM, Math.max( + 1e-12, + Math.min((width - 2 * padding) / Math.max(xSpan, 1e-12), (height - 2 * padding) / Math.max(ySpan, 1e-12)), + )); + fg.centerAt((bbox.x[0] + bbox.x[1]) / 2, (bbox.y[0] + bbox.y[1]) / 2, duration); + fg.zoom(zoom, duration); + } + function suppressNodeClick() { suppressNodeClickAfterDrag = true; cancelFrame(dragClickFrame); @@ -1438,12 +1459,29 @@ if (reused) invalidate(); if (fit) { clearTimeout(fitTimer); - fitTimer = setTimeout(() => { if (!destroyed) fg.zoomToFit(motion ? 600 : 0, 40); }, motion ? 320 : 0); + fitTimer = setTimeout(() => { if (!destroyed) autoFit(motion ? 600 : 0, 40); }, motion ? 320 : 0); } if (opts.onStats) opts.onStats({ nodes: data.nodes.length, links: data.links.length, total: raw.nodes.length, totalLinks: raw.links.length, preset: (PRESETS[state.settings.mode] || PRESETS.compact).label, collapsed: collapsed, ghosts: data.nodes.filter(n => n.ghost).length, bridges: data.links.filter(l => l.bridge).length, suggested: data.links.filter(l => l.suggested).length }); } - fg.backgroundColor('rgba(0,0,0,0)').nodeRelSize(1).autoPauseRedraw(true) + function handleNodeClick(node) { + if (suppressNodeClickAfterDrag) { + suppressNodeClickAfterDrag = false; + return; + } + if (node.cluster) { + collapsed = false; + state.collapse = false; + render(false, true); + setTimeout(() => { fg.centerAt(node.x, node.y, 500); fg.zoom(1.6, 500); }, 60); + if (opts.onCollapseChange) opts.onCollapseChange(false); + return; + } + if (opts.onNodeClick) opts.onNodeClick(node); + } + + fg.backgroundColor('rgba(0,0,0,0)').nodeRelSize(1) + .enableNodeDrag(false).autoPauseRedraw(true) /* force-graph's default `nodeLabel`/`linkLabel` is the literal accessor "name", and its tooltip renders a string label with innerHTML. Node names here are entity labels extracted from ingested memories — untrusted input — so both accessors are set @@ -1486,14 +1524,9 @@ el.classList.toggle('engraphis-graph-node-hover', !!node); invalidate(); }) - .onNodeClick(node => { - if (suppressNodeClickAfterDrag) { - suppressNodeClickAfterDrag = false; - return; - } - if (node.cluster) { collapsed = false; state.collapse = false; render(false, true); setTimeout(() => { fg.centerAt(node.x, node.y, 500); fg.zoom(1.6, 500); }, 60); if (opts.onCollapseChange) opts.onCollapseChange(false); return; } - if (opts.onNodeClick) opts.onNodeClick(node); - }) + .onNodeClick(handleNodeClick) + // Kept as the pinning contract for embedders that opt back into vendor dragging; + // Ledger itself disables that path and uses the scoped pointer controller below. .onNodeDragEnd(node => { node.fx = node.x; node.fy = node.y; suppressNodeClick(); }) .onBackgroundClick(() => { if (opts.onBackgroundClick) opts.onBackgroundClick(); }) .onZoom(z => { @@ -1507,6 +1540,97 @@ } }); + /* force-graph's built-in drag always reheats the entire simulation. Ledger treats manual + placement as a pin, so install a small scoped drag controller and leave global physics + changes to the explicit Reheat control. Capturing pointer-down prevents the vendor's + drag handler from seeing node gestures while preserving its background pan/zoom path. */ + let detachManualDrag = null; + if (typeof window !== 'undefined' && typeof window.addEventListener === 'function' + && typeof el.addEventListener === 'function' && typeof el.querySelector === 'function') { + let manualDrag = null; + const graphPoint = event => { + const canvas = el.querySelector('canvas'); + if (!canvas || !canvas.getBoundingClientRect || !fg.screen2GraphCoords) return null; + const box = canvas.getBoundingClientRect(); + return fg.screen2GraphCoords(event.clientX - box.left, event.clientY - box.top); + }; + const endManualDrag = event => { + if (!manualDrag || (event.pointerId != null && event.pointerId !== manualDrag.pointerId)) return; + const current = manualDrag; + manualDrag = null; + window.removeEventListener('pointermove', moveManualDrag, true); + window.removeEventListener('pointerup', endManualDrag, true); + window.removeEventListener('pointercancel', endManualDrag, true); + if (current.dragged) { + current.node.fx = current.node.x; + current.node.fy = current.node.y; + current.node.vx = 0; + current.node.vy = 0; + suppressNodeClick(); + } else if (event.type !== 'pointercancel') { + // Our capture listener owns the direct click. Suppress force-graph's + // later pointer-up callback only after dispatching this click ourselves. + handleNodeClick(current.node); + suppressNodeClick(); + } + }; + const moveManualDrag = event => { + if (!manualDrag || event.pointerId !== manualDrag.pointerId) return; + const point = graphPoint(event); + if (!point || !Number.isFinite(point.x) || !Number.isFinite(point.y)) return; + const dx = event.clientX - manualDrag.startClientX; + const dy = event.clientY - manualDrag.startClientY; + if (!manualDrag.dragged) { + if (Math.hypot(dx, dy) < 3) { + event.preventDefault(); + event.stopPropagation(); + return; + } + manualDrag.dragged = true; + } + const node = manualDrag.node; + node.x = node.fx = point.x + manualDrag.offsetX; + node.y = node.fy = point.y + manualDrag.offsetY; + node.vx = 0; + node.vy = 0; + invalidate(); + event.preventDefault(); + event.stopPropagation(); + }; + const beginManualDrag = event => { + if (event.button !== 0 || event.isPrimary === false) return; + const point = graphPoint(event); + if (!point) return; + let candidate = null; + let distance = Infinity; + (fg.graphData().nodes || []).forEach(node => { + if (!Number.isFinite(node.x) || !Number.isFinite(node.y)) return; + const d = Math.hypot(node.x - point.x, node.y - point.y); + const hitRadius = (node.radius || 1) + 5 / Math.max(zoom, 0.1); + if (d <= hitRadius && d < distance) { candidate = node; distance = d; } + }); + if (!candidate) return; + manualDrag = { + node: candidate, pointerId: event.pointerId, startClientX: event.clientX, + startClientY: event.clientY, offsetX: candidate.x - point.x, + offsetY: candidate.y - point.y, dragged: false, + }; + window.addEventListener('pointermove', moveManualDrag, true); + window.addEventListener('pointerup', endManualDrag, true); + window.addEventListener('pointercancel', endManualDrag, true); + event.preventDefault(); + event.stopPropagation(); + }; + el.addEventListener('pointerdown', beginManualDrag, true); + detachManualDrag = () => { + manualDrag = null; + el.removeEventListener('pointerdown', beginManualDrag, true); + window.removeEventListener('pointermove', moveManualDrag, true); + window.removeEventListener('pointerup', endManualDrag, true); + window.removeEventListener('pointercancel', endManualDrag, true); + }; + } + api.setData = data => { const inputNodes = Array.isArray(data && data.nodes) ? data.nodes : []; const nodes = inputNodes @@ -1802,6 +1926,7 @@ clearTimeout(fitTimer); cancelFrame(dragClickFrame); try { + if (detachManualDrag) { detachManualDrag(); detachManualDrag = null; } if (api._ro) { api._ro.disconnect(); api._ro = null; } // `_destructor` pauses the rAF and drops the graph data; it does not detach the // canvas, so clear the container too or a re-create leaves the old one attached. @@ -1825,7 +1950,7 @@ if (w > 0 && h > 0) fg.width(w).height(h); }; measure(); - requestAnimationFrame(() => { if (destroyed) return; measure(); fg.zoomToFit(reduced() ? 0 : 400, 40); }); + requestAnimationFrame(() => { if (destroyed) return; measure(); autoFit(reduced() ? 0 : 400, 40); }); if (typeof ResizeObserver !== 'undefined') { api._ro = new ResizeObserver(() => measure()); api._ro.observe(el); diff --git a/engraphis/dashboard_assets/ledger.js b/engraphis/dashboard_assets/ledger.js index 5db73011..33d5778f 100644 --- a/engraphis/dashboard_assets/ledger.js +++ b/engraphis/dashboard_assets/ledger.js @@ -47,7 +47,6 @@ const text = value => value == null ? '' : String(value); const number = value => Number.isFinite(Number(value)) ? Number(value) : 0; const CLOUD_SYNC_PRIVACY_NOTICE = 'Cloud Sync encrypts eligible shared-workspace changes end-to-end before they leave this device. Engraphis Cloud cannot read their contents; secret and session-scoped memories stay local.'; - const MANAGED_COMPUTE_PRIVACY_NOTICE = 'For managed compute, Engraphis Cloud must read the bounded snapshot you submit to produce results. It travels over HTTPS; secret and session-scoped memories stay local.'; const EXTERNAL_LLM_PRIVACY_NOTICE = 'Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; const truncate = (value, length = 260) => { const source = text(value).trim(); @@ -227,7 +226,7 @@ '/v2-assets/vendor/force-graph.min.js?v=20260727-final', 'ForceGraph', )).then(() => loadScript( - '/v2-assets/engraphis-graph.js?v=20260728-connected-memories', + '/v2-assets/engraphis-graph.js?v=20260730-drag-stability', 'EngraphisGraph', )); graphAssetsPromise.catch(() => {}); @@ -2329,7 +2328,7 @@ automationNumber('automation-dream-min', 'Minimum new memories', Math.max(1, Number(policy.dream_min_new) || 25), 1, 100000), automationNumber('automation-dream-idle', 'Idle minutes before Dreaming', Math.max(0, Number(policy.dream_idle_minutes) || 0), 0, 10080), automationCheckbox('automation-infer', 'Allow hosted relationship inference proposals', policy.infer), - node('p', 'automation-policy-note', `Cloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE} Managed compute: saving an enabled policy submits a bounded snapshot of this workspace’s normal and sensitive memory content to Engraphis Cloud. ${MANAGED_COMPUTE_PRIVACY_NOTICE} Cloud work returns proposals and never silently changes the local database.`), + node('p', 'automation-policy-note', `Cloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE} Managed compute: saving an enabled policy submits a bounded snapshot of this workspace’s normal and sensitive memory content to Engraphis Cloud. Cloud work returns proposals and never silently changes the local database.`), ); const actions = node('div', 'automation-policy-actions'); const save = node('button', 'primary-button', enabled ? 'Save & send policy to Cloud' : 'Save hosted policy'); @@ -2352,7 +2351,7 @@ infer: byId('automation-infer').checked, }; if (policy.enabled && !window.confirm( - `Save this hosted policy for ${state.workspace}? Engraphis will submit a bounded snapshot of that workspace’s normal and sensitive memory content to Cloud for managed compute.\n\nCloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE}\n\nManaged compute: ${MANAGED_COMPUTE_PRIVACY_NOTICE}`, + `Save this hosted policy for ${state.workspace}? Engraphis will submit a bounded snapshot of that workspace’s normal and sensitive memory content to Cloud for managed compute.\n\nCloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE}`, )) return; const save = form.querySelector('button[type="submit"]'); if (save) { diff --git a/engraphis/inspector/app.py b/engraphis/inspector/app.py index 532d1f3e..c19e3310 100644 --- a/engraphis/inspector/app.py +++ b/engraphis/inspector/app.py @@ -252,6 +252,10 @@ async def audit_log(workspace: str, limit: int = 100): async def receipts(workspace: str, limit: int = 100): return svc().receipt_log(workspace=workspace, limit=limit) + @app.get("/api/context-savings") + async def context_savings(workspace: str, repo: Optional[str] = None): + return svc().context_savings(workspace=workspace, repo=repo) + @app.get("/api/receipts/verify") async def receipts_verify(workspace: str): return svc().verify_receipts(workspace=workspace) diff --git a/engraphis/mcp_server.py b/engraphis/mcp_server.py index 07756801..b29bed68 100644 --- a/engraphis/mcp_server.py +++ b/engraphis/mcp_server.py @@ -118,6 +118,7 @@ def _err(exc: Exception) -> str: "engraphis_code_impact", "engraphis_export_code_graph", "engraphis_receipts", + "engraphis_context_savings", "engraphis_verify_receipts", "engraphis_export_receipts", "engraphis_stats", @@ -264,6 +265,9 @@ def engraphis_recall( retrieval_profile: Annotated[str, Field( description="Retrieval profile: balanced (legacy hybrid), auto, lexical, graph, " "or code. Auto is opt-in until benchmarks demonstrate a win.")] = "balanced", + candidate_depth: Annotated[str, Field( + description="Candidate depth: fixed preserves the legacy pool; adaptive is an opt-in " + "profile-aware performance experiment.")] = "fixed", response_mode: Annotated[str, Field( description="full preserves legacy memory bodies; compact omits bodies already " "represented in the packed context.")] = "full", @@ -289,7 +293,8 @@ def engraphis_recall( query, workspace=workspace, repo=repo, session_id=session_id, mtypes=mtypes, k=k, as_of=as_of, valid_at=valid_at, known_at=known_at, token_budget=token_budget, - retrieval_profile=retrieval_profile, response_mode=response_mode, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + response_mode=response_mode, diagnostics=diagnostics, )) except Exception as exc: # noqa: BLE001 @@ -319,6 +324,8 @@ def engraphis_recall_context( ge=0, le=32_768)] = 1024, retrieval_profile: Annotated[str, Field( description="balanced, auto, lexical, graph, or code.")] = "balanced", + candidate_depth: Annotated[str, Field( + description="fixed preserves the legacy pool; adaptive is profile-aware and opt-in.")] = "fixed", as_of: Annotated[Optional[float], Field( description="Compatibility alias for valid_at.")] = None, valid_at: Annotated[Optional[float], Field( @@ -348,6 +355,7 @@ def engraphis_recall_context( known_at=known_at, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, response_mode="compact", diagnostics=diagnostics, intent="recall_context", @@ -421,6 +429,8 @@ def engraphis_recall_grounded( description="Hard packed-context budget (0-32768).", ge=0, le=32_768)] = None, retrieval_profile: Annotated[str, Field( description="balanced, auto, lexical, graph, or code.")] = "balanced", + candidate_depth: Annotated[str, Field( + description="fixed preserves the legacy pool; adaptive is profile-aware and opt-in.")] = "fixed", response_mode: Annotated[str, Field( description="full includes citation bodies; compact omits bodies already present " "in the cited answer.")] = "full", @@ -457,7 +467,8 @@ def engraphis_recall_grounded( query, workspace=workspace, repo=repo, session_id=session_id, mtypes=mtypes, k=k, as_of=as_of, valid_at=valid_at, known_at=known_at, token_budget=token_budget, - retrieval_profile=retrieval_profile, response_mode=response_mode, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + response_mode=response_mode, diagnostics=diagnostics, min_support=min_support, llm=llm, )) except Exception as exc: # noqa: BLE001 @@ -497,6 +508,8 @@ def engraphis_answer( description="Hard packed-context budget (0-32768).", ge=0, le=32_768)] = None, retrieval_profile: Annotated[str, Field( description="balanced, auto, lexical, graph, or code.")] = "balanced", + candidate_depth: Annotated[str, Field( + description="fixed preserves the legacy pool; adaptive is profile-aware and opt-in.")] = "fixed", response_mode: Annotated[str, Field( description="full includes citation bodies; compact omits them.")] = "full", diagnostics: Annotated[bool, Field( @@ -511,6 +524,7 @@ def engraphis_answer( query=query, workspace=workspace, repo=repo, session_id=None, mtypes=None, k=k, as_of=as_of, valid_at=valid_at, known_at=known_at, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, response_mode=response_mode, diagnostics=diagnostics, min_support=min_support, synthesize=synthesize, ) @@ -1149,6 +1163,24 @@ def engraphis_receipts( return _err(exc) +@mcp.tool( + name="engraphis_context_savings", + annotations={"title": "Summarize context savings", "readOnlyHint": True, + "destructiveHint": False, "idempotentHint": True, "openWorldHint": False}, +) +def engraphis_context_savings( + workspace: Annotated[str, Field(description="Workspace whose receipt usage to summarize.", + min_length=1, max_length=200)], + repo: Annotated[Optional[str], Field(description="Optional repo scope within the workspace.", + max_length=200)] = None, +) -> str: + """Summarize content-free context savings, separated by token-counter identity.""" + try: + return _ok(service().context_savings(workspace=workspace, repo=repo)) + except Exception as exc: # noqa: BLE001 + return _err(exc) + + @mcp.tool( name="engraphis_verify_receipts", annotations={"title": "Verify an operation receipt chain", "readOnlyHint": True, diff --git a/engraphis/read_only_api.py b/engraphis/read_only_api.py index d6d64af9..5c9dff08 100644 --- a/engraphis/read_only_api.py +++ b/engraphis/read_only_api.py @@ -24,6 +24,7 @@ class IntentRecallRequest(BaseModel): known_at: Optional[float] = None token_budget: Optional[int] = None retrieval_profile: str = "balanced" + candidate_depth: str = "fixed" response_mode: str = "compact" diagnostics: bool = False @@ -90,12 +91,14 @@ def recall(query: str, workspace: Optional[str] = None, known_at: Optional[float] = None, token_budget: Optional[int] = None, retrieval_profile: str = "balanced", + candidate_depth: str = "fixed", response_mode: str = "compact", diagnostics: bool = False): return run( svc.recall, query, workspace=workspace, repo=repo, k=k, as_of=as_of, valid_at=valid_at, known_at=known_at, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, response_mode=response_mode, diagnostics=diagnostics, reinforce=False, intent="http_read_only", record_receipt=False, ) @@ -108,6 +111,7 @@ def intent_recall(req: IntentRecallRequest): k=req.k, as_of=req.as_of, valid_at=req.valid_at, known_at=req.known_at, token_budget=req.token_budget, retrieval_profile=req.retrieval_profile, + candidate_depth=req.candidate_depth, response_mode=req.response_mode, diagnostics=req.diagnostics, reinforce=False, record_receipt=False, ) @@ -167,6 +171,10 @@ def code_export(workspace: str, repo: str, def receipts(workspace: str, limit: int = 100): return run(svc.receipt_log, workspace=workspace, limit=limit) + @app.get("/context-savings") + def context_savings(workspace: str, repo: Optional[str] = None): + return run(svc.context_savings, workspace=workspace, repo=repo) + @app.get("/receipts/verify") def verify_receipts(workspace: str, expected_head: str = "", expected_count: Optional[int] = None): diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index b9fd5fdd..4d2b9190 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -956,7 +956,8 @@ def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, mtype: Optional[str] = None, as_of: Optional[float] = None, valid_at: Optional[float] = None, known_at: Optional[float] = None, token_budget: Optional[int] = Query(default=None, ge=0, le=32_768), - retrieval_profile: str = "balanced", response_mode: str = "full", + retrieval_profile: str = "balanced", candidate_depth: str = "fixed", + response_mode: str = "full", diagnostics: bool = False): ws = workspace or _default_ws() mtypes = [mtype] if mtype else None @@ -965,6 +966,7 @@ def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, q, workspace=ws, k=k, mtypes=mtypes, as_of=as_of, valid_at=valid_at, known_at=known_at, reinforce=False, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, response_mode=response_mode, diagnostics=diagnostics, ) except ValidationError: @@ -998,6 +1000,9 @@ def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, return {"query": q, "workspace": ws, "count": len(mems), "context": "", "memories": mems, "mode": "keyword", "response_mode": response_mode, "retrieval_profile": retrieval_profile, + "candidate_depth": candidate_depth, + "candidate_k_requested": 50, "candidate_k_used": 0, + "candidate_depth_reason": "keyword fallback", "valid_at": valid_at if valid_at is not None else as_of, "known_at": known_at, "historical": historical, "packed_sources": [], @@ -1030,6 +1035,7 @@ class _AnswerReq(BaseModel): known_at: Optional[float] = None token_budget: Optional[int] = Field(default=None, ge=0, le=32_768) retrieval_profile: str = "balanced" + candidate_depth: str = "fixed" response_mode: str = "full" diagnostics: bool = False @@ -1055,6 +1061,7 @@ def answer(req: _AnswerReq): known_at=req.known_at, token_budget=req.token_budget, retrieval_profile=req.retrieval_profile, + candidate_depth=req.candidate_depth, response_mode=req.response_mode, diagnostics=req.diagnostics, max_citations=req.max_citations, @@ -1205,6 +1212,12 @@ def receipts(workspace: Optional[str] = None, limit: int = 100): return _run(service().receipt_log, workspace=ws, limit=limit) +@router.get("/context-savings") +def context_savings(workspace: Optional[str] = None, repo: Optional[str] = None): + ws = workspace or _require_ws() + return _run(service().context_savings, workspace=ws, repo=repo) + + @router.get("/receipts/verify") def receipts_verify(workspace: Optional[str] = None, expected_head: str = "", expected_count: Optional[int] = None): @@ -1380,6 +1393,7 @@ class _IntentRecallReq(BaseModel): known_at: Optional[float] = None token_budget: Optional[int] = Field(default=None, ge=0, le=32_768) retrieval_profile: str = "balanced" + candidate_depth: str = "fixed" response_mode: str = "compact" diagnostics: bool = False @@ -1392,6 +1406,7 @@ def intent_recall(req: _IntentRecallReq): mtypes=req.mtypes, k=req.k, as_of=req.as_of, valid_at=req.valid_at, known_at=req.known_at, token_budget=req.token_budget, retrieval_profile=req.retrieval_profile, + candidate_depth=req.candidate_depth, response_mode=req.response_mode, diagnostics=req.diagnostics, ) diff --git a/engraphis/service.py b/engraphis/service.py index 644960b9..f1125f7b 100644 --- a/engraphis/service.py +++ b/engraphis/service.py @@ -42,7 +42,7 @@ from engraphis.core.graph_layers import normalize_graph_layer from engraphis.core.ids import new_id as make_id from engraphis.core.interfaces import Edge, GraphLayer, MemoryType, Node, Scope, SearchFilter -from engraphis.core.retrieval_policy import RETRIEVAL_PROFILES +from engraphis.core.retrieval_policy import CANDIDATE_DEPTH_MODES, RETRIEVAL_PROFILES from engraphis.core.store import ( _loads, _merge_edge_provenance, @@ -1160,6 +1160,7 @@ def intent_recall(self, query: str, *, intent: str = "recall", known_at: Optional[float] = None, token_budget: Optional[int] = None, retrieval_profile: str = "balanced", + candidate_depth: str = "fixed", response_mode: str = "full", diagnostics: bool = False, reinforce: bool = False, @@ -1182,6 +1183,7 @@ def intent_recall(self, query: str, *, intent: str = "recall", query, workspace=workspace, repo=repo, mtypes=mtypes, k=k, as_of=as_of, valid_at=valid_at, known_at=known_at, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, response_mode=response_mode, diagnostics=diagnostics, intent=intent_clean, graph_layers=layers, reinforce=reinforce, record_receipt=record_receipt, @@ -1665,6 +1667,7 @@ def recall(self, query: str, *, workspace: Optional[str] = None, graph_layers: Optional[list] = None, token_budget: Optional[int] = None, retrieval_profile: str = "balanced", + candidate_depth: str = "fixed", response_mode: str = "full", diagnostics: bool = False, record_receipt: bool = True) -> dict: @@ -1698,6 +1701,10 @@ def recall(self, query: str, *, workspace: Optional[str] = None, if retrieval_profile not in RETRIEVAL_PROFILES: choices = ", ".join(sorted(RETRIEVAL_PROFILES)) raise ValidationError(f"retrieval_profile must be one of: {choices}") + candidate_depth = str(candidate_depth or "fixed").strip().casefold() + if candidate_depth not in CANDIDATE_DEPTH_MODES: + choices = ", ".join(sorted(CANDIDATE_DEPTH_MODES)) + raise ValidationError(f"candidate_depth must be one of: {choices}") response_mode = str(response_mode or "full").strip().casefold() if response_mode not in RESPONSE_MODES: raise ValidationError("response_mode must be one of: compact, full") @@ -1716,7 +1723,8 @@ def recall(self, query: str, *, workspace: Optional[str] = None, if wid is None: return _empty_recall( query, token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, note=f"no workspace named '{ws}' yet", ) if repo: @@ -1724,8 +1732,9 @@ def recall(self, query: str, *, workspace: Optional[str] = None, rid = self._lookup_repo(wid, rp) if rid is None: return _empty_recall( - query, token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + query, token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, note=f"no repo named '{rp}' in workspace '{ws}' yet", ) @@ -1737,7 +1746,8 @@ def recall(self, query: str, *, workspace: Optional[str] = None, if session is None: return _empty_recall( query, token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, note=f"no session with id '{sid}'", ) if session["workspace_id"] != wid or ( @@ -1757,6 +1767,7 @@ def recall(self, query: str, *, workspace: Optional[str] = None, k=k, reinforce=reinforce, token_budget=token_budget, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, diagnostics=bool(diagnostics), ) memories = [] @@ -1803,6 +1814,10 @@ def recall(self, query: str, *, workspace: Optional[str] = None, "known_at": result.known_at, "historical": result.historical, "retrieval_profile": result.retrieval_profile, + "candidate_depth": result.candidate_depth_mode, + "candidate_k_requested": result.candidate_k_requested, + "candidate_k_used": result.candidate_k_used, + "candidate_depth_reason": result.candidate_depth_reason, "response_mode": response_mode, } if diagnostics: @@ -1815,6 +1830,9 @@ def recall(self, query: str, *, workspace: Optional[str] = None, "result_count": result.count, "graph_layers": [layer.value for layer in layers] if layers else [], "retrieval_profile": result.retrieval_profile, + "candidate_depth": result.candidate_depth_mode, + "candidate_k_requested": result.candidate_k_requested, + "candidate_k_used": result.candidate_k_used, "response_mode": response_mode, "historical": result.historical, "token_usage": usage}, @@ -1831,6 +1849,7 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, max_citations: int = 5, llm=None, token_budget: Optional[int] = None, retrieval_profile: str = "balanced", + candidate_depth: str = "fixed", response_mode: str = "full", diagnostics: bool = False) -> dict: """Grounded recall: an answer built strictly from retrieved memories, with @@ -1869,6 +1888,10 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, if retrieval_profile not in RETRIEVAL_PROFILES: choices = ", ".join(sorted(RETRIEVAL_PROFILES)) raise ValidationError(f"retrieval_profile must be one of: {choices}") + candidate_depth = str(candidate_depth or "fixed").strip().casefold() + if candidate_depth not in CANDIDATE_DEPTH_MODES: + choices = ", ".join(sorted(CANDIDATE_DEPTH_MODES)) + raise ValidationError(f"candidate_depth must be one of: {choices}") response_mode = str(response_mode or "full").strip().casefold() if response_mode not in RESPONSE_MODES: raise ValidationError("response_mode must be one of: compact, full") @@ -1895,7 +1918,8 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, return _empty_grounded( query, reason=f"no workspace named '{ws}' yet", token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, ) if repo: @@ -1906,7 +1930,8 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, query, reason=f"no repo named '{rp}' in workspace '{ws}' yet", token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, ) if session_id: @@ -1918,7 +1943,8 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, return _empty_grounded( query, reason=f"no session with id '{sid}'", token_budget=token_budget, response_mode=response_mode, - retrieval_profile=retrieval_profile, valid_at=valid_at, + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + valid_at=valid_at, known_at=known_at, ) if session["workspace_id"] != wid or ( @@ -1934,7 +1960,8 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, as_of=as_of, valid_at=valid_at, known_at=known_at, k=k, llm=llm, min_support=min_support, max_citations=max_citations, token_budget=token_budget, - retrieval_profile=retrieval_profile, diagnostics=bool(diagnostics), + retrieval_profile=retrieval_profile, candidate_depth=candidate_depth, + diagnostics=bool(diagnostics), ) out = {"query": query, **ans.to_dict()} out["response_mode"] = response_mode @@ -1953,6 +1980,9 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, metadata={"intent": "grounded", "grounded": bool(out.get("grounded")), "citations": len(out.get("citations") or []), "retrieval_profile": out.get("retrieval_profile"), + "candidate_depth": out.get("candidate_depth"), + "candidate_k_requested": out.get("candidate_k_requested"), + "candidate_k_used": out.get("candidate_k_used"), "response_mode": response_mode, "historical": bool(out.get("historical")), "token_usage": out.get("usage") or {}}, @@ -3651,6 +3681,17 @@ def receipt_log(self, *, workspace: str, limit: int = 100) -> dict: "entries": entries, } + def context_savings(self, *, workspace: str, repo: Optional[str] = None) -> dict: + """Return cumulative packed-context savings from content-free operation receipts.""" + ws = self._clean_ws(workspace) + rp = _clean_name(repo, field="repo") if repo else None + wid, rid = self._require_scope(ws, rp) + return { + "format": "engraphis-context-savings/1", + "scope": {"workspace": ws, **({"repo": rp} if rp else {})}, + **self.store.context_savings(workspace_id=wid, repo_id=rid), + } + def verify_receipts(self, *, workspace: str, expected_head: str = "", expected_count: Optional[int] = None) -> dict: """Verify the local chain and optionally compare an externally saved anchor.""" @@ -7056,7 +7097,7 @@ def _compact_provenance(value: Any) -> dict: def _empty_recall(query: str, *, token_budget: int, response_mode: str, - retrieval_profile: str, valid_at: Optional[float], + retrieval_profile: str, candidate_depth: str, valid_at: Optional[float], known_at: Optional[float], note: str) -> dict: """Stable empty response for unknown scopes, including additive v2 accounting.""" return { @@ -7079,19 +7120,24 @@ def _empty_recall(query: str, *, token_budget: int, response_mode: str, "known_at": known_at, "historical": valid_at is not None or known_at is not None, "retrieval_profile": retrieval_profile, + "candidate_depth": candidate_depth, + "candidate_k_requested": 50, + "candidate_k_used": 50, + "candidate_depth_reason": "no retrieval for unknown scope", "response_mode": response_mode, "note": note, } def _empty_grounded(query: str, *, reason: str, token_budget: int, - response_mode: str, retrieval_profile: str, + response_mode: str, retrieval_profile: str, candidate_depth: str, valid_at: Optional[float], known_at: Optional[float]) -> dict: payload = _empty_recall( query, token_budget=token_budget, response_mode=response_mode, retrieval_profile=retrieval_profile, + candidate_depth=candidate_depth, valid_at=valid_at, known_at=known_at, note=reason, diff --git a/engraphis/static/dashboard.js b/engraphis/static/dashboard.js index a54d4f3d..fbd23d73 100644 --- a/engraphis/static/dashboard.js +++ b/engraphis/static/dashboard.js @@ -251,11 +251,10 @@ function renderAnalytics(a,isPortfolio){const t=a.totals||{},f=a.decay_forecast| function managedConsentHtml(feature){const automation=/automation/i.test(feature),featureKey=`managed_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const primary=hostedCta('pro',featureKey),annual=primary.kind==='account'?'':{label:'Annual Pro option',href:hostedPlanUrl('pro',false,'annual',`${featureKey}_annual`),kind:'subscribe'},actions=`${ctaLinkHtml(primary,'btn btn-primary',featureKey)}${annual.href?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':licAccessState()==='lapsed'?'Your subscription needs billing attention. Update billing to restore hosted insights and maintenance.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Subscribe to Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} function managedConsentRequired(error){return error&&error.status===409&&error.detail&&error.detail.code==='consent_required'} const CLOUD_SYNC_PRIVACY_COPY='Cloud Sync encrypts eligible shared-workspace changes end-to-end before they leave this device. Engraphis Cloud cannot read their contents; secret and session-scoped memories stay local.'; -const MANAGED_COMPUTE_PRIVACY_COPY='Engraphis Cloud must read the bounded snapshot you submit to produce results. It travels over HTTPS but is not end-to-end encrypted; secret and session-scoped memories stay local.'; const EXTERNAL_LLM_PRIVACY_COPY='Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; -async function confirmCloudTransfer(title,summary,submit,privacyCopy){return confirmAction(title,summary+'\n\nPrivacy: '+(privacyCopy||MANAGED_COMPUTE_PRIVACY_COPY),submit||'Continue')} +async function confirmCloudTransfer(title,summary,submit,privacyCopy){return confirmAction(title,summary+(privacyCopy?'\n\nPrivacy: '+privacyCopy:''),submit||'Continue')} const managedConsentHtmlBase=managedConsentHtml; -managedConsentHtml=function(feature){const privacyCopy=/cloud sync/i.test(feature)?CLOUD_SYNC_PRIVACY_COPY:MANAGED_COMPUTE_PRIVACY_COPY;return managedConsentHtmlBase(feature).replace('',`
Privacy, by design. ${esc(privacyCopy)}
`)}; +managedConsentHtml=function(feature){return managedConsentHtmlBase(feature)}; /* Only an unconfigured local installation may turn a 401 into trial signup. A revoked or expired Cloud session is also a 401, but ``trial.available`` is false there and it must remain a reconnect error instead of offering a trial the control plane rejects. */ @@ -408,7 +407,7 @@ async function doTimeline(){ /* audit */ async function loadAudit(){const el=document.getElementById('audit-body');el.innerHTML='';try{const d=await api('/audit?workspace='+encodeURIComponent(WS||'')+'&limit=200');const rows=d.entries||d.audit||[];if(!rows.length){el.innerHTML='
No governance actions recorded.
';return}el.innerHTML='
'+rows.map(r=>`
${esc(r.action||r.op||r.kind||'edit')}${esc(r.memory_id||r.target||r.detail||'')}${esc(r.actor||'')}${r.ts||r.at?fmtRel(r.ts||r.at):''}
`).join('')+'
'}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} -async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{const d=await api('/receipts?workspace='+encodeURIComponent(WS||'')+'&limit=500');const v=await api('/receipts/verify?workspace='+encodeURIComponent(WS||''));const rows=d.entries||[];el.innerHTML=`
Receipt chain ${v.valid?'verified':'invalid'}
${v.count||0} receipts · head ${esc((v.head||'').slice(0,24))}
`+(rows.length?'
'+rows.map(r=>`
${esc(r.operation||'operation')}${esc((r.hash||'').slice(0,20))} · ${esc(r.status||'ok')} · ${r.target_count||0} target(s)${r.ts_ms?fmtRel(r.ts_ms/1000):''}
`).join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} +async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{const q='workspace='+encodeURIComponent(WS||'');const [d,v,s]=await Promise.all([api('/receipts?'+q+'&limit=500'),api('/receipts/verify?'+q),api('/context-savings?'+q)]);const rows=d.entries||[],counters=s.by_token_counter||[];const savings=counters.map(x=>`
${esc(x.token_counter||'unknown')}${x.context_tokens||0} packed / ${x.source_tokens||0} retrieved-source tokens; ${x.saved_tokens||0} not injected (${((x.savings_ratio||0)*100).toFixed(1)}%)
`).join('');const savingCard=`
Packed context efficiency
${s.savings_receipt_count||0} packed recalls; this measures retrieved source versus injected context, grouped by token counter.
${savings||'
No complete context-usage receipts yet.
'}
`;el.innerHTML=savingCard+`
Receipt chain ${v.valid?'verified':'invalid'}
${v.count||0} receipts · head ${esc((v.head||'').slice(0,24))}
`+(rows.length?'
'+rows.map(r=>`
${esc(r.operation||'operation')}${esc((r.hash||'').slice(0,20))} · ${esc(r.status||'ok')} · ${r.target_count||0} target(s)${r.ts_ms?fmtRel(r.ts_ms/1000):''}
`).join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} async function downloadReceipts(){try{const d=await api('/receipts/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-receipts-'+(WS||'workspace')+'.json';a.click();URL.revokeObjectURL(a.href);toast('Privacy-safe receipts exported','ok')}catch(e){toast(e.message,'err')}} /* consolidate */ @@ -1160,7 +1159,7 @@ function loadGraphEngine(){ if(GRAPH_ENGINE_LOADING)return GRAPH_ENGINE_LOADING; GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{ const script=document.createElement('script'); - script.src='/v2-assets/engraphis-graph.js?v=20260728-reference-materials'; + script.src='/v2-assets/engraphis-graph.js?v=20260730-drag-stability'; /* A 200 that never registers the global is a corrupt/truncated asset, not a success — resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */ script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()}; diff --git a/eval/EVIDENCE.md b/eval/EVIDENCE.md new file mode 100644 index 00000000..e33da9f8 --- /dev/null +++ b/eval/EVIDENCE.md @@ -0,0 +1,30 @@ +# External benchmark evidence + +`eval.benchmark.report_envelope()` is the public-artifact boundary. It records +the dataset and optional source digests, commit and dirty-state digest, command, +configuration digest, environment, model metadata, and token-counting scope. +It removes raw questions, answers, returned context, and prompts from every +record while retaining SHA-256 digests for same-input verification. + +After an official LongMemEval-V2 run, keep the upstream `per_question.jsonl` +private and create a redacted artifact: + +```bash +python -m eval.longmemeval_v2_evidence \ + --per-question output/per_question.jsonl \ + --questions data/questions.json \ + --haystack data/haystack.json \ + --trajectories data/trajectories.json \ + --memory-config eval/configs/longmemeval_v2_engraphis.json \ + --output artifacts/longmemeval-v2.json +``` + +The command writes sorted JSON and an adjacent `.sha256` checksum, refusing to +replace a different artifact. It preserves the official harness QA score and +its fixed-reader memory-context item-content token count. That count excludes +chat-prompt framing and inter-item separators, so it is not a total provider +prompt-token claim. It is not a canonical Engraphis retrieval artifact until a +complete run also supplies the required five-budget evidence curve. + +The evidence exporter records an intentionally redacted command label. Keep +API keys and raw prompt material only in the private official-run environment. diff --git a/eval/agent_benchmarks.py b/eval/agent_benchmarks.py new file mode 100644 index 00000000..26171dc1 --- /dev/null +++ b/eval/agent_benchmarks.py @@ -0,0 +1,630 @@ +"""Offline adapters for agent-memory benchmark datasets. + +This module intentionally does not vendor or import any benchmark repository. +It translates public JSON/JSONL exports into the ``eval.harness`` case schema +and runs the shipped Engraphis write/recall pipeline with the deterministic +embedder by default. + +Supported formats: + +* ``memoryagentbench`` — the public ``{"data": [...]}`` export or Hugging Face + dataset-server ``{"rows": [{"row": ...}]}`` envelope containing a long + ``context`` plus aligned ``questions``/``answers`` lists. Optional structured + ``memory_events`` preserve incremental order and conflict keys. +* ``locomo_plus`` — the public unified-input records (``input_prompt``, + ``trigger``, ``evidence``, ``category``). It measures retrieval of the + earlier cue, not the repository's LLM-as-judge answer score. +* ``mem2actbench`` — the paired public ``qa_dataset.jsonl`` and + ``toolmem_conversation.jsonl`` exports. It measures whether packed memory + covers the expected tool-call arguments; Engraphis does not itself generate + a tool call, so this is explicitly not action-success accuracy. + +All loaders are strict: malformed or unmappable records raise ``ValueError`` +instead of silently dropping benchmark rows. The included fixtures are +deterministic plumbing/contract checks, not external leaderboard results. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +from typing import Any, Callable, Optional + +from engraphis.backends import DeterministicEmbedder +from engraphis.backends.embedder_st import get_embedder +from eval.benchmark import report_envelope, write_canonical_artifact +from eval.harness import run + + +_PINNED_EMBED_REVISION = re.compile(r"[0-9a-f]{40}\Z") + + +def _read_records(path: str) -> list[dict[str, Any]]: + """Read a JSON list/object or JSONL file, rejecting non-object rows.""" + source = Path(path) + try: + text = source.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"could not read {source}: {exc}") from exc + try: + parsed = json.loads(text) + except json.JSONDecodeError: + rows = [] + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{source}:{number} is not valid JSON") from exc + rows.append(row) + parsed = rows + if isinstance(parsed, dict) and isinstance(parsed.get("rows"), list): + rows = parsed["rows"] + if not all( + isinstance(item, dict) and isinstance(item.get("row"), dict) + for item in rows + ): + raise ValueError(f"{source} has a malformed Hugging Face rows envelope") + return [item["row"] for item in rows] + if isinstance(parsed, dict): + return [parsed] + if not isinstance(parsed, list) or not all(isinstance(row, dict) for row in parsed): + raise ValueError(f"{source} must contain a JSON object, JSON list of objects, or JSONL objects") + return parsed + + +def _text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label} must be a non-empty string") + return value.strip() + + +def _as_texts(value: Any, label: str) -> list[str]: + if isinstance(value, str): + return [_text(value, label)] + if not isinstance(value, list) or not value: + raise ValueError(f"{label} must be a non-empty string or list of strings") + return [_text(item, label) for item in value] + + +def _answer_rows(value: Any, label: str) -> list[tuple[str, list[str]]]: + """Normalize one answer or a list of accepted answer variants per question.""" + if not isinstance(value, list) or not value: + raise ValueError(f"{label} must be a non-empty list") + output = [] + for number, item in enumerate(value): + variants = _as_texts(item, f"{label}[{number}]") + output.append((variants[0], variants)) + return output + + +def _chunks(text: str, prefix: str, *, max_chars: int = 900) -> list[dict[str, str]]: + """Make stable, dialogue-safe context records without external tokenizers.""" + paragraphs = [ + part.strip() + for part in text.replace("\r\n", "\n").split("\n\n") + if part.strip() + ] + if not paragraphs: + raise ValueError("context has no non-empty paragraphs") + output: list[dict[str, str]] = [] + + def emit(value: str) -> None: + output.append({"tag": f"{prefix}:{len(output)}", "text": value}) + + for paragraph in paragraphs: + pending = "" + for unit in (line.strip() for line in paragraph.splitlines() if line.strip()): + candidate = f"{pending}\n{unit}" if pending else unit + if len(candidate) <= max_chars: + pending = candidate + continue + if pending: + emit(pending) + pending = "" + while len(unit) > max_chars: + cut = unit.rfind(" ", 0, max_chars) + cut = cut if cut > max_chars // 2 else max_chars + emit(unit[:cut].strip()) + unit = unit[cut:].strip() + pending = unit + if pending: + emit(pending) + return output + + +def _case_id(row: dict[str, Any], prefix: str, number: int) -> str: + for key in ("id", "case_id", "sample_id", "session_id", "question_id"): + if row.get(key) is not None and str(row[key]).strip(): + return str(row[key]).strip() + return f"{prefix}-{number}" + + +def _limited_rows(rows: list[dict[str, Any]], limit: Optional[int]) -> list[dict[str, Any]]: + """Apply an explicit positive limit without Python's surprising negative slices.""" + if limit is None: + return rows + if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0: + raise ValueError("limit must be a positive integer when supplied") + return rows[:limit] + + +def load_memoryagentbench(path: str, *, limit: Optional[int] = None) -> list[dict]: + """Load MemoryAgentBench's public context/question export. + + Its upstream conversation creator accepts a top-level ``data`` array, then + reads ``context`` and aligned ``questions``/``answers`` fields. Some + downstream exports preserve individual memory events; when present we use + them directly so ``subject_key``/``claim_kind`` can exercise the actual + conflict-resolution write path. + """ + roots = _read_records(path) + if len(roots) == 1 and isinstance(roots[0].get("data"), list): + rows = roots[0]["data"] + else: + rows = roots + if not all(isinstance(row, dict) for row in rows): + raise ValueError("MemoryAgentBench data must be objects") + cases = [] + for number, row in enumerate(_limited_rows(rows, limit)): + case_id = _case_id(row, "mab", number) + events = row.get("memory_events") + if events is not None: + if not isinstance(events, list) or not events: + raise ValueError(f"MemoryAgentBench {case_id}: memory_events must be a non-empty list") + memories = [] + for event_number, event in enumerate(events): + if not isinstance(event, dict): + raise ValueError(f"MemoryAgentBench {case_id}: memory_events[{event_number}] must be an object") + memories.append({ + "tag": str(event.get("id") or f"{case_id}:event:{event_number}"), + "text": _text(event.get("text") or event.get("content"), "memory event text"), + "valid_from": float(event_number), + "subject_key": str(event.get("subject_key") or ""), + "claim_kind": str(event.get("claim_kind") or ""), + }) + else: + memories = _chunks(_text(row.get("context"), f"MemoryAgentBench {case_id}.context"), case_id) + + questions = _as_texts(row.get("questions"), f"MemoryAgentBench {case_id}.questions") + answers = _answer_rows(row.get("answers"), f"MemoryAgentBench {case_id}.answers") + if len(questions) != len(answers): + raise ValueError(f"MemoryAgentBench {case_id}: questions and answers must have equal length") + metadata = row.get("metadata") + metadata = metadata if isinstance(metadata, dict) else {} + ids = ( + row.get("qa_pair_ids") + or row.get("question_ids") + or metadata.get("qa_pair_ids") + or metadata.get("question_ids") + or [] + ) + if ids and (not isinstance(ids, list) or len(ids) != len(questions)): + raise ValueError(f"MemoryAgentBench {case_id}: qa_pair_ids must align with questions") + supporting_rows = row.get("supporting_ids") or row.get("evidence_ids") or [] + if supporting_rows and (not isinstance(supporting_rows, list) or len(supporting_rows) != len(questions)): + raise ValueError(f"MemoryAgentBench {case_id}: supporting_ids must align with questions") + normalized_questions = [] + for q_number, (question, answer_row) in enumerate(zip(questions, answers)): + answer, answer_variants = answer_row + if supporting_rows: + source_ids = supporting_rows[q_number] + if not isinstance(source_ids, list) or not all(str(item).strip() for item in source_ids): + raise ValueError( + f"MemoryAgentBench {case_id}: supporting_ids[{q_number}] must be a list of IDs" + ) + supporting = [str(item) for item in source_ids] + else: + supporting = [ + memory["tag"] + for memory in memories + if any( + variant.casefold() in memory["text"].casefold() + for variant in answer_variants + ) + ] + normalized_questions.append({ + "id": str(ids[q_number]) if ids else f"{case_id}:q:{q_number}", + "q": question, + "answer": answer, + "answer_variants": answer_variants, + "supporting": supporting, + "category": str( + row.get("sub_dataset") + or row.get("dataset") + or metadata.get("source") + or "memoryagentbench" + ), + # Upstream exports do not always expose evidence IDs. Keep + # answer-token coverage scored while publishing that caveat. + "gold_evidence_available": bool(supporting), + }) + cases.append({"id": case_id, "memories": memories, "questions": normalized_questions}) + if not cases: + raise ValueError("MemoryAgentBench source contained no cases") + return cases + + +def _evidence_matches(memories: list[dict[str, str]], evidence: list[str], label: str) -> list[str]: + tags = [] + for needle in evidence: + lines = [line.strip() for line in needle.splitlines() if line.strip()] + fragments = [] + for line in lines or [needle]: + # Official unified LoCoMo-Plus evidence is rendered as + # ``Speaker：utterance`` while input_prompt renders + # ``Speaker said, "utterance"``. Match the evidence-bearing + # utterance rather than requiring the formatting wrapper. + parts = re.split(r"[:：]", line, maxsplit=1) + fragment = (parts[1] if len(parts) == 2 else parts[0]).strip(" \t\"'") + if fragment: + fragments.append(fragment) + for fragment in fragments: + folded = fragment.casefold() + matched = [ + memory["tag"] + for memory in memories + if folded in memory["text"].casefold() + ] + if not matched: + raise ValueError(f"{label}: evidence text did not occur in input_prompt") + tags.extend(matched) + return list(dict.fromkeys(tags)) + + +def load_locomo_plus( + path: str, + *, + limit: Optional[int] = None, + include_original_locomo: bool = False, +) -> list[dict]: + """Load Locomo-Plus unified input and score cue retrieval deterministically. + + The official unified file also contains the five original LoCoMo categories. + The default selects only the new Cognitive category so a run measures implicit + cue-to-trigger memory instead of quietly becoming another factual LoCoMo run. + """ + rows = _read_records(path) + if len(rows) == 1 and isinstance(rows[0].get("data"), list): + rows = rows[0]["data"] + if not include_original_locomo: + rows = [ + row + for row in rows + if str(row.get("category") or "").strip().casefold() == "cognitive" + ] + cases = [] + for number, row in enumerate(_limited_rows(rows, limit)): + case_id = _case_id(row, "locomo-plus", number) + prompt = _text(row.get("input_prompt"), f"Locomo-Plus {case_id}.input_prompt") + trigger = _text(row.get("trigger"), f"Locomo-Plus {case_id}.trigger") + evidence = _as_texts(row.get("evidence"), f"Locomo-Plus {case_id}.evidence") + memories = _chunks(prompt, case_id) + supporting = _evidence_matches(memories, evidence, f"Locomo-Plus {case_id}") + cases.append({ + "id": case_id, + "memories": memories, + "questions": [{ + "id": str(row.get("question_id") or f"{case_id}:trigger"), + "q": trigger, + # Cognitive examples may intentionally omit a reference answer. + # Evidence-token coverage remains a reproducible retrieval measure. + "answer": _text(row.get("answer"), f"Locomo-Plus {case_id}.answer") + if row.get("answer") else " ".join(evidence), + "supporting": supporting, + "category": str(row.get("category") or "Cognitive"), + }], + }) + if not cases: + raise ValueError("Locomo-Plus source contained no cases") + return cases + + +def _tool_call_text(call: dict[str, Any], label: str) -> str: + if not isinstance(call, dict): + raise ValueError(f"{label}.tool_call must be an object") + name = _text(call.get("name"), f"{label}.tool_call.name") + arguments = call.get("arguments") + if not isinstance(arguments, dict): + raise ValueError(f"{label}.tool_call.arguments must be an object") + return json.dumps({"name": name, "arguments": arguments}, ensure_ascii=False, sort_keys=True) + + +def load_mem2actbench(qa_path: str, conversation_path: str, *, limit: Optional[int] = None) -> list[dict]: + """Load Mem2ActBench's paired QA/session JSONL exports.""" + sessions = _read_records(conversation_path) + by_source: dict[str, list[dict[str, str]]] = {} + for number, session in enumerate(sessions): + session_id = _case_id(session, "mem2act-session", number) + source_ids = session.get("original_conversation_ids") + turns = session.get("turns") + if not isinstance(source_ids, list) or not source_ids or not isinstance(turns, list) or not turns: + raise ValueError(f"Mem2Act session {session_id} requires original_conversation_ids and turns") + grouped: dict[str, list[str]] = {str(source): [] for source in source_ids} + for turn_number, turn in enumerate(turns): + if not isinstance(turn, dict): + raise ValueError(f"Mem2Act session {session_id}: turns[{turn_number}] must be an object") + content = turn.get("content") + if not isinstance(content, str) or not content.strip(): + continue + source = str(turn.get("source_id") or "") + if source in grouped: + grouped[source].append(f"{turn.get('role', 'unknown')}: {content.strip()}") + for source, lines in grouped.items(): + if lines: + by_source.setdefault(source, []).append({"tag": source, "text": "\n".join(lines)}) + + cases = [] + for number, qa in enumerate(_limited_rows(_read_records(qa_path), limit)): + qa_id = _case_id(qa, "mem2act", number) + source_ids = qa.get("source_conversation_ids") + if not isinstance(source_ids, list) or not source_ids: + raise ValueError(f"Mem2Act QA {qa_id}: source_conversation_ids must be a non-empty list") + memories = [memory for source in source_ids for memory in by_source.get(str(source), [])] + if not memories: + raise ValueError(f"Mem2Act QA {qa_id}: no session turns matched source_conversation_ids") + call = qa.get("tool_call") + expected = _tool_call_text(call, f"Mem2Act QA {qa_id}") + complexity = qa.get("complexity_metadata") or {} + cases.append({ + "id": qa_id, + "memories": memories, + "questions": [{ + "id": qa_id, + "q": _text(qa.get("query"), f"Mem2Act QA {qa_id}.query"), + "answer": expected, + "supporting": [str(source) for source in source_ids], + "category": str(complexity.get("level") or "tool_argument_grounding"), + }], + }) + if not cases: + raise ValueError("Mem2Act source contained no QA rows") + return cases + + +LOADERS: dict[str, Callable[..., list[dict]]] = { + "memoryagentbench": load_memoryagentbench, + "locomo_plus": load_locomo_plus, + "mem2actbench": load_mem2actbench, +} + + +def _claim_boundary(fmt: str) -> str: + if fmt == "mem2actbench": + return ("Retrieval/context coverage of expected tool-call JSON only; Engraphis is not a " + "tool-calling agent, so this is not end-to-end action success.") + if fmt == "locomo_plus": + return ("Cue-evidence retrieval only; this is not Locomo-Plus LLM-as-judge answer scoring.") + return ("Retrieval and answer-token context coverage only; upstream answer/Judge metrics are " + "not reproduced by this offline adapter.") + + +_RETRIEVAL_METRICS = ( + "recall_at_k", + "hit_at_k", + "mrr_at_k", + "ndcg_at_k", + "recall_at_1", + "recall_at_5", + "recall_at_10", + "hit_at_1", + "hit_at_5", + "hit_at_10", + "mrr_at_1", + "mrr_at_5", + "mrr_at_10", + "ndcg_at_1", + "ndcg_at_5", + "ndcg_at_10", +) + + +def _separate_unlabeled_retrieval(report: dict) -> None: + """Do not award perfect retrieval to questions with no gold evidence IDs.""" + detail = list(report.get("detail") or []) + retrieval_rows = [] + for row in detail: + if row.get("supporting_ids"): + retrieval_rows.append(row) + else: + row["retrieval_excluded"] = "no_gold_evidence" + for field in _RETRIEVAL_METRICS: + row.pop(field, None) + report["retrieval_scored_questions"] = len(retrieval_rows) + for field in ("recall_at_k", "hit_at_k", "mrr_at_k", "ndcg_at_k"): + report[field] = ( + round( + sum(float(row[field]) for row in retrieval_rows) + / len(retrieval_rows), + 4, + ) + if retrieval_rows + else None + ) + + +def public_artifact( + report: dict, + *, + fmt: str, + dataset: str, + conversations: Optional[str], + k: int, + limit: Optional[int], + embed_model: Optional[str], + embed_revision: Optional[str], + include_original_locomo: bool, + embedder: Optional[object], + resolve_conflicts: bool, +) -> dict: + """Build a redacted immutable envelope from a private adapter report.""" + if bool(embed_model) != bool(embed_revision): + raise ValueError("embed_model and embed_revision must be used together") + if embed_revision and _PINNED_EMBED_REVISION.fullmatch(embed_revision) is None: + raise ValueError("embed_revision must be an immutable lowercase 40-character commit") + detail = list(report.get("detail") or []) + first_usage = detail[0].get("usage") if detail else {} + first_usage = first_usage if isinstance(first_usage, dict) else {} + token_identity = str(first_usage.get("token_counter") or "unspecified") + metric_names = ( + "questions", + "scored_questions", + "retrieval_scored_questions", + "recall_at_k", + "hit_at_k", + "mrr_at_k", + "ndcg_at_k", + "answer_token_recall", + ) + metrics = {name: report[name] for name in metric_names if name in report} + metrics["claim_boundary"] = _claim_boundary(fmt) + source_paths = [dataset, *([conversations] if conversations else [])] + command = [ + "python", "-m", "eval.agent_benchmarks", + "--dataset", "", + "--format", fmt, + "--k", str(k), + ] + if limit is not None: + command.extend(["--limit", str(limit)]) + if embed_model: + command.extend(["--embed-model", embed_model]) + command.extend(["--embed-revision", str(embed_revision)]) + if not resolve_conflicts: + command.append("--no-resolve") + if include_original_locomo: + command.append("--include-original-locomo") + if conversations: + command.extend(["--conversations", ""]) + selected_embedder = embedder or DeterministicEmbedder() + model_id = getattr(selected_embedder, "model_name", type(selected_embedder).__name__) + revision = getattr(selected_embedder, "revision", None) + return report_envelope( + suite=f"Engraphis {fmt}", + dataset_path=dataset, + source_paths=source_paths, + config={ + "format": fmt, + "k": k, + "limit": limit, + "embed_model": model_id, + "embedder_revision": revision, + "resolve_conflicts": resolve_conflicts, + "include_original_locomo": bool( + report.get("include_original_locomo") + ), + }, + command=command, + token_accounting={ + "identity": token_identity, + "revision": None, + "scope": "packed_retrieved_memory_context", + "method": str( + detail[0].get("context_token_method") + if detail else "unspecified" + ), + }, + models={ + "embedder": { + "model_id": model_id, + "revision": revision, + }, + }, + records=detail, + metrics=metrics, + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Run offline agent-memory benchmark adapters.") + parser.add_argument("--dataset", required=True, help="Benchmark JSON or JSONL export.") + parser.add_argument("--format", required=True, choices=sorted(LOADERS)) + parser.add_argument("--conversations", help="Mem2ActBench toolmem_conversation.jsonl (required there).") + parser.add_argument("--k", type=int, default=10) + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--embed-model", default=None, help="Optional sentence-transformers model.") + parser.add_argument( + "--embed-revision", + default=None, + help="Required immutable 40-character commit when --embed-model is selected.", + ) + parser.add_argument("--no-resolve", action="store_true", help="Disable write-path resolution.") + parser.add_argument( + "--include-original-locomo", + action="store_true", + help="For locomo_plus, include the five original LoCoMo categories too.", + ) + parser.add_argument("--json", dest="json_out", default=None, help="Write JSON report to this path.") + parser.add_argument( + "--artifact", + default=None, + help="Write a redacted immutable evidence envelope and adjacent SHA256 file.", + ) + args = parser.parse_args(argv) + try: + if args.k <= 0: + raise ValueError("k must be a positive integer") + if bool(args.embed_model) != bool(args.embed_revision): + raise ValueError("--embed-model and --embed-revision must be used together") + if args.embed_revision and _PINNED_EMBED_REVISION.fullmatch(args.embed_revision) is None: + raise ValueError("--embed-revision must be an immutable lowercase 40-character commit") + if args.format == "mem2actbench": + if not args.conversations: + raise ValueError("--conversations is required for mem2actbench") + cases = load_mem2actbench(args.dataset, args.conversations, limit=args.limit) + elif args.format == "locomo_plus": + cases = load_locomo_plus( + args.dataset, + limit=args.limit, + include_original_locomo=args.include_original_locomo, + ) + else: + cases = LOADERS[args.format](args.dataset, limit=args.limit) + embedder = ( + get_embedder(args.embed_model, revision=args.embed_revision) + if args.embed_model else None + ) + report = run(cases, k=args.k, embedder=embedder, resolve_conflicts=not args.no_resolve) + except ValueError as exc: + parser.error(str(exc)) + report.update({ + "format": args.format, + "dataset": args.dataset, + "offline": embedder is None or isinstance(embedder, DeterministicEmbedder), + "embedder": { + "model_id": getattr(embedder, "model_name", None) + if embedder is not None else "DeterministicEmbedder", + "revision": getattr(embedder, "revision", None), + "implementation": type(embedder).__name__ if embedder is not None else "DeterministicEmbedder", + }, + "include_original_locomo": bool(args.include_original_locomo), + "limit": args.limit, + "measures": _claim_boundary(args.format), + }) + _separate_unlabeled_retrieval(report) + output = json.dumps(report, indent=2, sort_keys=True) + print(output) + if args.json_out: + Path(args.json_out).write_text(output + "\n", encoding="utf-8") + if args.artifact: + artifact = public_artifact( + report, + fmt=args.format, + dataset=args.dataset, + conversations=args.conversations, + k=args.k, + limit=args.limit, + embed_model=args.embed_model, + embed_revision=args.embed_revision, + include_original_locomo=bool(args.include_original_locomo), + embedder=embedder, + resolve_conflicts=not args.no_resolve, + ) + write_canonical_artifact(artifact, args.artifact) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/benchmark.py b/eval/benchmark.py index 8ab377d0..dd34ce90 100644 --- a/eval/benchmark.py +++ b/eval/benchmark.py @@ -8,14 +8,18 @@ import argparse import hashlib +import importlib.metadata import json import math import platform import random +import re +import subprocess import sys from copy import deepcopy from pathlib import Path from typing import Any, Callable, Iterable, Optional, Protocol, Sequence, Union +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from engraphis.core.textutil import estimate_tokens from eval import metrics as retrieval_metrics @@ -73,7 +77,12 @@ def encode(self, text: str) -> Sequence[Any]: def canonical_json(value: Any) -> str: """Serialize config deterministically so its hash is portable.""" - return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + # JSON has no representation for NaN or infinities. Rejecting them here + # keeps a checksummed artifact valid for strict JSON readers instead of + # silently emitting Python's non-standard ``NaN``/``Infinity`` literals. + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ) def sha256_text(value: str) -> str: @@ -88,6 +97,253 @@ def sha256_file(path: Union[str, Path]) -> str: return digest.hexdigest() +def source_digest(path: Union[str, Path]) -> dict[str, Union[str, int]]: + """Return content-only provenance for one benchmark input. + + Paths deliberately reduce to their basename: public evidence needs to prove + the bytes used, not disclose an operator's directory layout. + """ + resolved = Path(path) + return { + "name": resolved.name, + "sha256": sha256_file(resolved), + "bytes": resolved.stat().st_size, + } + + +def git_provenance(cwd: Optional[Union[str, Path]] = None) -> dict[str, Union[str, bool]]: + """Capture commit and dirty state without exposing changed filenames.""" + root = str(cwd or Path.cwd()) + try: + commit = subprocess.check_output( + ["git", "-C", root, "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() + status = subprocess.check_output( + ["git", "-C", root, "status", "--porcelain=v1"], text=True, + stderr=subprocess.DEVNULL, + ) + except (OSError, subprocess.CalledProcessError): + return {"commit": "unknown", "dirty": True, "dirty_state_sha256": sha256_text("unavailable")} + return { + "commit": commit or "unknown", + "dirty": bool(status.strip()), + "dirty_state_sha256": sha256_text(status), + } + + +def environment_provenance() -> dict[str, Any]: + """Return a compact, JSON-safe execution environment fingerprint.""" + packages = {} + for distribution in ("engraphis", "numpy", "sentence-transformers", "transformers", "torch"): + try: + packages[distribution] = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + continue + return { + "python": sys.version.split()[0], + "implementation": platform.python_implementation(), + "platform": platform.platform(), + "machine": platform.machine(), + "packages": packages, + } + + +_PUBLIC_RECORD_FIELDS = frozenset({ + "question_id", "category", "retrieved_ids", "supporting_ids", "context_tokens", + "latency_ms", "abstained", "excluded", "answerable", "grounded", + "grounded_support", "answer_token_recall", "context_token_method", + "context_tokenizer_identity", "qa_score", "qa_correct", "retrieval_excluded", "usage", +}) +_PUBLIC_METRIC_PREFIXES = ("recall_at_", "hit_at_", "mrr_at_", "ndcg_at_") +_PUBLIC_USAGE_FIELDS = frozenset({ + "budget_tokens", "context_tokens", "source_tokens", "saved_tokens", "savings_ratio", + "packed_count", "omitted_count", "answer_tokens", "token_counter", + "memory_context_tokens", "memory_context_original_tokens", "reader_prompt_tokens", + "reader_completion_tokens", "adapter_reported_context_tokens", +}) +_RAW_QUERY_FIELDS = ("q", "query", "question", "question_text") +_RAW_ANSWER_FIELDS = ( + "answer", "answer_gold", "answer_variants", "response", "response_raw", + "response_parsed_boxed", "output", "completion", "model_output", "assistant_response", +) +_RAW_CONTEXT_FIELDS = ( + "context", "memory_context", "messages", "prompt_messages", "retrieved_context", +) +_SECRET_NAME_RE = re.compile( + r"(?:^|[-_])(?:api[-_]?key|access[-_]?token|auth(?:orization)?|bearer|credential|" + r"password|passwd|secret|token|signature|sig|private[-_]?key)$", + re.IGNORECASE, +) +_COMPOUND_SECRET_NAME_RE = re.compile( + r"(?:^|[-_])(?:api[-_]?key|access[-_]?key|secret[-_]?(?:access[-_]?)?key|" + r"authorization)(?:[-_]|$)", + re.IGNORECASE, +) +_HEADER_OPTIONS = frozenset({"--header", "--headers"}) +_USERINFO_OPTIONS = frozenset({"-u", "--user", "--user-name", "--password", "-p"}) + + +def _is_secret_name(value: str) -> bool: + """Recognise credential-bearing parameter names, without masking normal options.""" + name = value.strip().lstrip("-") + return bool(_SECRET_NAME_RE.search(name) or _COMPOUND_SECRET_NAME_RE.search(name)) + + +def _public_exclusion(value: Any) -> Optional[dict[str, Any]]: + """Keep an exclusion's reason but never allow a free-form detail to leak content.""" + if not isinstance(value, dict): + return None + public = { + key: deepcopy(value[key]) + for key in ("question_id", "reason") + if key in value + } + detail = value.get("detail") + if detail == "": + public["detail"] = "" + elif detail is not None: + public["detail_sha256"] = sha256_text(canonical_json(detail)) + return public + + +def _public_usage(value: Any) -> Optional[dict[str, Any]]: + if not isinstance(value, dict): + return None + return { + key: deepcopy(item) + for key, item in value.items() + if key in _PUBLIC_USAGE_FIELDS + } + + +def redact_public_record(record: dict[str, Any]) -> dict[str, Any]: + """Project a private evaluation row onto the audited public evidence schema. + + Public artifacts are evidence, not a lossless export. An allowlist prevents a new + adapter field from accidentally publishing prompts, contexts, model output, tool calls, + or other raw payloads before this boundary is reviewed. + """ + public: dict[str, Any] = {} + groups = ( + (_RAW_QUERY_FIELDS, "query_sha256"), + (_RAW_ANSWER_FIELDS, "answer_or_response_sha256"), + (_RAW_CONTEXT_FIELDS, "context_or_prompt_sha256"), + ) + for fields, digest_field in groups: + values = [ + {"field": field, "value": record[field]} + for field in fields + if field in record + ] + if values: + public[digest_field] = sha256_text(canonical_json(values)) + for key, value in record.items(): + if key == "excluded": + redacted = _public_exclusion(value) + if redacted is not None: + public[key] = redacted + elif key == "usage": + redacted = _public_usage(value) + if redacted is not None: + public[key] = redacted + elif key in _PUBLIC_RECORD_FIELDS or key.startswith(_PUBLIC_METRIC_PREFIXES): + public[key] = deepcopy(value) + return public + + +def _redact_url(value: str) -> str: + """Remove URL userinfo and credential-like query values without hiding the endpoint.""" + try: + parsed = urlsplit(value) + except ValueError: + return value + if not parsed.scheme or not parsed.netloc: + return value + + def redact_parameters(component: str) -> str: + # Fragments are often ordinary anchors. Only treat a fragment as a parameter list when + # it contains an assignment, preserving links such as ``#methodology`` verbatim. + if "=" not in component: + return component + return urlencode([ + (key, "" if _is_secret_name(key) else item) + for key, item in parse_qsl(component, keep_blank_values=True) + ]) + + try: + host = parsed.hostname or "" + port = parsed.port + except ValueError: + # A malformed port must not make a credential-bearing authority pass through unchanged. + # Keep the malformed host:port for diagnostics, but remove anything before its final @. + authority = parsed.netloc.rsplit("@", 1)[-1] + netloc = f"@{authority}" if "@" in parsed.netloc else authority + else: + if port is not None: + host = f"{host}:{port}" + netloc = f"@{host}" if parsed.username is not None else parsed.netloc + + return urlunsplit(( + parsed.scheme, + netloc, + parsed.path, + redact_parameters(parsed.query), + redact_parameters(parsed.fragment), + )) + + +def _command_assignment(value: str) -> Optional[tuple[str, str]]: + """Return a shell-style assignment without mistaking URL query parameters for one.""" + separator = value.find("=") + if separator <= 0 or "://" in value[:separator]: + return None + return value[:separator], value[separator + 1:] + + +def redact_command(command: Sequence[str]) -> list[str]: + """Preserve a reproducible command shape without retaining credentials or raw headers.""" + public: list[str] = [] + redact_next = False + for item in command: + value = str(item) + lowered = value.casefold() + assignment = _command_assignment(value) + if redact_next: + public.append("") + redact_next = False + elif any(lowered.startswith(option + "=") for option in _HEADER_OPTIONS): + public.extend([value.split("=", 1)[0], ""]) + elif lowered.startswith("--user="): + public.extend([value.split("=", 1)[0], ""]) + elif value == "-H" or lowered in _HEADER_OPTIONS or lowered in _USERINFO_OPTIONS: + public.append(value) + redact_next = True + elif value.startswith("-H") and len(value) > 2: + public.extend([value[:2], ""]) + elif lowered.startswith("-u") and len(value) > 2: + public.extend([value[:2], ""]) + elif lowered.startswith("-p") and len(value) > 2: + public.extend([value[:2], ""]) + elif lowered.startswith("--") and _is_secret_name(lowered.split("=", 1)[0]): + public.append(value.split("=", 1)[0] if "=" in value else value) + if "=" in value: + public.append("") + else: + redact_next = True + elif assignment: + key, assigned = assignment + public.append( + f"{key}=" if _is_secret_name(key) else f"{key}={_redact_url(assigned)}" + ) + elif "://" in value: + public.append(_redact_url(value)) + elif _is_secret_name(value.split(":", 1)[0]) and ":" in value: + public.append(value.split(":", 1)[0] + ": ") + else: + public.append(_redact_url(value)) + return public + + def canonical_benchmark_config( *, run_label: str, @@ -228,6 +484,22 @@ def validate_report(report: Any, *, canonical: bool = False) -> list[str]: errors.append("canonical system.git_commit must be an immutable lowercase 40-character commit") declared_config_hash = system.get("config_sha256") _sha256_error(declared_config_hash, "system.config_sha256", errors) + if "git_dirty" in system and not isinstance(system.get("git_dirty"), bool): + errors.append("system.git_dirty must be boolean") + if "dirty_state_sha256" in system: + _sha256_error(system.get("dirty_state_sha256"), "system.dirty_state_sha256", errors) + sources = suite.get("sources") + if sources is not None: + if not isinstance(sources, list): + errors.append("suite.sources must be an array when supplied") + else: + for item in sources: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + errors.append("each suite source requires a name") + continue + _sha256_error(item.get("sha256"), "suite source sha256", errors) + if not _is_nonnegative_integer(item.get("bytes")): + errors.append("each suite source requires non-negative bytes") if not isinstance(protocol.get("config"), dict): errors.append("protocol.config must be an object") else: @@ -236,6 +508,25 @@ def validate_report(report: Any, *, canonical: bool = False) -> list[str]: errors.append( "system.config_sha256 must match the canonical protocol.config digest" ) + command = protocol.get("command") + if command is not None and ( + not isinstance(command, list) + or not command + or not all(isinstance(item, str) and item for item in command) + ): + errors.append("protocol.command must be a non-empty string array when supplied") + accounting = protocol.get("token_accounting") + if accounting is not None: + required_accounting = ("identity", "revision", "scope", "method") + if not isinstance(accounting, dict) or any(field not in accounting for field in required_accounting): + errors.append("protocol.token_accounting must name identity, revision, scope, and method") + elif ( + not isinstance(accounting["identity"], str) + or not isinstance(accounting["scope"], str) + or not isinstance(accounting["method"], str) + or not (accounting["revision"] is None or isinstance(accounting["revision"], str)) + ): + errors.append("protocol.token_accounting fields have invalid types") record_ids: list[str] = [] embedded_exclusions: dict[str, dict] = {} for record in records: @@ -288,6 +579,17 @@ def validate_report(report: Any, *, canonical: bool = False) -> list[str]: errors.append("protocol.n_scored must equal records minus exclusions") if canonical: config = protocol.get("config") if isinstance(protocol.get("config"), dict) else {} + if not isinstance(system.get("git_dirty"), bool): + errors.append("canonical reports require system.git_dirty") + elif system["git_dirty"]: + errors.append("canonical reports require a clean git worktree") + if not isinstance(protocol.get("command"), list) or not protocol.get("command"): + errors.append("canonical reports require protocol.command") + if not isinstance(protocol.get("token_accounting"), dict): + errors.append("canonical reports require protocol.token_accounting") + privacy = report.get("privacy") + if not isinstance(privacy, dict) or privacy.get("raw_query_policy") != "redacted_sha256": + errors.append("canonical reports require raw-query redaction metadata") if protocol.get("complete_dataset") is not True: errors.append("canonical protocol.complete_dataset must be true") source_questions = protocol.get("source_questions") @@ -982,12 +1284,30 @@ def report_envelope( records: Sequence[dict], metrics: Optional[dict] = None, exclusions: Optional[Sequence[dict]] = None, - git_commit: str = "unknown", + git_commit: Optional[str] = None, + command: Optional[Sequence[str]] = None, + source_paths: Optional[Sequence[Union[str, Path]]] = None, + models: Optional[dict] = None, + token_accounting: Optional[dict] = None, ) -> dict: - """Build a JSON-safe, provenance-complete public benchmark envelope.""" + """Build a JSON-safe, provenance-complete public benchmark envelope. + + This is intentionally the one path through which public reports obtain + provenance. It redacts raw question/answer/context fields before any caller + can persist the returned envelope. + """ path = Path(dataset_path) - resolved_exclusions = list(exclusions or []) - resolved_exclusions.extend(record["excluded"] for record in records if record.get("excluded")) + observed_git = git_provenance() + resolved_commit = git_commit if git_commit is not None else str(observed_git["commit"]) + public_records = [redact_public_record(dict(record)) for record in records] + resolved_exclusions = [ + redacted + for item in exclusions or () + if (redacted := _public_exclusion(item)) is not None + ] + resolved_exclusions.extend( + record["excluded"] for record in public_records if record.get("excluded") + ) # An adapter may supply both top-level and per-record exclusions. Retain one # canonical representation so ``n_scored`` remains an honest denominator. unique_exclusions = [] @@ -999,15 +1319,40 @@ def report_envelope( seen_exclusions.add(marker) return { "schema": SCHEMA, - "suite": {"name": suite, "dataset": path.name, "sha256": sha256_file(path)}, - "system": {"git_commit": git_commit, "config_sha256": sha256_text(canonical_json(config))}, - "environment": { - "python": sys.version.split()[0], "platform": platform.platform(), + "suite": { + "name": suite, + "dataset": path.name, + "sha256": sha256_file(path), + "sources": [source_digest(item) for item in source_paths or ()], + }, + "system": { + "git_commit": resolved_commit, + "git_dirty": observed_git["dirty"], + "dirty_state_sha256": observed_git["dirty_state_sha256"], + "config_sha256": sha256_text(canonical_json(config)), + }, + "environment": environment_provenance(), + "protocol": { + "command": redact_command(command or ("in_process",)), + "config": config, + "token_accounting": dict(token_accounting or { + "identity": "unspecified", + "revision": None, + "scope": "unspecified", + "method": "unspecified", + }), + "n_total": len(public_records), + "n_scored": len(public_records) - len(unique_exclusions), + }, + "privacy": { + "raw_query_policy": "redacted_sha256", + "raw_answer_policy": "redacted_sha256", + "raw_context_policy": "redacted_sha256", + "digest_algorithm": "sha256", }, - "protocol": {"config": config, "n_total": len(records), - "n_scored": len(records) - len(unique_exclusions)}, + "models": dict(models or {}), "metrics": metrics or {}, "exclusions": unique_exclusions, - "records": list(records), + "records": public_records, } diff --git a/eval/chunking_eval.py b/eval/chunking_eval.py index aa511df6..d4c3871d 100644 --- a/eval/chunking_eval.py +++ b/eval/chunking_eval.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import Optional -from engraphis.core.textutil import estimate_tokens +from engraphis.backends.extractor import ChunkingExtractor, get_extractor from engraphis.service import MemoryService MODES = ("whole", "chunked") @@ -46,14 +46,28 @@ def load(path: str) -> list[dict]: def run_eval(cases: list[dict], *, mode: str, k: int = 5, - embed_model: Optional[str] = None, embed_dim: int = 256) -> dict: + embed_model: Optional[str] = None, embed_dim: int = 256, + chunk_extractor: Optional[ChunkingExtractor] = None) -> dict: """Ingest the corpus in one workspace under ``mode`` and score its questions.""" + if mode not in MODES: + raise ValueError(f"mode must be one of: {', '.join(MODES)}") + selected_chunker = chunk_extractor or get_extractor("chunk") + if not isinstance(selected_chunker, ChunkingExtractor): + raise TypeError("chunk_extractor must be a ChunkingExtractor") + count_tokens = selected_chunker.count_tokens svc = MemoryService.create(":memory:", embed_model=embed_model, embed_dim=embed_dim, extractor=("chunk" if mode == "chunked" else "none")) + if mode == "chunked": + svc.engine.extractor = selected_chunker memories = 0 + stored_tokens: list[int] = [] for c in cases: out = svc.ingest(c["document"], workspace="corpus", mtype="semantic") memories += out["count"] + for fact in out["facts"]: + record = svc.store.get_memory(fact["id"]) + if record is not None: + stored_tokens.append(count_tokens(record.content)) nq = hits = 0 ctx_tokens = evidence_tokens = 0 @@ -61,21 +75,33 @@ def run_eval(cases: list[dict], *, mode: str, k: int = 5, for q in c["questions"]: nq += 1 results = svc.recall(q["q"], workspace="corpus", k=k).get("memories") or [] - ctx_tokens += sum(estimate_tokens(m.get("content") or "") for m in results) + ctx_tokens += sum(count_tokens(m.get("content") or "") for m in results) holding = [m for m in results if q["evidence"] in (m.get("content") or "")] if holding: hits += 1 - evidence_tokens += min(estimate_tokens(m["content"]) for m in holding) + evidence_tokens += min(count_tokens(m["content"]) for m in holding) return { "mode": mode, "memories_stored": memories, "questions": nq, "recall_at_k": round(hits / nq, 3) if nq else 0.0, "mean_context_tokens": round(ctx_tokens / nq, 1) if nq else 0.0, "mean_evidence_tokens": round(evidence_tokens / hits, 1) if hits else 0.0, + "max_stored_tokens": max(stored_tokens, default=0), + "token_counter": selected_chunker.token_counter_identity, } def compare(cases: list[dict], *, k: int, embed_model: Optional[str]) -> dict: - reports = {m: run_eval(cases, mode=m, k=k, embed_model=embed_model) for m in MODES} + chunker = get_extractor("chunk") + reports = { + mode: run_eval( + cases, + mode=mode, + k=k, + embed_model=embed_model, + chunk_extractor=chunker, + ) + for mode in MODES + } whole, chunked = reports["whole"], reports["chunked"] reduction = 0.0 if whole["mean_context_tokens"]: @@ -99,9 +125,11 @@ def main() -> int: print(f"chunking eval — {len(cases)} docs · {result['reports']['whole']['questions']} " f"questions @ k={args.k} · embedder={embedder}\n") row = " {mode:<8} recall@k={recall_at_k:<6} ctx_tokens={mean_context_tokens:<8} " \ - "evidence_tokens={mean_evidence_tokens:<7} (memories={memories_stored})" + "evidence_tokens={mean_evidence_tokens:<7} max_stored={max_stored_tokens:<6} " \ + "(memories={memories_stored})" for mode in MODES: print(row.format(**result["reports"][mode])) + print(f" token counter: {result['reports']['chunked']['token_counter']}") print(f"\n context reduction (chunked vs whole): {result['context_reduction_pct']}%") return 0 diff --git a/eval/code_agent_ab.py b/eval/code_agent_ab.py new file mode 100644 index 00000000..53821718 --- /dev/null +++ b/eval/code_agent_ab.py @@ -0,0 +1,238 @@ +"""Paired code-agent A/B analysis for full-history versus Engraphis runs. + +This module deliberately does not launch an agent. The selected agent controller and +task sandbox own execution; this analyzer enforces the paired evidence contract after +both conditions have produced content-free run records. It prevents unmatched tasks, +different success oracles, and unpaired averages from becoming a marketing claim. + +Each JSON/JSONL row requires:: + + { + "task_id": "repo/task-1", + "condition": "full_history", + "oracle": "pytest:tests/test_task.py", + "success": true, + "input_tokens": 1000, + "output_tokens": 120, + "tool_tokens": 80, + "retries": 0, + "latency_ms": 2500, + "cost_usd": 0.01 + } + +``cost_usd`` is optional; every other metric is required and non-negative. +""" +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any, Optional + +from eval.benchmark import paired_bootstrap_ci + + +CONDITIONS = ("full_history", "engraphis") +NUMERIC_FIELDS = ( + "input_tokens", + "output_tokens", + "tool_tokens", + "retries", + "latency_ms", +) + + +def _records(path: str | Path) -> list[dict[str, Any]]: + source = Path(path) + text = source.read_text(encoding="utf-8") + try: + value = json.loads(text) + except json.JSONDecodeError: + value = [ + json.loads(line) + for line in text.splitlines() + if line.strip() + ] + if isinstance(value, dict): + value = [value] + if not isinstance(value, list) or not value or not all( + isinstance(row, dict) for row in value + ): + raise ValueError(f"{source} must contain one or more JSON object records") + return value + + +def load_runs(path: str | Path, *, condition: str) -> dict[str, dict[str, Any]]: + """Load and strictly validate one experiment condition.""" + if condition not in CONDITIONS: + raise ValueError(f"condition must be one of: {', '.join(CONDITIONS)}") + runs: dict[str, dict[str, Any]] = {} + for number, row in enumerate(_records(path), start=1): + task_id = row.get("task_id") + oracle = row.get("oracle") + if not isinstance(task_id, str) or not task_id.strip(): + raise ValueError(f"{path}:{number} requires a non-empty task_id") + task_id = task_id.strip() + if task_id in runs: + raise ValueError(f"{path} contains duplicate task_id {task_id!r}") + if row.get("condition") != condition: + raise ValueError( + f"{path}:{number} condition must be {condition!r}" + ) + if not isinstance(oracle, str) or not oracle.strip(): + raise ValueError(f"{path}:{number} requires a deterministic oracle label") + oracle = oracle.strip() + if not isinstance(row.get("success"), bool): + raise ValueError(f"{path}:{number} success must be boolean") + normalized = { + "task_id": task_id, + "condition": condition, + "oracle": oracle, + "success": row["success"], + } + for field in NUMERIC_FIELDS: + value = row.get(field) + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or value < 0 + or not math.isfinite(float(value)) + ): + raise ValueError(f"{path}:{number} {field} must be non-negative") + normalized[field] = float(value) + cost = row.get("cost_usd") + if cost is not None and ( + not isinstance(cost, (int, float)) + or isinstance(cost, bool) + or cost < 0 + or not math.isfinite(float(cost)) + ): + raise ValueError(f"{path}:{number} cost_usd must be non-negative") + normalized["cost_usd"] = float(cost) if cost is not None else None + normalized["total_tokens"] = sum( + normalized[field] + for field in ("input_tokens", "output_tokens", "tool_tokens") + ) + runs[task_id] = normalized + return runs + + +def _mean(values: list[float]) -> float: + return round(sum(values) / len(values), 6) if values else 0.0 + + +def _summary(rows: list[dict[str, Any]]) -> dict[str, Any]: + metrics = { + "success_rate": _mean([float(row["success"]) for row in rows]), + } + for field in (*NUMERIC_FIELDS, "total_tokens"): + metrics[f"mean_{field}"] = _mean([float(row[field]) for row in rows]) + costs = [float(row["cost_usd"]) for row in rows if row["cost_usd"] is not None] + metrics["mean_cost_usd"] = _mean(costs) if len(costs) == len(rows) else None + return metrics + + +def evaluate( + baseline: dict[str, dict[str, Any]], + candidate: dict[str, dict[str, Any]], + *, + iterations: int = 5000, + seed: int = 20260730, +) -> dict[str, Any]: + """Return paired deltas as ``Engraphis - full_history`` with bootstrap CIs.""" + if type(iterations) is not int or iterations <= 0: + raise ValueError("iterations must be a positive integer") + if type(seed) is not int: + raise ValueError("seed must be an integer") + if set(baseline) != set(candidate): + missing_candidate = sorted(set(baseline) - set(candidate)) + missing_baseline = sorted(set(candidate) - set(baseline)) + raise ValueError( + "paired task IDs differ: " + f"missing_engraphis={missing_candidate}, missing_full_history={missing_baseline}" + ) + task_ids = sorted(baseline) + for task_id in task_ids: + if baseline[task_id]["oracle"] != candidate[task_id]["oracle"]: + raise ValueError(f"task {task_id!r} used different success oracles") + + baseline_rows = [baseline[task_id] for task_id in task_ids] + candidate_rows = [candidate[task_id] for task_id in task_ids] + fields = { + "success_rate": "success", + **{f"mean_{field}": field for field in (*NUMERIC_FIELDS, "total_tokens")}, + } + deltas = {} + for label, field in fields.items(): + pairs = [ + (float(candidate[task_id][field]), float(baseline[task_id][field])) + for task_id in task_ids + ] + deltas[label] = paired_bootstrap_ci( + pairs, iterations=iterations, seed=seed, + ) + if all(row["cost_usd"] is not None for row in baseline_rows + candidate_rows): + deltas["mean_cost_usd"] = paired_bootstrap_ci( + [ + ( + float(candidate[task_id]["cost_usd"]), + float(baseline[task_id]["cost_usd"]), + ) + for task_id in task_ids + ], + iterations=iterations, + seed=seed, + ) + + return { + "schema": "engraphis-code-agent-ab/v1", + "paired_tasks": len(task_ids), + "conditions": { + "baseline": "full_history", + "candidate": "engraphis", + }, + "delta_direction": "engraphis_minus_full_history", + "interpretation": { + "success_rate": "positive_is_better", + "tokens_retries_latency_cost": "negative_is_better", + }, + "full_history": _summary(baseline_rows), + "engraphis": _summary(candidate_rows), + "paired_bootstrap": deltas, + # The pairing checks above deliberately happen before aggregation, but + # task IDs and oracle commands can reveal private repository layout. + # The public-safe result therefore retains only the checked count. + "matched_oracle_count": len(task_ids), + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Analyze paired full-history versus Engraphis code-agent runs." + ) + parser.add_argument("--full-history", required=True) + parser.add_argument("--engraphis", required=True) + parser.add_argument("--iterations", type=int, default=5000) + parser.add_argument("--seed", type=int, default=20260730) + parser.add_argument("--output") + args = parser.parse_args(argv) + try: + report = evaluate( + load_runs(args.full_history, condition="full_history"), + load_runs(args.engraphis, condition="engraphis"), + iterations=args.iterations, + seed=args.seed, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + payload = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + if args.output: + Path(args.output).write_text(payload + "\n", encoding="utf-8") + else: + print(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/context_economy.py b/eval/context_economy.py new file mode 100644 index 00000000..6a51e662 --- /dev/null +++ b/eval/context_economy.py @@ -0,0 +1,438 @@ +"""Deterministic workload-level context economy benchmark. + +This benchmark measures *reader context* under three executable strategies over +the ordinary ``eval.harness`` JSONL schema: + +* ``full_history`` replays the complete case corpus for every question; +* ``recency_window`` admits the newest source memories that fit the same token + budget given to Engraphis; and +* ``engraphis`` uses the shipped hybrid recall pipeline and context packer. + +By default it is an offline, token-accounting benchmark, not a provider +billing model; callers may inject a real embedder for a retrieval comparison. +Counts use the named ``engraphis.regex.v1`` counter exactly; +they exclude system prompts, question text, output/completion tokens, provider +tokenizer differences, cached-input pricing, and any compute or storage costs. +The indexing-inclusive total conservatively adds one complete source-corpus +token pass once to Engraphis query context. That makes the break-even point +explicit without pretending it is a dollar or invoice estimate. + +The input format is the existing harness schema:: + + {"id": "case", "memories": [{"tag": "f1", "text": "..."}], + "questions": [{"q": "...", "answer": "...", "supporting": ["f1"]}]} + +Run with ``python -m eval.context_economy --dataset ...``. stdout is always a +single JSON document so the command is safe to consume from automation. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Callable, Optional + +from engraphis.backends import DeterministicEmbedder, NumpyVectorIndex +from engraphis.backends.embedder_st import get_embedder +from engraphis.backends.reranker import IdentityReranker +from engraphis.core.context import DeterministicContextPacker, RegexTokenCounter +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import MemoryType, Scope +from engraphis.core.store import Store +from eval import metrics +from eval.external import LOADERS +from eval.harness import _seed_case_graph, load_dataset + + +DEFAULT_TOKEN_BUDGET = 512 +DEFAULT_K = 5 +TOKEN_COUNTER_IDENTITY = RegexTokenCounter.identity +NON_BILLING_SCOPE = ( + "This is deterministic reader-context token accounting, not provider billing. " + "It excludes system prompts, question text, completion tokens, provider tokenizer " + "differences, cached-input pricing, and compute or storage costs." +) + + +def _mean(rows: list[dict], key: str) -> float: + return round(sum(float(row.get(key, 0.0)) for row in rows) / max(len(rows), 1), 6) + + +def _scored(question: dict) -> bool: + """Match the harness convention: unanswerable rows consume context but not quality.""" + return question.get("answerable") is not False + + +def _truncate_to_budget(text: str, budget: int, counter: Callable[[str], int]) -> str: + """Return the longest regex-token prefix that fits ``budget`` exactly. + + The benchmark pins the built-in regex counter, so preserving the whitespace + between source tokens is sufficient and deterministic. A whole source that + fits is returned byte-for-byte to avoid inventing a recency summarizer. + """ + source = str(text or "") + if budget <= 0: + return "" + if counter(source) <= budget: + return source + import re + + matches = list(re.finditer(r"\w+|[^\w\s]", source, re.UNICODE)) + if not matches: + return "" + limit = min(len(matches), budget) + return source[:matches[limit - 1].end()].rstrip() + + +def _recency_context( + memories: list[dict], *, budget: int, counter: Callable[[str], int], +) -> tuple[str, list[str]]: + """Select newest raw source records under the common reader-context budget.""" + remaining = max(0, int(budget)) + selected: list[tuple[str, str]] = [] + for memory in reversed(memories): + text = str(memory.get("text", "")) + tokens = counter(text) + if tokens <= remaining: + selected.append((str(memory.get("tag", "")), text)) + remaining -= tokens + elif not selected and remaining: + excerpt = _truncate_to_budget(text, remaining, counter) + if excerpt: + selected.append((str(memory.get("tag", "")), excerpt)) + remaining = 0 + else: + # A recency *window* is contiguous: once the next older source + # cannot fit, it must not skip backward into an older record. + break + if remaining <= 0: + break + # A conversation window is presented chronologically even though it is + # selected from the newest end of the source sequence. + selected.reverse() + return "\n\n".join(text for _, text in selected), [tag for tag, _ in selected if tag] + + +def _quality( + *, retrieved_tags: list[str], retrieved_texts: list[str], question: dict, +) -> dict: + """Quality of evidence actually placed into the reader's context.""" + supporting = [str(tag) for tag in question.get("supporting", [])] + answer = str(question.get("answer", question.get("evidence", ""))) + return { + "retrieval_recall": metrics.recall_at_k(retrieved_tags, supporting), + "retrieval_hit": metrics.hit_at_k(retrieved_tags, supporting), + "answer_token_recall": metrics.answer_token_recall(retrieved_texts, answer), + } + + +def _method_summary(rows: list[dict]) -> dict: + scored = [row for row in rows if row["scored"]] + context_tokens = sum(int(row["context_tokens"]) for row in rows) + return { + "queries": len(rows), + "scored_queries": len(scored), + "cumulative_query_context_tokens": context_tokens, + "mean_query_context_tokens": round(context_tokens / max(len(rows), 1), 6), + "quality": { + "retrieval_recall": _mean(scored, "retrieval_recall"), + "retrieval_hit_rate": _mean(scored, "retrieval_hit"), + "answer_token_recall": _mean(scored, "answer_token_recall"), + }, + } + + +def _engraphis_rows( + case: dict, + *, + k: int, + token_budget: int, + embedder: object, + counter: Callable[[str], int], + resolve_conflicts: bool, +) -> list[dict]: + """Run production ingestion, hybrid recall, and packing for one case.""" + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("context-economy") + repo_id = store.get_or_create_repo(workspace_id, str(case.get("id", "case"))) + engine = MemoryEngine( + store, + embedder, + NumpyVectorIndex(store), + IdentityReranker(), + ) + # Pin both strategy accounting and shipped packing to the same named, + # offline counter. This makes budgets directly comparable. + engine.recall_engine.context_packer = DeterministicContextPacker( + token_counter=counter, token_counter_identity=TOKEN_COUNTER_IDENTITY, + ) + _seed_case_graph(store, workspace_id=workspace_id, repo_id=repo_id, case=case) + + id_to_tags: dict[str, list[str]] = {} + for memory in case.get("memories", []): + content = str(memory.get("text", "")) + memory_id = engine.remember( + content, + workspace_id=workspace_id, + repo_id=repo_id, + mtype=MemoryType.EPISODIC, + scope=Scope.REPO, + title=str(memory.get("title", "")), + valid_from=memory.get("valid_from"), + subject_key=str(memory.get("subject_key", "")), + claim_kind=str(memory.get("claim_kind", "")), + resolve_conflicts=resolve_conflicts, + ) + tag = memory.get("tag") + if tag is not None: + id_to_tags.setdefault(memory_id, []).append(str(tag)) + + rows = [] + for number, question in enumerate(case.get("questions", [])): + result = engine.recall( + str(question.get("q", "")), workspace_id=workspace_id, + repo_id=repo_id, k=k, token_budget=token_budget, + ) + # ``chunks`` can contain candidates omitted by packing. Reader + # quality must only receive the chunks actually admitted to context. + packed_ids = [chunk.id for chunk in result.packed_chunks] + tags = [tag for memory_id in packed_ids for tag in id_to_tags.get(memory_id, [])] + texts = [chunk.excerpt for chunk in result.packed_chunks] + quality = _quality(retrieved_tags=tags, retrieved_texts=texts, question=question) + rows.append({ + "question_id": str(question.get("id") or f"{case.get('id')}:{number}"), + "case_id": str(case.get("id", "case")), + "scored": _scored(question), + "context_tokens": counter(result.context), + "retrieved_tags": tags, + **quality, + }) + return rows + finally: + store.close() + + +def run( + dataset: list[dict], *, k: int = DEFAULT_K, token_budget: int = DEFAULT_TOKEN_BUDGET, + dim: int = 256, embedder: Optional[object] = None, resolve_conflicts: bool = True, +) -> dict: + """Benchmark aggregate workload context and evidence quality. + + ``token_budget`` applies identically to the recency and Engraphis methods. + Full history is intentionally uncapped: it is the complete-corpus replay + comparator whose query-context cost the other two methods seek to avoid. + The default embedder is deterministic/offline; callers may inject a real + implementation without changing the named reader-token accounting. + """ + if isinstance(token_budget, bool) or int(token_budget) < 0: + raise ValueError("token_budget must be a non-negative integer") + if isinstance(k, bool) or int(k) <= 0: + raise ValueError("k must be a positive integer") + if isinstance(dim, bool) or int(dim) <= 0: + raise ValueError("dim must be a positive integer") + + token_budget, k, dim = int(token_budget), int(k), int(dim) + counter = RegexTokenCounter() + selected_embedder = embedder if embedder is not None else DeterministicEmbedder(dim=dim) + is_offline = isinstance(selected_embedder, DeterministicEmbedder) + full_history_rows: list[dict] = [] + recency_rows: list[dict] = [] + engraphis_rows: list[dict] = [] + indexing_tokens = 0 + + for case in dataset: + memories = list(case.get("memories", [])) + source_texts = [str(memory.get("text", "")) for memory in memories] + source_tags = [str(memory.get("tag", "")) for memory in memories if memory.get("tag") is not None] + full_context = "\n\n".join(source_texts) + full_tokens = counter(full_context) + indexing_tokens += sum(counter(text) for text in source_texts) + per_case_engraphis = _engraphis_rows( + case, + k=k, + token_budget=token_budget, + embedder=selected_embedder, + counter=counter, + resolve_conflicts=bool(resolve_conflicts), + ) + engraphis_rows.extend(per_case_engraphis) + + for number, question in enumerate(case.get("questions", [])): + question_id = str(question.get("id") or f"{case.get('id')}:{number}") + common = { + "question_id": question_id, + "case_id": str(case.get("id", "case")), + "scored": _scored(question), + } + full_quality = _quality( + retrieved_tags=source_tags, retrieved_texts=source_texts, question=question, + ) + full_history_rows.append({ + **common, "context_tokens": full_tokens, "retrieved_tags": source_tags, **full_quality, + }) + recency_context, recency_tags = _recency_context( + memories, budget=token_budget, counter=counter, + ) + recency_quality = _quality( + retrieved_tags=recency_tags, retrieved_texts=[recency_context], question=question, + ) + recency_rows.append({ + **common, + "context_tokens": counter(recency_context), + "retrieved_tags": recency_tags, + **recency_quality, + }) + + methods = { + "full_history": _method_summary(full_history_rows), + "recency_window": _method_summary(recency_rows), + "engraphis": _method_summary(engraphis_rows), + } + full_tokens = methods["full_history"]["cumulative_query_context_tokens"] + engraphis_tokens = methods["engraphis"]["cumulative_query_context_tokens"] + saved_tokens = full_tokens - engraphis_tokens + savings_ratio = (saved_tokens / full_tokens) if full_tokens else 0.0 + per_query_savings = ( + methods["full_history"]["mean_query_context_tokens"] + - methods["engraphis"]["mean_query_context_tokens"] + ) + if per_query_savings > 0: + break_even: Optional[int] = max(1, int(-(-indexing_tokens // per_query_savings))) + else: + break_even = None + indexing_inclusive_total = indexing_tokens + engraphis_tokens + + return { + "benchmark": { + "name": "engraphis-context-economy/v1", + "offline": is_offline, + "embedder": { + "name": type(selected_embedder).__name__, + "model_id": getattr(selected_embedder, "model_name", None), + "revision": getattr(selected_embedder, "revision", None), + "dimension": getattr(selected_embedder, "dim", None), + }, + "token_counter": TOKEN_COUNTER_IDENTITY, + "token_budget": token_budget, + "k": k, + "resolve_conflicts": bool(resolve_conflicts), + "non_billing_scope": NON_BILLING_SCOPE, + "indexing_assumption": ( + "One complete source-memory token pass is charged once to Engraphis; " + "this is an intentionally conservative accounting proxy, not a provider price." + ), + }, + "workload": { + "cases": len(dataset), + "queries": len(full_history_rows), + "scored_queries": sum(1 for row in full_history_rows if row["scored"]), + "one_time_indexing_tokens": indexing_tokens, + }, + "methods": methods, + "engraphis_vs_full_history": { + "cumulative_query_context_tokens": engraphis_tokens, + "query_context_tokens_saved": saved_tokens, + "query_context_savings_ratio": round(savings_ratio, 6), + "one_time_indexing_inclusive_total_tokens": indexing_inclusive_total, + "indexing_inclusive_tokens_saved": full_tokens - indexing_inclusive_total, + "break_even_query_count": break_even, + "break_even_definition": ( + "Smallest whole query count where one source-corpus indexing pass plus " + "Engraphis mean reader context is no greater than full-history mean reader context; " + "null means Engraphis does not save reader-context tokens per query." + ), + }, + "detail": { + "full_history": full_history_rows, + "recency_window": recency_rows, + "engraphis": engraphis_rows, + }, + } + + +def _console_report(evaluation: dict) -> dict: + """Return the aggregate-only report that the command-line interface may print. + + ``run`` intentionally retains per-question source tags for in-process evaluation. Those + identifiers can be private dataset content, so command-line output is restricted to the + reproducible aggregate evidence rather than logging the detailed rows. + """ + benchmark = evaluation["benchmark"] + return { + "benchmark": { + "name": benchmark["name"], + "offline": benchmark["offline"], + "embedder": benchmark["embedder"], + "token_counter": benchmark["token_counter"], + "token_budget": benchmark["token_budget"], + "k": benchmark["k"], + "resolve_conflicts": benchmark["resolve_conflicts"], + "indexing_assumption": benchmark["indexing_assumption"], + "dataset_format": benchmark["dataset_format"], + }, + "workload": { + "cases": evaluation["workload"]["cases"], + "queries": evaluation["workload"]["queries"], + "scored_queries": evaluation["workload"]["scored_queries"], + "one_time_indexing_tokens": evaluation["workload"]["one_time_indexing_tokens"], + }, + "methods": { + name: { + "queries": result["queries"], + "scored_queries": result["scored_queries"], + "cumulative_query_context_tokens": result["cumulative_query_context_tokens"], + "mean_query_context_tokens": result["mean_query_context_tokens"], + "quality": result["quality"], + } + for name, result in evaluation["methods"].items() + }, + "engraphis_vs_full_history": evaluation["engraphis_vs_full_history"], + } + + +def main(argv: Optional[list[str]] = None) -> None: + parser = argparse.ArgumentParser(description="Run the offline Engraphis context economy benchmark.") + parser.add_argument( + "--dataset", + default=str(Path(__file__).resolve().parent / "datasets" / "sample.jsonl"), + ) + parser.add_argument( + "--format", choices=("harness",) + tuple(sorted(LOADERS)), default="harness", + help="dataset format: harness JSONL (default), locomo, or longmemeval", + ) + parser.add_argument("--token-budget", type=int, default=DEFAULT_TOKEN_BUDGET) + parser.add_argument("--k", type=int, default=DEFAULT_K) + parser.add_argument("--dim", type=int, default=256) + parser.add_argument( + "--embed-model", default=None, + help="optional sentence-transformers model; defaults to the deterministic offline embedder", + ) + parser.add_argument( + "--no-resolve", + action="store_true", + help="keep repeated turn-level memories separate instead of running write resolution", + ) + args = parser.parse_args(argv) + try: + embedder = get_embedder(args.embed_model, args.dim) if args.embed_model else None + dataset = ( + load_dataset(args.dataset) + if args.format == "harness" + else LOADERS[args.format](args.dataset) + ) + evaluation = run( + dataset, k=args.k, token_budget=args.token_budget, dim=args.dim, + embedder=embedder, resolve_conflicts=not args.no_resolve, + ) + evaluation["benchmark"]["dataset_format"] = args.format + except (OSError, ValueError, json.JSONDecodeError) as exc: + # Keep stdout machine-readable even for automation failures. argparse + # still owns malformed flag syntax, which is its conventional contract. + print(json.dumps({"error": str(exc)}, sort_keys=True)) + raise SystemExit(2) + print(json.dumps(_console_report(evaluation), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/eval/longmemeval_v2_evidence.py b/eval/longmemeval_v2_evidence.py new file mode 100644 index 00000000..be9d35ff --- /dev/null +++ b/eval/longmemeval_v2_evidence.py @@ -0,0 +1,252 @@ +"""Convert official LongMemEval-V2 output into a redacted evidence artifact. + +The official harness writes rich per-question logs containing prompts, gold +answers, and reader output. Those files remain private run material. This +module extracts only scores, timings, token counts, stable IDs, and digests, +then uses :mod:`eval.benchmark` to make an immutable public artifact. +""" +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +import re +from typing import Any, Optional, Sequence + +from eval.benchmark import report_envelope, write_canonical_artifact +from eval.run_longmemeval_v2 import PINNED_READER_MODEL, PINNED_READER_REVISION + + +def _load_jsonl(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError(f"per-question record {line_number} must be an object") + question_id = value.get("question_id") + if not isinstance(question_id, str) or not question_id: + raise ValueError(f"per-question record {line_number} has no question_id") + records.append(value) + if not records: + raise ValueError("per-question output contains no records") + if len({record["question_id"] for record in records}) != len(records): + raise ValueError("per-question output has duplicate question_id values") + return records + + +def _finite_number(value: Any, label: str) -> float: + """Validate an official numeric field before it enters public evidence.""" + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(float(value)) + ): + raise ValueError(f"official per-question {label} must be a finite number") + return float(value) + + +def _nonnegative_integer(value: Any, label: str) -> int: + number = _finite_number(value, label) + if number < 0 or not number.is_integer(): + raise ValueError(f"official per-question {label} must be a non-negative integer") + return int(number) + + +def _required_bool(value: Any, label: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"official per-question {label} must be boolean") + return value + + +def _normalized_record(row: dict[str, Any], *, expected_tokenizer: str) -> dict[str, Any]: + """Project one private official-harness row into a public-safe row.""" + metadata = row.get("memory_post_query_metadata") + metadata = metadata if isinstance(metadata, dict) else {} + adapter_usage = metadata.get("usage") + adapter_usage = adapter_usage if isinstance(adapter_usage, dict) else {} + reader_usage = row.get("usage") + reader_usage = reader_usage if isinstance(reader_usage, dict) else {} + tokenizer = metadata.get("tokenizer") + if tokenizer != expected_tokenizer: + raise ValueError( + "official per-question metadata does not prove the pinned reader tokenizer: " + f"expected {expected_tokenizer!r}, found {tokenizer!r}" + ) + source_ids = metadata.get("source_ids") + source_ids = [str(item) for item in source_ids] if isinstance(source_ids, list) else [] + context_tokens = _nonnegative_integer( + row.get("memory_context_token_count"), "memory_context_token_count" + ) + is_abstention = _required_bool( + row.get("is_abstention_problem"), "is_abstention_problem" + ) + is_unknown = _required_bool(row.get("is_unknown"), "is_unknown") + score = _finite_number(row.get("score"), "score") + score_bool = _required_bool(row.get("score_bool"), "score_bool") + latency_seconds = _finite_number( + row.get("memory_query_duration_seconds"), "memory_query_duration_seconds" + ) + if latency_seconds < 0: + raise ValueError("official per-question memory_query_duration_seconds must be non-negative") + raw = { + "question_id": row["question_id"], + "category": str(row.get("category") or "unknown"), + "question_text": row.get("question_text", ""), + "answer_gold": row.get("answer_gold", ""), + "response_raw": row.get("response_raw", ""), + "response_parsed_boxed": row.get("response_parsed_boxed", ""), + "memory_context": row.get("memory_context", []), + "prompt_messages": row.get("prompt_messages", []), + "retrieved_ids": source_ids, + "supporting_ids": [], + "answerable": not is_abstention, + "abstained": is_unknown, + "qa_score": score, + "qa_correct": score_bool, + "latency_ms": round(latency_seconds * 1000, 6), + "context_tokens": context_tokens, + "context_token_method": "official_harness_reader_memory_context_tokens", + "context_tokenizer_identity": tokenizer, + "usage": { + "memory_context_tokens": context_tokens, + "token_counter": tokenizer, + }, + } + optional_usage = ( + ("memory_context_original_tokens", row.get("memory_context_original_token_count")), + ("reader_prompt_tokens", reader_usage.get("prompt_tokens")), + ("reader_completion_tokens", reader_usage.get("completion_tokens")), + ("adapter_reported_context_tokens", adapter_usage.get("context_tokens")), + ) + for key, value in optional_usage: + if value is not None: + raw["usage"][key] = _nonnegative_integer(value, key) + return raw + + +def _qa_metrics(records: Sequence[dict[str, Any]]) -> dict[str, Any]: + scores = [float(record["qa_score"]) for record in records] + abstentions = [record for record in records if not record["answerable"]] + answered = [record for record in records if record["answerable"]] + return { + "official_qa": { + "available": True, + "metric": "official_harness_score", + "mean_score": sum(scores) / len(scores), + "n": len(records), + "n_answerable": len(answered), + "n_abstention": len(abstentions), + "unknown_rate": sum(bool(record["abstained"]) for record in records) / len(records), + }, + "memory_context": { + "mean_final_tokens": sum(record["context_tokens"] for record in records) / len(records), + "mean_query_latency_ms": sum(record["latency_ms"] for record in records) / len(records), + }, + } + + +def build_evidence_report( + *, + per_question_path: str | Path, + questions_path: str | Path, + haystack_path: str | Path, + trajectories_path: str | Path, + memory_config_path: str | Path, + reader_model: str = PINNED_READER_MODEL, + reader_revision: str = PINNED_READER_REVISION, + evaluator_model: Optional[str] = None, + evaluator_revision: Optional[str] = None, + command: Optional[Sequence[str]] = None, +) -> dict[str, Any]: + """Build a public-safe artifact from one completed official V2 run. + + This reports official QA scores but deliberately does not mark the result + as ``canonical``. The canonical Engraphis contract also requires a complete + five-budget retrieval curve, which an individual official reader run does + not produce. + """ + per_question = Path(per_question_path) + if re.fullmatch(r"[0-9a-f]{40}", reader_revision) is None: + raise ValueError("reader_revision must be an immutable lowercase 40-character commit") + source_paths = [ + per_question, + Path(haystack_path), + Path(trajectories_path), + Path(memory_config_path), + ] + private_rows = _load_jsonl(per_question) + tokenizer_identity = f"{reader_model}@{reader_revision}" + records = [ + _normalized_record(row, expected_tokenizer=tokenizer_identity) + for row in private_rows + ] + return report_envelope( + suite="LongMemEval-V2", + dataset_path=questions_path, + source_paths=source_paths, + config={ + "official_harness": "LongMemEval-V2", + "reader_model": reader_model, + "reader_revision": reader_revision, + "evaluator_model": evaluator_model, + "evaluator_revision": evaluator_revision, + "per_question_schema": "official_harness/per_question.jsonl", + }, + command=command or ("python", "-m", "eval.run_longmemeval_v2", ""), + token_accounting={ + "identity": tokenizer_identity, + "revision": reader_revision, + "scope": "official_harness_memory_context_item_content_excluding_prompt_framing", + "method": "official_harness_count_memory_context_tokens", + }, + models={ + "reader": {"model_id": reader_model, "revision": reader_revision}, + "evaluator": { + "model_id": evaluator_model or "not_recorded", + "revision": evaluator_revision, + }, + }, + records=records, + metrics=_qa_metrics(records), + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Redact official LongMemEval-V2 output into an immutable Engraphis evidence artifact." + ) + parser.add_argument("--per-question", required=True) + parser.add_argument("--questions", required=True) + parser.add_argument("--haystack", required=True) + parser.add_argument("--trajectories", required=True) + parser.add_argument("--memory-config", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--reader-model", default=PINNED_READER_MODEL) + parser.add_argument("--reader-revision", default=PINNED_READER_REVISION) + parser.add_argument("--evaluator-model", default=None) + parser.add_argument("--evaluator-revision", default=None) + args = parser.parse_args(argv) + try: + report = build_evidence_report( + per_question_path=args.per_question, + questions_path=args.questions, + haystack_path=args.haystack, + trajectories_path=args.trajectories, + memory_config_path=args.memory_config, + reader_model=args.reader_model, + reader_revision=args.reader_revision, + evaluator_model=args.evaluator_model, + evaluator_revision=args.evaluator_revision, + ) + result = write_canonical_artifact(report, args.output) + except (OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/performance.py b/eval/performance.py index 981f1c8a..95241998 100644 --- a/eval/performance.py +++ b/eval/performance.py @@ -35,6 +35,7 @@ from engraphis.core.context import RegexTokenCounter from engraphis.core.engine import MemoryEngine from engraphis.core.interfaces import MemoryType, Scope, SearchFilter +from engraphis.core.retrieval_policy import CANDIDATE_DEPTH_MODES, RETRIEVAL_PROFILES from engraphis.core.store import Store from eval import metrics from eval.harness import load_dataset @@ -79,6 +80,7 @@ class _Measurements: source_tokens: list[int] full_payload_tokens: list[int] compact_payload_tokens: list[int] + candidate_depths: list[int] quality: list[dict] @@ -178,11 +180,21 @@ def _measure_recall( search_filter: SearchFilter, *, k: int, + candidate_k: int, + candidate_depth: str, token_budget: int, + retrieval_profile: str, ) -> tuple[dict, float]: started = time.perf_counter_ns() result = engine.recall_engine.recall( - question["q"], search_filter, k=k, reinforce=False, token_budget=token_budget + question["q"], + search_filter, + k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, + reinforce=False, + token_budget=token_budget, + retrieval_profile=retrieval_profile, ) return result, (time.perf_counter_ns() - started) / 1_000_000 @@ -193,12 +205,24 @@ def _measure_batch( search_filter: SearchFilter, *, k: int, + candidate_k: int, + candidate_depth: str, token_budget: int, + retrieval_profile: str, concurrency: int, ) -> list[tuple[dict, float]]: if concurrency == 1: return [ - _measure_recall(engine, question, search_filter, k=k, token_budget=token_budget) + _measure_recall( + engine, + question, + search_filter, + k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, + token_budget=token_budget, + retrieval_profile=retrieval_profile, + ) for question in questions ] with ThreadPoolExecutor(max_workers=concurrency) as executor: @@ -209,7 +233,10 @@ def _measure_batch( question, search_filter, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, token_budget=token_budget, + retrieval_profile=retrieval_profile, ) for question in questions ] @@ -220,11 +247,14 @@ def _run_single( dataset: list[dict], *, k: int, + candidate_k: int, + candidate_depth: str, dim: int, warmups: int, iterations: int, filler_memories: int, token_budget: int, + retrieval_profile: str, config: AcceptanceConfig, process_number: int, embedder: Optional[DeterministicEmbedder] = None, @@ -283,7 +313,10 @@ def _run_single( questions, search_filter, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, token_budget=token_budget, + retrieval_profile=retrieval_profile, concurrency=config.concurrency, ) for _ in range(warmups): @@ -292,11 +325,14 @@ def _run_single( questions, search_filter, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, token_budget=token_budget, + retrieval_profile=retrieval_profile, concurrency=config.concurrency, ) - measurements = _Measurements([], [], [], [], [], [], []) + measurements = _Measurements([], [], [], [], [], [], [], []) counter = RegexTokenCounter() for iteration in range(iterations): for question_number, (result, latency_ms) in enumerate(_measure_batch( @@ -304,7 +340,10 @@ def _run_single( questions, search_filter, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, token_budget=token_budget, + retrieval_profile=retrieval_profile, concurrency=config.concurrency, )): measurements.warm_latencies_ms.append(latency_ms) @@ -324,6 +363,7 @@ def _run_single( measurements.compact_payload_tokens.append( _serialized_tokens(compact_payload, counter) ) + measurements.candidate_depths.append(result.candidate_k_used) question = questions[question_number] measurements.quality.append({ "question": question_number, @@ -362,9 +402,12 @@ def _build_report( measurements: list[_Measurements], *, k: int, + candidate_k: int, + candidate_depth: str, warmups: int, iterations: int, token_budget: int, + retrieval_profile: str, config: AcceptanceConfig, question_count: int, resources: list[dict], @@ -375,6 +418,7 @@ def _build_report( source_tokens = [value for item in measurements for value in item.source_tokens] full_payload_tokens = [value for item in measurements for value in item.full_payload_tokens] compact_payload_tokens = [value for item in measurements for value in item.compact_payload_tokens] + candidate_depths = [value for item in measurements for value in item.candidate_depths] quality = [value for item in measurements for value in item.quality] full_total = sum(full_payload_tokens) compact_total = sum(compact_payload_tokens) @@ -396,6 +440,13 @@ def _build_report( "corpus": base["corpus"], "run": { "k": k, + "candidate_k": candidate_k, + "candidate_depth": candidate_depth, + "actual_candidate_k": { + "min": min(candidate_depths, default=0), + "max": max(candidate_depths, default=0), + "mean": round(sum(candidate_depths) / max(len(candidate_depths), 1), 2), + }, "warmups": warmups, "iterations": iterations, # Kept for compatibility: these are the warm, steady-state timed recalls. @@ -403,6 +454,7 @@ def _build_report( "cold_timed_recalls": len(cold_latencies), "warm_timed_recalls": len(warm_latencies), "token_budget": token_budget, + "retrieval_profile": retrieval_profile, }, "acceptance": { "concurrency": config.concurrency, @@ -455,11 +507,14 @@ def run( dataset: list[dict], *, k: int = 5, + candidate_k: int = 50, + candidate_depth: str = "fixed", dim: int = 256, warmups: int = 1, iterations: int = 5, filler_memories: int = 0, token_budget: int = 1500, + retrieval_profile: str = "balanced", embedder: Optional[DeterministicEmbedder] = None, concurrency: int = 1, processes: int = 1, @@ -473,10 +528,19 @@ def run( intentionally limited to the established single-process API. """ k = max(1, int(k)) + candidate_k = max(1, int(candidate_k)) + candidate_depth = str(candidate_depth or "").strip().casefold() + if candidate_depth not in CANDIDATE_DEPTH_MODES: + choices = ", ".join(sorted(CANDIDATE_DEPTH_MODES)) + raise ValueError(f"candidate_depth must be one of: {choices}") warmups = max(0, int(warmups)) iterations = max(1, int(iterations)) filler_memories = max(0, int(filler_memories)) token_budget = max(0, int(token_budget)) + retrieval_profile = str(retrieval_profile or "").strip().casefold() + if retrieval_profile not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValueError(f"retrieval_profile must be one of: {choices}") if canonical: raise ValueError("canonical acceptance requires run_acceptance_matrix") config = AcceptanceConfig( @@ -494,11 +558,14 @@ def run( base, measurement = _run_single( dataset, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, dim=dim, warmups=warmups, iterations=iterations, filler_memories=filler_memories, token_budget=token_budget, + retrieval_profile=retrieval_profile, config=config, process_number=0, embedder=embedder, @@ -507,9 +574,12 @@ def run( base, [measurement], k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, warmups=warmups, iterations=iterations, token_budget=token_budget, + retrieval_profile=retrieval_profile, config=config, question_count=question_count, resources=[base["resources"]], @@ -517,11 +587,14 @@ def run( worker_args = { "k": k, + "candidate_k": candidate_k, + "candidate_depth": candidate_depth, "dim": dim, "warmups": warmups, "iterations": iterations, "filler_memories": filler_memories, "token_budget": token_budget, + "retrieval_profile": retrieval_profile, "config": config, } with ProcessPoolExecutor(max_workers=config.processes) as executor: @@ -535,9 +608,12 @@ def run( base, [measurement for _, measurement in process_results], k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, warmups=warmups, iterations=iterations, token_budget=token_budget, + retrieval_profile=retrieval_profile, config=config, question_count=question_count, resources=[result["resources"] for result, _ in process_results], @@ -558,11 +634,14 @@ def run_acceptance_matrix( dataset: list[dict], *, k: int = 5, + candidate_k: int = 50, + candidate_depth: str = "fixed", dim: int = 256, warmups: int = 1, iterations: int = 5, filler_memories: int = 0, token_budget: int = 1500, + retrieval_profile: str = "balanced", processes: int = 5, minimum_queries: int = 1000, concurrencies: Optional[list[int]] = None, @@ -588,11 +667,14 @@ def run_acceptance_matrix( slices[str(concurrency)] = run( dataset, k=k, + candidate_k=candidate_k, + candidate_depth=candidate_depth, dim=dim, warmups=warmups, iterations=iterations, filler_memories=filler_memories, token_budget=token_budget, + retrieval_profile=retrieval_profile, concurrency=concurrency, processes=processes, minimum_queries=effective_minimum, @@ -623,7 +705,11 @@ def _print(report: dict) -> None: print( "Engraphis performance — " f"{corpus['memories']} memories · {corpus['questions']} questions · " - f"{run_info['timed_recalls']} warm timed recalls @ k={run_info['k']}" + f"{run_info['timed_recalls']} warm timed recalls @ k={run_info['k']} " + f"(candidates={run_info['candidate_k']}, " + f"actual={run_info['actual_candidate_k']['mean']:.1f}, " + f"depth={run_info['candidate_depth']}, " + f"profile={run_info['retrieval_profile']})" ) print( " environment : " @@ -684,11 +770,29 @@ def main(argv: Optional[list[str]] = None) -> int: "--dataset", default=str(Path(__file__).resolve().parent / "datasets" / "codemem.jsonl"), ) + parser.add_argument( + "--candidate-depth", + choices=sorted(CANDIDATE_DEPTH_MODES), + default="fixed", + help="fixed preserves the requested depth; adaptive uses a profile-aware bounded pool", + ) parser.add_argument("--k", type=int, default=5) + parser.add_argument( + "--candidate-k", + type=int, + default=50, + help="per-arm candidate depth; sweep this to measure quality/latency tradeoffs", + ) parser.add_argument("--dim", type=int, default=256) parser.add_argument("--warmups", type=int, default=1) parser.add_argument("--iterations", type=int, default=5) parser.add_argument("--token-budget", type=int, default=1500) + parser.add_argument( + "--retrieval-profile", + choices=sorted(RETRIEVAL_PROFILES), + default="balanced", + help="retrieval policy to benchmark (default: balanced)", + ) parser.add_argument( "--filler-memories", type=int, @@ -731,11 +835,14 @@ def main(argv: Optional[list[str]] = None) -> int: report = run_acceptance_matrix( dataset, k=args.k, + candidate_k=args.candidate_k, + candidate_depth=args.candidate_depth, dim=args.dim, warmups=args.warmups, iterations=args.iterations, filler_memories=args.filler_memories, token_budget=args.token_budget, + retrieval_profile=args.retrieval_profile, processes=args.processes, minimum_queries=args.minimum_queries, ) @@ -743,11 +850,14 @@ def main(argv: Optional[list[str]] = None) -> int: report = run( dataset, k=args.k, + candidate_k=args.candidate_k, + candidate_depth=args.candidate_depth, dim=args.dim, warmups=args.warmups, iterations=args.iterations, filler_memories=args.filler_memories, token_budget=args.token_budget, + retrieval_profile=args.retrieval_profile, concurrency=args.concurrency, processes=args.processes, minimum_queries=args.minimum_queries, diff --git a/pyproject.toml b/pyproject.toml index ffe4ba53..57c006f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,7 +199,7 @@ include = ["engraphis*", "scripts*", "eval*"] "engraphis.classic_assets" = ["*.html", "*.css", "*.js", "*.png", "*.ico", "vendor/*", "vendor/**/*"] "engraphis.dashboard_assets" = ["*.html", "*.css", "*.js", "*.png", "*.ico", "vendor/*", "vendor/**/*"] "engraphis" = ["commercial_manifest.json"] -"eval" = ["BASELINES.md", "configs/*.json", "datasets/*.jsonl"] +"eval" = ["BASELINES.md", "EVIDENCE.md", "configs/*.json", "datasets/*.jsonl"] [tool.setuptools.exclude-package-data] "*" = ["*.pyc", "*.pyo", "__pycache__/*"] diff --git a/scripts/check_codeql_sarif.py b/scripts/check_codeql_sarif.py index 57802831..44a1e6c7 100644 --- a/scripts/check_codeql_sarif.py +++ b/scripts/check_codeql_sarif.py @@ -11,11 +11,9 @@ MAX_REPORTED_FINDINGS = 50 -def _location(result: dict[str, Any]) -> str: - locations = result.get("locations") - if not isinstance(locations, list) or not locations: +def _physical_location(physical: Any) -> str: + if not isinstance(physical, dict): return "" - physical = locations[0].get("physicalLocation", {}) artifact = physical.get("artifactLocation", {}) region = physical.get("region", {}) path = artifact.get("uri", "") @@ -23,6 +21,37 @@ def _location(result: dict[str, Any]) -> str: return f"{path}:{line}" if isinstance(line, int) else str(path) +def _location(result: dict[str, Any]) -> str: + locations = result.get("locations") + if not isinstance(locations, list) or not locations: + return "" + return _physical_location(locations[0].get("physicalLocation")) + + +def _code_flows(result: dict[str, Any]) -> list[str]: + """Return compact source-to-sink paths from a SARIF path-problem result.""" + flows: list[str] = [] + for code_flow in result.get("codeFlows", []): + if not isinstance(code_flow, dict): + continue + for thread_flow in code_flow.get("threadFlows", []): + if not isinstance(thread_flow, dict): + continue + locations = thread_flow.get("locations", []) + if not isinstance(locations, list) or not locations: + continue + endpoints = [] + for location in (locations[0], locations[-1]): + if not isinstance(location, dict): + continue + entry = location.get("location", location) + if isinstance(entry, dict): + endpoints.append(_physical_location(entry.get("physicalLocation"))) + if endpoints: + flows.append(" -> ".join(endpoints)) + return flows + + def findings_in(path: Path) -> list[str]: """Return bounded, human-readable findings from one SARIF file.""" @@ -32,7 +61,9 @@ def findings_in(path: Path) -> list[str]: for result in run.get("results", []): rule = result.get("ruleId", "") message = result.get("message", {}).get("text", "") - findings.append(f"{rule} at {_location(result)}: {message}") + flow = _code_flows(result) + suffix = f" [flow: {'; '.join(flow)}]" if flow else "" + findings.append(f"{rule} at {_location(result)}: {message}{suffix}") return findings diff --git a/scripts/externalize_dashboard_assets.py b/scripts/externalize_dashboard_assets.py index 0d5b5ae5..a89affe0 100644 --- a/scripts/externalize_dashboard_assets.py +++ b/scripts/externalize_dashboard_assets.py @@ -39,6 +39,10 @@ #: script moves it out of the parsed ``" + ) + + _styles, scripts = assets._inline_assets(html) + + assert [asset.content for asset in scripts] == ["first()", "second()"] + assert [html[asset.start:asset.end] for asset in scripts] == [ + '", + ] + + def test_migrate_uses_parsed_asset_boundaries(tmp_path, monkeypatch): index = tmp_path / "index.html" css = tmp_path / "dashboard.css" diff --git a/tests/test_graph_engine_asset.py b/tests/test_graph_engine_asset.py index 5945e1da..ca19c885 100644 --- a/tests/test_graph_engine_asset.py +++ b/tests/test_graph_engine_asset.py @@ -158,7 +158,7 @@ def test_opt_in_graph_asset_is_lazily_loaded_after_its_dependencies() -> None: source = DASHBOARD.read_text(encoding="utf-8") assert "script.src='/static/vendor/force-graph.min.js'" in source assert ( - "script.src='/v2-assets/engraphis-graph.js?v=20260728-reference-materials'" + "script.src='/v2-assets/engraphis-graph.js?v=20260730-drag-stability'" in source ) render = source[source.index("function graphRender("):] @@ -279,7 +279,7 @@ def test_graph_engine_deep_link_reaches_the_next_engine_after_a_lazy_load() -> N report = _run_routing("loads") assert report["appended"] == [ - "/v2-assets/engraphis-graph.js?v=20260728-reference-materials" + "/v2-assets/engraphis-graph.js?v=20260730-drag-stability" ] # It waits rather than rendering something wrong in the meantime. assert report["beforeSettle"] == {"engine": 0, "classic": 0} @@ -294,7 +294,7 @@ def test_classic_route_reaches_the_canonical_engine_without_a_query_flag() -> No report = _run_routing("classic") assert report["appended"] == [ - "/v2-assets/engraphis-graph.js?v=20260728-reference-materials" + "/v2-assets/engraphis-graph.js?v=20260730-drag-stability" ] assert report["beforeSettle"] == {"engine": 0, "classic": 0} assert report["engine"] == 1 @@ -401,6 +401,21 @@ def test_node_geometry_stays_compact_for_small_overviews_and_is_style_neutral() assert "if(sun)r*=1.7;" not in DASHBOARD.read_text(encoding="utf-8") +@requires_node +def test_auto_fit_cap_does_not_limit_manual_graph_inspection() -> None: + """The auto-fit guard must not become a global force-graph zoom limit.""" + report = _run_engine( + """ + G.create(el, {}); + emit({ maxZoom: store.maxZoom === undefined ? null : store.maxZoom }); + """ + ) + assert report["maxZoom"] is None + source = ASSET.read_text(encoding="utf-8") + assert "function autoFit(" in source + assert "api.fit = () => { if (!destroyed) fg.zoomToFit" in source + + def test_dashboard_falls_back_to_the_classic_renderer_when_the_engine_throws() -> None: source = DASHBOARD.read_text(encoding="utf-8") # The opt-in flag must be latched off after a failure, and the render path must catch. @@ -2147,6 +2162,25 @@ def test_engine_exposes_a_teardown_and_the_dashboard_drives_it() -> None: assert "GRAPH_ENGINE.destroy()" in dashboard +def test_manual_drag_controller_detaches_with_the_graph() -> None: + """Reopening Ledger must not leave stale pointer controllers on the shared pane.""" + source = ASSET.read_text(encoding="utf-8") + assert "let detachManualDrag = null;" in source + assert "el.addEventListener('pointerdown', beginManualDrag, true);" in source + assert "el.removeEventListener('pointerdown', beginManualDrag, true);" in source + assert "window.removeEventListener('pointermove', moveManualDrag, true);" in source + assert "event.type !== 'pointercancel'" in source + direct_click = source[source.index("} else if (event.type !== 'pointercancel') {"):] + direct_click = direct_click[:direct_click.index(" };", 1)] + assert direct_click.index("handleNodeClick(current.node);") < direct_click.index("suppressNodeClick();") + move = source[source.index("const moveManualDrag = event => {"):] + move = move[:move.index(" const beginManualDrag", 1)] + assert "if (!manualDrag.dragged)" in move + assert move.index("if (Math.hypot(dx, dy) < 3)") < move.index("const node = manualDrag.node;") + teardown = source[source.index("api.destroy = () => {"):] + assert "if (detachManualDrag) { detachManualDrag(); detachManualDrag = null; }" in teardown + + def test_reduced_motion_is_honoured_by_the_opt_in_renderer() -> None: source = ASSET.read_text(encoding="utf-8") dashboard = DASHBOARD.read_text(encoding="utf-8") diff --git a/tests/test_install_shortcuts.py b/tests/test_install_shortcuts.py new file mode 100644 index 00000000..a8c574a7 --- /dev/null +++ b/tests/test_install_shortcuts.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import sys + +import pytest + +from scripts import install_shortcuts +from scripts.install_shortcuts import _desktop_path, _remove_shortcuts, _shortcut_paths + + +def test_windows_desktop_path_uses_the_known_folder(monkeypatch, tmp_path): + home = tmp_path / "Home" + redirected = tmp_path / "OneDrive" / "Desktop" + + class Result: + stdout = str(redirected) + "\n" + + monkeypatch.setattr(install_shortcuts.subprocess, "run", lambda *args, **kwargs: Result()) + + assert _desktop_path("Windows", home) == redirected + + +def test_windows_uninstall_uses_the_same_known_desktop_folder(monkeypatch, tmp_path): + home = tmp_path / "Home" + redirected = tmp_path / "OneDrive" / "Desktop" + captured = {} + + monkeypatch.setattr(sys, "argv", ["install-shortcuts", "--uninstall"]) + monkeypatch.setattr(install_shortcuts.platform, "system", lambda: "Windows") + monkeypatch.setattr(install_shortcuts.Path, "home", lambda: home) + monkeypatch.setattr(install_shortcuts, "_desktop_path", lambda system, received_home: redirected) + monkeypatch.setattr( + install_shortcuts, + "_remove_shortcuts", + lambda system, desktop, start_menu, *, home: captured.update( + system=system, desktop=desktop, start_menu=start_menu, home=home + ) or [], + ) + + install_shortcuts.main() + + assert captured["system"] == "Windows" + assert captured["desktop"] == redirected + assert captured["home"] == home + + +@pytest.mark.parametrize("system", ["Windows", "Darwin", "Linux"]) +def test_remove_shortcuts_removes_only_known_artifacts_and_is_idempotent(tmp_path, system): + desktop = tmp_path / "Desktop" + start_menu = tmp_path / "Start Menu" / "Programs" + home = tmp_path / "Home" + desktop.mkdir(parents=True) + + expected = _shortcut_paths(system, desktop, start_menu, home=home) + for path in expected: + if path.suffix == ".app": + (path / "Contents").mkdir(parents=True) + (path / "Contents" / "Info.plist").write_text("owned artifact") + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("owned artifact") + + untouched = home / "Applications" / "Unrelated.app" + untouched.mkdir(parents=True) + (untouched / "keep.txt").write_text("keep") + nearby = desktop / "other-shortcut.desktop" + nearby.write_text("keep") + + assert _remove_shortcuts(system, desktop, start_menu, home=home) == expected + assert all(not path.exists() and not path.is_symlink() for path in expected) + assert untouched.is_dir() + assert nearby.read_text() == "keep" + + assert _remove_shortcuts(system, desktop, start_menu, home=home) == [] + + +def test_uninstall_cli_needs_no_desktop_and_does_not_prompt(monkeypatch, tmp_path): + home = tmp_path / "Home" + captured = {} + + monkeypatch.setattr(sys, "argv", ["install-shortcuts", "--uninstall"]) + monkeypatch.setattr(install_shortcuts.platform, "system", lambda: "Linux") + monkeypatch.setattr(install_shortcuts.Path, "home", lambda: home) + monkeypatch.setattr( + install_shortcuts, + "_remove_shortcuts", + lambda system, desktop, start_menu, *, home: captured.update( + system=system, desktop=desktop, start_menu=start_menu, home=home + ) or [], + ) + + install_shortcuts.main() + + assert captured["system"] == "Linux" + assert captured["desktop"] == home / "Desktop" + assert captured["home"] == home diff --git a/tests/test_longmemeval_v2_evidence.py b/tests/test_longmemeval_v2_evidence.py new file mode 100644 index 00000000..ff896680 --- /dev/null +++ b/tests/test_longmemeval_v2_evidence.py @@ -0,0 +1,135 @@ +import json + +import pytest + +from eval.benchmark import validate_report, write_canonical_artifact +from eval.longmemeval_v2_evidence import build_evidence_report + + +def _write_json(path, value): + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +def test_official_v2_evidence_export_redacts_private_prompt_material(tmp_path): + questions = _write_json(tmp_path / "questions.json", [{"question_id": "q1"}]) + haystack = _write_json(tmp_path / "haystack.json", {"q1": ["trajectory-1"]}) + trajectories = _write_json(tmp_path / "trajectories.json", [{"id": "trajectory-1"}]) + config = _write_json(tmp_path / "memory.json", {"memory_type": "engraphis"}) + private = { + "question_id": "q1", + "category": "static", + "question_text": "private question text", + "answer_gold": "private gold answer", + "response_raw": "private reader answer", + "response_parsed_boxed": "private boxed answer", + "memory_context": [{"type": "text", "value": "private retrieved context"}], + "prompt_messages": [{"role": "user", "content": "private prompt"}], + "memory_query_duration_seconds": 0.0125, + "memory_context_original_token_count": 19, + "memory_context_token_count": 11, + "memory_post_query_metadata": { + "tokenizer": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "source_ids": ["mem_1"], + "usage": {"context_tokens": 9}, + }, + "usage": {"prompt_tokens": 101, "completion_tokens": 7}, + "is_abstention_problem": False, + "is_unknown": False, + "score": 1.0, + "score_bool": True, + } + per_question = tmp_path / "per_question.jsonl" + per_question.write_text(json.dumps(private) + "\n", encoding="utf-8") + + report = build_evidence_report( + per_question_path=per_question, + questions_path=questions, + haystack_path=haystack, + trajectories_path=trajectories, + memory_config_path=config, + evaluator_model="example/evaluator", + evaluator_revision="b" * 40, + ) + + assert validate_report(report) == [] + record = report["records"][0] + for field in ( + "question_text", "answer_gold", "response_raw", "response_parsed_boxed", + "memory_context", "prompt_messages", + ): + assert field not in record + assert len(record["query_sha256"]) == 64 + assert len(record["answer_or_response_sha256"]) == 64 + assert len(record["context_or_prompt_sha256"]) == 64 + assert record["retrieved_ids"] == ["mem_1"] + assert report["metrics"]["official_qa"]["mean_score"] == 1.0 + assert report["protocol"]["token_accounting"]["method"] == ( + "official_harness_count_memory_context_tokens" + ) + assert report["protocol"]["token_accounting"]["scope"] == ( + "official_harness_memory_context_item_content_excluding_prompt_framing" + ) + assert {item["name"] for item in report["suite"]["sources"]} == { + "per_question.jsonl", "haystack.json", "trajectories.json", "memory.json", + } + serialized = json.dumps(report) + for private_value in ( + "private question text", "private gold answer", "private reader answer", + "private retrieved context", "private prompt", + ): + assert private_value not in serialized + + artifact = tmp_path / "public.json" + written = write_canonical_artifact(report, artifact) + assert written["sha256"] in artifact.with_name("public.json.sha256").read_text("ascii") + + +def test_official_v2_evidence_export_rejects_unpinned_reader_metadata(tmp_path): + source_paths = [ + _write_json(tmp_path / name, {} if name != "questions.json" else []) + for name in ("questions.json", "haystack.json", "trajectories.json", "memory.json") + ] + per_question = tmp_path / "per_question.jsonl" + per_question.write_text(json.dumps({ + "question_id": "q1", + "memory_post_query_metadata": {"tokenizer": "engraphis.regex.v1"}, + }) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="pinned reader tokenizer"): + build_evidence_report( + per_question_path=per_question, + questions_path=source_paths[0], + haystack_path=source_paths[1], + trajectories_path=source_paths[2], + memory_config_path=source_paths[3], + ) + + +def test_official_v2_evidence_export_rejects_malformed_measured_fields(tmp_path): + source_paths = [ + _write_json(tmp_path / name, {} if name != "questions.json" else []) + for name in ("questions.json", "haystack.json", "trajectories.json", "memory.json") + ] + per_question = tmp_path / "per_question.jsonl" + per_question.write_text(json.dumps({ + "question_id": "q1", + "memory_post_query_metadata": { + "tokenizer": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a", + }, + "memory_context_token_count": -1, + "is_abstention_problem": False, + "is_unknown": False, + "score": 1.0, + "score_bool": True, + "memory_query_duration_seconds": 0.1, + }) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="memory_context_token_count"): + build_evidence_report( + per_question_path=per_question, + questions_path=source_paths[0], + haystack_path=source_paths[1], + trajectories_path=source_paths[2], + memory_config_path=source_paths[3], + ) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 8e623f51..b22ad4c8 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -62,7 +62,8 @@ def _recall_side_effect_snapshot(srv): "engraphis_stats", "engraphis_proactive_context", "engraphis_recall_grounded", "engraphis_answer", "engraphis_ingest", "engraphis_consolidate", "engraphis_ingest_postgres_schema", - "engraphis_receipts", "engraphis_verify_receipts", "engraphis_export_receipts", + "engraphis_receipts", "engraphis_context_savings", "engraphis_verify_receipts", + "engraphis_export_receipts", "engraphis_check_update", } @@ -79,17 +80,18 @@ def test_server_identity_and_tools_registered(): assert "engraphis_end_session" in srv.mcp.instructions assert "open_threads=[]" in srv.mcp.instructions tools = {t.name: t for t in asyncio.run(srv.mcp.list_tools())} - assert len(_ALL_TOOLS) == 30 + assert len(_ALL_TOOLS) == 31 assert set(tools) == _ALL_TOOLS + assert srv.minimum_role("engraphis_context_savings") == "viewer" kilo = (ROOT / "docs" / "KILO_CODE_INTEGRATION.md").read_text(encoding="utf-8") - full_surface = kilo.split("## 4. The 30 tools", 1)[1].split("\n---", 1)[0] + full_surface = kilo.split("## 4. The 31 tools", 1)[1].split("\n---", 1)[0] assert set(re.findall(r"`(engraphis_[a-z_]+)`", full_surface)) == _ALL_TOOLS # Flat schema (not a nested "params" object) so agents can call fields directly. props = tools["engraphis_remember"].inputSchema.get("properties", {}) assert "content" in props and "workspace" in props and "params" not in props assert {"valid_from", "subject_key", "claim_kind"} <= set(props) assert "as_of" in tools["engraphis_recall"].inputSchema.get("properties", {}) - assert {"valid_at", "known_at", "token_budget", "retrieval_profile", + assert {"valid_at", "known_at", "token_budget", "retrieval_profile", "candidate_depth", "response_mode", "diagnostics"} <= set( tools["engraphis_recall"].inputSchema.get("properties", {}) ) @@ -97,7 +99,7 @@ def test_server_identity_and_tools_registered(): "token_budget" ]["default"] == 1024 assert "as_of" in tools["engraphis_recall_grounded"].inputSchema.get("properties", {}) - assert {"valid_at", "known_at", "token_budget", "retrieval_profile", "response_mode"} <= set( + assert {"valid_at", "known_at", "token_budget", "retrieval_profile", "candidate_depth", "response_mode"} <= set( tools["engraphis_answer"].inputSchema.get("properties", {}) ) assert {"as_of", "valid_at", "known_at"} <= set( @@ -558,6 +560,9 @@ def test_receipt_tools(monkeypatch): ) listed = json.loads(srv.engraphis_receipts(workspace="acme")) assert listed["entries"][0]["operation"] == "remember" + savings = json.loads(srv.engraphis_context_savings(workspace="acme")) + assert savings["receipt_count"] == 1 + assert savings["savings_receipt_count"] == 0 verified = json.loads(srv.engraphis_verify_receipts(workspace="acme")) assert verified["valid"] is True exported = json.loads(srv.engraphis_export_receipts(workspace="acme")) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 5778451c..9207a319 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -123,11 +123,15 @@ def test_distribution_configuration_includes_public_evidence_tools(): pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8") assert 'include = ["engraphis*", "scripts*", "eval*"]' in pyproject - assert '"eval" = ["BASELINES.md", "configs/*.json", "datasets/*.jsonl"]' in pyproject + assert ( + '"eval" = ["BASELINES.md", "EVIDENCE.md", "configs/*.json", "datasets/*.jsonl"]' + in pyproject + ) for rule in ( "include LICENSE NOTICE README.md CHANGELOG.md BENCHMARKS.md", "recursive-include eval *.py", "include eval/BASELINES.md", + "include eval/EVIDENCE.md", "recursive-include eval/configs *.json", "recursive-include eval/datasets *.jsonl", ): diff --git a/tests/test_read_only_api.py b/tests/test_read_only_api.py index 633a7793..4633de32 100644 --- a/tests/test_read_only_api.py +++ b/tests/test_read_only_api.py @@ -18,17 +18,28 @@ def test_read_only_api_requires_token_and_does_not_reinforce(): ).fetchone()["n"] client = TestClient(create_read_only_app(svc, token="secret")) assert client.get("/recall", params={"query": "database", "workspace": "w"}).status_code == 401 + # Receipt-derived savings are still scoped usage information, so the new + # endpoint must stay behind the same bearer gate as recall. + assert client.get("/context-savings", params={"workspace": "w"}).status_code == 401 response = client.get( "/recall", - params={"query": "database", "workspace": "w"}, + params={"query": "database", "workspace": "w", "candidate_depth": "adaptive"}, headers={"Authorization": "Bearer secret"}, ) assert response.status_code == 200 and response.json()["count"] == 1 + assert response.json()["candidate_depth"] == "adaptive" + assert response.json()["candidate_k_used"] < response.json()["candidate_k_requested"] lowercase = client.get( "/recall", params={"query": "database", "workspace": "w"}, headers={"Authorization": "bearer secret"}, ) assert lowercase.status_code == 200 + savings = client.get( + "/context-savings", params={"workspace": "w"}, + headers={"Authorization": "Bearer secret"}, + ) + assert savings.status_code == 200 + assert savings.json()["format"] == "engraphis-context-savings/1" assert response.headers["x-frame-options"] == "DENY" assert svc.store.get_memory(memory["id"]).access_count == before assert svc.store.conn.execute( @@ -51,10 +62,30 @@ def test_read_only_api_serves_graph_and_intent_recall(): assert client.get("/graph?workspace=w&layers=").json()["edges"] == [] response = client.post( "/intent/recall", - json={"query": "Alice", "intent": "explain", "workspace": "w"}, + json={ + "query": "Alice", "intent": "explain", "workspace": "w", + "candidate_depth": "adaptive", + }, ) assert response.status_code == 200 assert response.json()["operation"] == "recall" + assert response.json()["candidate_depth"] == "adaptive" + + +def test_read_only_api_serves_content_free_context_savings(): + svc = MemoryService.create(":memory:", graph_extractor="none") + svc.remember("Context savings test.", workspace="w", scope="workspace") + svc.recall("context savings", workspace="w", token_budget=64) + + response = TestClient(create_read_only_app(svc)).get( + "/context-savings", params={"workspace": "w"} + ) + + assert response.status_code == 200 + body = response.json() + assert body["format"] == "engraphis-context-savings/1" + assert body["savings_receipt_count"] == 1 + assert body["by_token_counter"][0]["source_tokens"] >= body["by_token_counter"][0]["saved_tokens"] def test_read_only_code_search_forwards_bitemporal_anchors(): diff --git a/tests/test_receipts.py b/tests/test_receipts.py index c692416e..403ac3ef 100644 --- a/tests/test_receipts.py +++ b/tests/test_receipts.py @@ -494,6 +494,127 @@ def write(index): store.close() +def test_context_savings_is_scoped_content_free_and_groups_token_counters(): + service = MemoryService.create(":memory:") + stored = service.remember( + "Receipt saving marker PURPLE-FOX-177.", workspace="acme", repo="api" + ) + wid = service.store.get_or_create_workspace("acme") + rid = service._lookup_repo(wid, "api") + assert rid is not None + service.store.record_receipt( + "recall", workspace_id=wid, repo_id=rid, actor="PURPLE-FOX-177", + metadata={"token_usage": { + "budget_tokens": 80, "source_tokens": 100, "context_tokens": 25, + "saved_tokens": 75, "savings_ratio": 0.75, "packed_count": 2, + "omitted_count": 3, "token_counter": "engraphis.regex.v1", + }}, + ) + service.store.record_receipt( + "grounded_recall", workspace_id=wid, repo_id=rid, + metadata={"token_usage": { + "budget_tokens": 40, "source_tokens": 40, "context_tokens": 40, + "saved_tokens": 0, "savings_ratio": 0.0, "packed_count": 1, + "omitted_count": 0, "token_counter": "engraphis.regex.v1", + }}, + ) + service.store.record_receipt( + "recall", workspace_id=wid, + metadata={"token_usage": { + "budget_tokens": 20, "source_tokens": 20, "context_tokens": 5, + "saved_tokens": 15, "savings_ratio": 0.75, "packed_count": 1, + "omitted_count": 1, "token_counter": "estimate_tokens", + }}, + ) + service.store.record_receipt( + "recall", workspace_id=wid, repo_id=rid, + metadata={"token_usage": { + "budget_tokens": 10, "source_tokens": 10, "context_tokens": 8, + "saved_tokens": 9, "token_counter": "engraphis.regex.v1", + }}, + ) + + repo_summary = service.context_savings(workspace="acme", repo="api") + assert repo_summary["scope"] == {"workspace": "acme", "repo": "api"} + assert repo_summary["receipt_chain_valid"] is True + assert repo_summary["receipt_chain_error_count"] == 0 + assert repo_summary["receipt_count"] == 4 + assert repo_summary["savings_receipt_count"] == 2 + assert repo_summary["incomplete_usage_receipt_count"] == 1 + assert repo_summary["by_token_counter"] == [{ + "token_counter": "engraphis.regex.v1", + "receipt_count": 2, + "source_tokens": 140, + "context_tokens": 65, + "saved_tokens": 75, + "budget_tokens": 120, + "packed_count": 3, + "omitted_count": 3, + "savings_ratio": 75 / 140, + "by_operation": [ + {"operation": "grounded_recall", "receipt_count": 1, + "source_tokens": 40, "context_tokens": 40, "saved_tokens": 0, + "budget_tokens": 40, "packed_count": 1, "omitted_count": 0, + "savings_ratio": 0.0}, + {"operation": "recall", "receipt_count": 1, + "source_tokens": 100, "context_tokens": 25, "saved_tokens": 75, + "budget_tokens": 80, "packed_count": 2, "omitted_count": 3, + "savings_ratio": 0.75}, + ], + }] + workspace_summary = service.context_savings(workspace="acme") + assert [row["token_counter"] for row in workspace_summary["by_token_counter"]] == [ + "engraphis.regex.v1", "estimate_tokens" + ] + assert "PURPLE-FOX-177" not in json.dumps(workspace_summary) + assert stored["receipt"]["operation"] == "remember" + + poisoned = service.store.record_receipt("recall", workspace_id=wid) + service.store.conn.execute( + "UPDATE operation_receipts SET payload=? WHERE id=?", + ('{"query":"PURPLE-FOX-177"}', poisoned["id"]), + ) + service.store.conn.commit() + poisoned_summary = service.context_savings(workspace="acme") + assert poisoned_summary["invalid_receipt_count"] == 1 + assert poisoned_summary["receipt_chain_valid"] is False + assert poisoned_summary["receipt_chain_error_count"] > 0 + assert "PURPLE-FOX-177" not in json.dumps(poisoned_summary) + + +def test_context_savings_excludes_receipts_with_reassigned_repo_ids(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + original_repo = store.get_or_create_repo(workspace_id, "api") + reassigned_repo = store.get_or_create_repo(workspace_id, "other") + receipt = store.record_receipt( + "recall", + workspace_id=workspace_id, + repo_id=original_repo, + metadata={"token_usage": { + "source_tokens": 100, + "context_tokens": 20, + "saved_tokens": 80, + "token_counter": "estimate_tokens", + }}, + ) + + # The signed payload remains valid, but the relational repo column is not its scope. + store.conn.execute( + "UPDATE operation_receipts SET repo_id=? WHERE id=?", + (reassigned_repo, receipt["id"]), + ) + store.conn.commit() + + assert store.verify_receipts(workspace_id=workspace_id)["valid"] is True + summary = store.context_savings(workspace_id=workspace_id, repo_id=reassigned_repo) + assert summary["receipt_count"] == 1 + assert summary["invalid_receipt_count"] == 1 + assert summary["usage_receipt_count"] == 0 + assert summary["savings_receipt_count"] == 0 + assert summary["by_token_counter"] == [] + + def test_service_records_and_exports_operation_receipts(): service = MemoryService.create(":memory:") stored = service.remember( diff --git a/tests/test_release_infrastructure.py b/tests/test_release_infrastructure.py index e14f2a19..20264381 100644 --- a/tests/test_release_infrastructure.py +++ b/tests/test_release_infrastructure.py @@ -49,13 +49,16 @@ def test_published_image_and_railway_template_fail_safe_to_customer_mode(): assert removed not in template["variables"] -def test_compose_api_profile_requires_a_token_for_its_non_loopback_bind(): +def test_compose_api_profile_defers_token_gate_until_profile_startup(): compose = _text("docker-compose.yml") api_profile = compose.split(" engraphis-api:\n", 1)[1].split("\nvolumes:", 1)[0] readme = _text("README.md") + launcher = _text("scripts/start_server.py") assert "ENGRAPHIS_HOST: 0.0.0.0" in api_profile - assert "ENGRAPHIS_API_TOKEN: ${ENGRAPHIS_API_TOKEN:?" in api_profile + assert "ENGRAPHIS_API_TOKEN: ${ENGRAPHIS_API_TOKEN:-}" in api_profile + assert 'not os.environ.get("ENGRAPHIS_API_TOKEN", "").strip()' in launcher + assert 'ap.error("non-loopback serving requires ENGRAPHIS_API_TOKEN")' in launcher assert "ENGRAPHIS_API_TOKEN='generate-a-strong-unique-value'" in readme @@ -234,7 +237,7 @@ def test_primary_github_release_targets_repository_without_checkout(): def test_public_capability_and_support_docs_match_the_shipped_tree(): server = _text("engraphis/mcp_server.py") tools = re.findall(r'@mcp\.tool\(\s*name="(engraphis_[^"]+)"', server) - assert len(tools) == len(set(tools)) == 30 + assert len(tools) == len(set(tools)) == 31 readme = _text("README.md") architecture = _text("docs/ARCHITECTURE_V3.md") @@ -245,8 +248,8 @@ def test_public_capability_and_support_docs_match_the_shipped_tree(): assert "28 MCP tools" not in content assert "28-tool" not in content assert "(28 of them)" not in content - assert "30 MCP tools" in architecture - assert "(30 of them)" in skill + assert "31 MCP tools" in architecture + assert "(31 of them)" in skill assert "recall_context (compact)" in architecture assert "engraphis_recall_context" in readme assert "`engraphis_check_update`" in readme diff --git a/tests/test_retrieval_policy.py b/tests/test_retrieval_policy.py index 07f9a9c7..bae80951 100644 --- a/tests/test_retrieval_policy.py +++ b/tests/test_retrieval_policy.py @@ -83,6 +83,62 @@ def test_empty_requested_profile_defaults_to_balanced() -> None: assert DeterministicRetrievalPolicy().resolve("", "src/api.py -> Handler.handle()").name == "balanced" +@pytest.mark.parametrize( + ("profile", "expected"), + [ + ("lexical", 10), + ("balanced", 15), + ("graph", 30), + ("code", 30), + ], +) +def test_adaptive_candidate_depth_is_profile_aware_and_bounded( + profile: str, expected: int +) -> None: + depth, reason = DeterministicRetrievalPolicy().candidate_depth( + "ordinary query", k=5, ceiling=50, profile=profile, mode="adaptive" + ) + + assert depth == expected + assert profile in reason + + +def test_fixed_candidate_depth_preserves_the_requested_ceiling() -> None: + depth, reason = DeterministicRetrievalPolicy().candidate_depth( + "ordinary query", k=5, ceiling=7, profile="balanced", mode="fixed" + ) + + assert depth == 7 + assert reason == "fixed requested depth" + + +def test_recall_exposes_the_adaptive_candidate_depth_used(): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("eval") + repo_id = engine.store.get_or_create_repo(workspace_id, "candidate-depth") + engine.remember( + "The API authenticates with PASETO v4 tokens.", + workspace_id=workspace_id, + repo_id=repo_id, + mtype=MemoryType.SEMANTIC, + scope=Scope.REPO, + resolve_conflicts=False, + ) + + result = engine.recall( + "Which token format authenticates the API?", + workspace_id=workspace_id, + repo_id=repo_id, + k=5, + candidate_depth="adaptive", + ) + + assert result.candidate_depth_mode == "adaptive" + assert result.candidate_k_requested == 50 + assert result.candidate_k_used == 15 + assert result.candidate_depth_reason == "adaptive balanced floor" + + @pytest.mark.parametrize("name", ["auto", "", "unknown"]) def test_profile_config_requires_a_concrete_profile(name: str) -> None: with pytest.raises(ValueError, match="resolve to one of"): diff --git a/tests/test_sync.py b/tests/test_sync.py index 9b0a1aa3..6fb8ff3b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -278,7 +278,7 @@ def test_sync_v2_preserves_closed_memory_link_history(): for memory_id in ("mem_a", "mem_b"): source.add_memory(MemoryRecord( id=memory_id, content=memory_id, workspace_id=source_ws, - valid_from=1.0, ingested_at=1.0, + scope=Scope.WORKSPACE, valid_from=1.0, ingested_at=1.0, )) source.add_link( "mem_a", "mem_b", relation="related", layer="semantic", reason="old", @@ -321,7 +321,7 @@ def peer(valid_from: float, ingested_at: float): for memory_id in ("mem_a", "mem_b"): store.add_memory(MemoryRecord( id=memory_id, content=memory_id, workspace_id=workspace_id, - valid_from=1.0, ingested_at=1.0, + scope=Scope.WORKSPACE, valid_from=1.0, ingested_at=1.0, )) store.add_link( "mem_a", "mem_b", relation="related", layer="semantic", reason="peer", @@ -911,6 +911,172 @@ def test_remote_bundle_cannot_overwrite_existing_row_across_session_boundary( assert existing.scope == local_scope +def test_remote_bundle_rejects_global_scope_with_repo_pointer(): + store = Store(":memory:") + workspace = store.get_or_create_workspace("w") + repo_b = store.get_or_create_repo(workspace, "repo-b") + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {"remote-a": "repo-a"}, + "memories": [{ + "id": "mem_malformed", + "content": "repo-a-only sentinel", + "scope": "workspace", + "repo_id": "remote-a", + }], + "mem_links": [], + } + + report = SyncEngine(store).apply_bundle(bundle) + + assert report["rejected"] == 1 and report["added"] == 0 + assert store.get_memory("mem_malformed") is None + visible_in_repo_b = store.list_memories(SearchFilter( + workspace_id=workspace, repo_id=repo_b, include_ancestors=True, + )) + assert all(memory.id != "mem_malformed" for memory in visible_in_repo_b) + + +def test_remote_bundle_rejects_repo_scope_without_repo_pointer(): + store = Store(":memory:") + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {}, + "memories": [{ + "id": "mem_orphaned_repo", + "content": "repo fact with no owner", + "scope": "repo", + }], + "mem_links": [], + } + + report = SyncEngine(store).apply_bundle(bundle) + + assert report["rejected"] == 1 and report["added"] == 0 + assert store.get_memory("mem_orphaned_repo") is None + + +def test_remote_bundle_rejects_invalid_scope_change_on_existing_repo_memory(): + store = Store(":memory:") + workspace = store.get_or_create_workspace("w") + repo_a = store.get_or_create_repo(workspace, "repo-a") + store.add_memory(MemoryRecord( + id="mem_existing_repo", + content="local repo fact", + workspace_id=workspace, + repo_id=repo_a, + scope=Scope.REPO, + last_access=1.0, + )) + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {"remote-a": "repo-a"}, + "memories": [{ + "id": "mem_existing_repo", + "content": "malformed global overwrite", + "scope": "workspace", + "repo_id": "remote-a", + "last_access": time.time() + 86_400, + }], + "mem_links": [], + } + + report = SyncEngine(store).apply_bundle(bundle) + + assert report["rejected"] == 1 and report["updated"] == 0 + existing = store.get_memory("mem_existing_repo") + assert existing.content == "local repo fact" + assert existing.scope == Scope.REPO + assert existing.repo_id == repo_a + + +@pytest.mark.parametrize( + ("local_scope", "incoming_scope", "include_remote_repo"), + [ + (Scope.REPO, "workspace", False), + (Scope.WORKSPACE, "repo", True), + ], +) +def test_remote_bundle_cannot_change_existing_memory_visibility( + local_scope, incoming_scope, include_remote_repo): + """A valid incoming row must still not re-scope an existing local identity.""" + store = Store(":memory:") + workspace = store.get_or_create_workspace("w") + repo_a = store.get_or_create_repo(workspace, "repo-a") + store.add_memory(MemoryRecord( + id="mem_scope_stable", + content="local visibility sentinel", + workspace_id=workspace, + repo_id=repo_a if local_scope == Scope.REPO else None, + scope=local_scope, + last_access=1.0, + )) + remote_repo_id = "remote-a" if include_remote_repo else None + memory = { + "id": "mem_scope_stable", + "content": "remote scope rewrite", + "scope": incoming_scope, + "last_access": time.time() + 86_400, + } + if remote_repo_id is not None: + memory["repo_id"] = remote_repo_id + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {"remote-a": "repo-a"} if include_remote_repo else {}, + "memories": [memory], + "mem_links": [], + } + + report = SyncEngine(store).apply_bundle(bundle) + + assert report["rejected"] == 1 and report["updated"] == 0 + existing = store.get_memory("mem_scope_stable") + assert existing.scope == local_scope + assert existing.repo_id == (repo_a if local_scope == Scope.REPO else None) + + +def test_remote_bundle_cannot_choose_visibility_for_legacy_orphaned_memory(): + store = Store(":memory:") + workspace = store.get_or_create_workspace("w") + store.add_memory(MemoryRecord( + id="mem_legacy_orphan", + content="legacy local value", + workspace_id=workspace, + repo_id=None, + scope=Scope.REPO, + last_access=1.0, + )) + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {}, + "memories": [{ + "id": "mem_legacy_orphan", + "content": "remote elevation attempt", + "scope": "workspace", + "last_access": time.time() + 86_400, + }], + "mem_links": [], + } + + report = SyncEngine(store).apply_bundle(bundle) + + assert report["rejected"] == 1 and report["updated"] == 0 + existing = store.get_memory("mem_legacy_orphan") + assert existing.content == "legacy local value" + assert existing.scope == Scope.REPO + assert existing.repo_id is None + + def test_remote_bundle_cannot_overwrite_or_downgrade_local_secret(): store = Store(":memory:") workspace = store.get_or_create_workspace("w") @@ -1126,7 +1292,8 @@ def test_replaying_a_bundle_reports_all_unchanged(remote_content): wid = store.get_or_create_workspace("w") syncer = SyncEngine(store) store.add_memory(MemoryRecord(id="mem_a", content="local", workspace_id=wid, - last_access=100.0, ingested_at=90.0, valid_from=1.0)) + scope=Scope.WORKSPACE, last_access=100.0, + ingested_at=90.0, valid_from=1.0)) # valid_from is set explicitly here, exactly as export_bundle/record_to_dict emit it. # A bundle that OMITS it converges too, but only because apply_bundle inherits # store-defaulted fields from the existing row — see the dedicated test below. @@ -1164,7 +1331,7 @@ def _valid_from_less_bundle(content): the content-hash tiebreak — the only place the omission can decide anything.""" return { "format": SYNC_FORMAT, "version": 1, "workspace_name": "w", "repos": {}, - "memories": [{"id": "mem_a", "content": content, + "memories": [{"id": "mem_a", "content": content, "scope": "workspace", "last_access": 100.0, "ingested_at": 90.0}], "mem_links": [], } @@ -1195,7 +1362,8 @@ def test_bundle_omitting_valid_from_never_rewrites_the_stored_default(content): wid = store.get_or_create_workspace("w") syncer = SyncEngine(store) store.add_memory(MemoryRecord(id="mem_a", content=content, workspace_id=wid, - last_access=100.0, ingested_at=90.0, valid_from=1000.0)) + scope=Scope.WORKSPACE, last_access=100.0, + ingested_at=90.0, valid_from=1000.0)) bundle = _valid_from_less_bundle(content) for _ in range(6): @@ -1246,7 +1414,8 @@ def test_incoming_valid_from_still_wins_when_genuinely_supplied(): wid = store.get_or_create_workspace("w") syncer = SyncEngine(store) store.add_memory(MemoryRecord(id="mem_a", content="local", workspace_id=wid, - last_access=100.0, ingested_at=90.0, valid_from=1.0)) + scope=Scope.WORKSPACE, last_access=100.0, + ingested_at=90.0, valid_from=1.0)) bundle = { "format": SYNC_FORMAT, "version": 1, "workspace_name": "w", "repos": {}, "memories": [{"id": "mem_a", "content": "remote", "valid_from": 5000.0, diff --git a/tests/test_sync_cli.py b/tests/test_sync_cli.py index fc84c860..b4d26b17 100644 --- a/tests/test_sync_cli.py +++ b/tests/test_sync_cli.py @@ -205,6 +205,23 @@ def test_cli_selects_folder(db_with_workspace, _capture_transport, tmp_path): assert rc == 0 assert _capture_transport["kind"] == "folder" assert _capture_transport["kw"]["root"] == share + assert _capture_transport["kw"]["create"] is True + + +def test_cli_folder_dry_run_does_not_create_missing_remote(db_with_workspace, tmp_path): + share = tmp_path / "missing-share" + + rc = sync_main([ + "--db", db_with_workspace, + "--workspace", "acme", + "--remote", str(share), + "--dry-run", + ]) + + assert rc == 0 + assert not share.exists() + engine = MemoryEngine.create(db_with_workspace) + assert engine.store.get_sync_state("device_id") is None def test_cli_bare_relay_falls_back_to_config(db_with_workspace, _capture_transport, monkeypatch):