diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256 index f10cbbff..445d3791 100644 --- a/.claude-plugin/skill-assets.sha256 +++ b/.claude-plugin/skill-assets.sha256 @@ -1,6 +1,6 @@ a972f840492bf1041f1e9525452127a6d5d117d5fcd231230b1d2ae656d85d13 .claude-plugin/marketplace.json dba3e0713559b267581f45cfd5907aae4689df77790b02bbcf22309d9ffa73ef .claude-plugin/plugin.json -89bf2728e44a3c877e31982d387fcfab10fbe065eb6441b2792375ca5105c07c skills/engraphis-memory/SKILL.md -a295b0448e2ff372ddd8ea4e0bc8dc53f3d4bd56ff1fbd0d88cf4b6179511ce1 skills/engraphis-memory/references/CONVENTIONS.md +656caf07c9064b219eb974e018180e6a7a88f2c058fb1b1c6a8a36074d67e9cc skills/engraphis-memory/SKILL.md +9751b6e7310151c14e6bb7f5d683943a53a95951ff207876053d082e65eecfd5 skills/engraphis-memory/references/CONVENTIONS.md 45f4b4ad9dbfd39f2b377083d9b3eec5eed7cba7cb8fa139e3f420bdd6105343 skills/engraphis-memory/references/SCOPING.md -12980f16face01fee5f65cde0c4aa2297200b2252861b341ec7762e32600da60 skills/engraphis-memory/references/TOOLS.md +dc83c48d1a57122e7b58da29d32ae3b8ebd4ffff0b6d0570a8833e920e365109 skills/engraphis-memory/references/TOOLS.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f2aabf5..6698426a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,9 +161,10 @@ jobs: run: | python -m pip install --upgrade pip build pip-audit python -m build + python scripts/verify_distribution_contents.py dist/* python -m venv .audit-venv .audit-venv/bin/python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" .audit-venv/bin/python -m pip install dist/*.whl AUDIT_SITE=$(.audit-venv/bin/python -c "import site; print(site.getsitepackages()[0])") python -m pip_audit --path "$AUDIT_SITE" - .audit-venv/bin/python -c "import engraphis; print('wheel import OK')" + .audit-venv/bin/python -c "import engraphis, eval.harness; print('wheel imports OK')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76a1b606..dddee14d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,13 +61,18 @@ jobs: ruff check . python -c "import fastapi, httpx, mcp, multipart, pydantic, uvicorn" python -m pytest -o addopts="" tests/ -q -rs + python -m pytest -o addopts="" tests/test_public_research_boundary.py -q + python -m pytest -o addopts="" tests/test_compact_recall.py tests/test_eval_performance.py -q + python -m pytest -o addopts="" tests/test_eval_harness.py tests/test_benchmark_evidence.py -q python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 python -m eval.ablation python -m pip_audit --local - name: Build source and universal wheel distributions - run: python -m build + run: | + python -m build + python scripts/verify_distribution_contents.py dist/* - name: Validate distributions run: python -m twine check dist/* @@ -134,7 +139,7 @@ jobs: npm ci npx playwright install --with-deps chromium - name: Playwright desktop/mobile, keyboard, CSP, console, and axe checks - run: npx playwright test + run: npm run test:e2e docker-smoke: name: Production image release gate @@ -176,9 +181,58 @@ jobs: if: always() run: docker rm -f engraphis-release || true + release-evidence: + name: Generate public release evidence + needs: [build, python-matrix, browser-accessibility, docker-smoke] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + - name: Install SBOM generator and project dependencies + run: >- + python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" + cyclonedx-bom==7.3.0 ".[all,test]" + - name: Download distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: python-package-distributions + path: dist/ + - name: Generate evidence and reproducible SBOM after all release gates + shell: bash + run: | + mkdir release-evidence + sbom="release-evidence/engraphis-${GITHUB_REF_NAME#v}.cdx.json" + cyclonedx-py environment --output-reproducible --of JSON --pyproject pyproject.toml -o "$sbom" + python scripts/release_evidence.py --dist dist --commit "$GITHUB_SHA" \ + --tag "$GITHUB_REF_NAME" \ + --sbom "$sbom" \ + --verified-check ruff \ + --verified-check pytest \ + --verified-check privacy-boundary \ + --verified-check token-efficiency \ + --verified-check benchmark-schema-evidence \ + --verified-check browser-e2e \ + --verified-check dependency-audit \ + --verified-check container-smoke \ + --verified-check retrieval-sample \ + --verified-check retrieval-codemem \ + --verified-check retrieval-ablation \ + --output release-evidence/release-evidence.json + - name: Store public release evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: public-release-evidence + path: release-evidence/ + publish: name: Publish to PyPI - needs: [build, python-matrix, browser-accessibility, docker-smoke] + needs: release-evidence # Manual dispatch is intentionally build/check-only. Publication requires a pushed # semver tag, whose value was matched to pyproject.toml in the build job above. if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') @@ -236,6 +290,12 @@ jobs: name: python-package-distributions path: dist/ + - name: Download public release evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: public-release-evidence + path: release-evidence/ + - name: Create GitHub Release env: GH_TOKEN: ${{ github.token }} @@ -246,11 +306,11 @@ jobs: # A previous partial attempt may have created the release before every # canonical package asset uploaded. Reconcile same-named assets from the # exact aggregate that passed the publish gate. - gh release upload "$GITHUB_REF_NAME" dist/* \ + gh release upload "$GITHUB_REF_NAME" dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json \ --repo "$GH_REPO" \ --clobber else - gh release create "$GITHUB_REF_NAME" dist/* \ + gh release create "$GITHUB_REF_NAME" dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json \ --repo "$GH_REPO" \ --verify-tag \ --generate-notes \ @@ -319,11 +379,44 @@ jobs: (.conclusion == "success" or .conclusion == "failure"))] | length' \ <<<"$jobs")" -eq 1 + test "$(jq '[.jobs[] | select(.name == "Generate public release evidence" and + .conclusion == "success")] | length' \ + <<<"$jobs")" -eq 1 gh run download "$run_id" \ --repo "$GH_REPO" \ --name python-package-distributions \ --dir dist + gh run download "$run_id" \ + --repo "$GH_REPO" \ + --name public-release-evidence \ + --dir release-evidence + python - "$RELEASE_TAG" "$tag_sha" <<'PY' + import hashlib + import json + import sys + from pathlib import Path + + tag, commit = sys.argv[1:] + with open("release-evidence/release-evidence.json", encoding="utf-8") as handle: + evidence = json.load(handle) + assert evidence.get("format") == "engraphis-release-evidence/2" + assert evidence.get("package", {}).get("version") == tag.removeprefix("v") + assert evidence.get("tag") == tag + assert evidence.get("commit") == commit + assert evidence.get("provenance", {}).get("source") == {"tag": tag, "commit": commit} + expected = { + item["filename"]: item["sha256"] + for item in evidence.get("artifacts", []) + } + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in Path("dist").iterdir() + if path.is_file() and (path.name.endswith(".whl") or path.name.endswith(".tar.gz")) + } + assert expected == actual + PY + - name: Verify any previously published subset env: RELEASE_TAG: ${{ inputs.release_tag }} @@ -361,11 +454,11 @@ jobs: shell: bash run: | if gh release view "$RELEASE_TAG" --repo "$GH_REPO" >/dev/null 2>&1; then - gh release upload "$RELEASE_TAG" dist/* \ + gh release upload "$RELEASE_TAG" dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json \ --repo "$GH_REPO" \ --clobber else - gh release create "$RELEASE_TAG" dist/* \ + gh release create "$RELEASE_TAG" dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json \ --repo "$GH_REPO" \ --verify-tag \ --generate-notes \ diff --git a/AGENTS.md b/AGENTS.md index 2947ff38..bd7b30e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ most common mistake here. | Status | Primary scoped, bi-temporal, interface-driven implementation. | Compatibility/reference implementation with flat namespaces. | | Model | Scoped + bi-temporal + typed; interface-driven. | Single flat `namespace` string per memory. | | Code | `engraphis/core/`, `engraphis/backends/`, `eval/`, `tests/`, `scripts/migrate_to_v2.py` | `engraphis/app.py`, `config.py`, `models.py`, `routes/`, `stores/`, `engines/`, `llm/`, `static/` | -| Data | new v2 schema (`SCHEMA_VERSION = 4`) | `engraphis_v1.db` | +| Data | new v2 schema (`SCHEMA_VERSION = 6`) | `engraphis_v1.db` | | Entry | `MemoryEngine.create()` → `core/engine.py` | `python -m scripts.start_server` → FastAPI on :8700 | **Rule:** build new capability on **v2** (`core/` + `backends/`) behind the interfaces. @@ -79,8 +79,6 @@ python -m scripts.cli recall "what do we know about X" -n vault # CLI: ingest python -m scripts.migrate_to_v2 --old engraphis_v1.db --new engraphis_v2.db --dry-run python -m scripts.migrate_to_v2 --old engraphis_v1.db --new engraphis_v2.db -# ── Seed memories from an Obsidian/markdown vault (v1) ─────────────────────── -python -m scripts.seed_from_obsidian "C:/path/to/Vault" --namespace vault ``` `requires-python >= 3.9` (ruff targets `py39`); CI and the recommended dev environment use **3.11**. @@ -93,15 +91,17 @@ python -m scripts.seed_from_obsidian "C:/path/to/Vault" --namespace vault ``` query - └─ SearchFilter (scope + as_of time anchor) core/interfaces.py - └─ 3 retrieval arms (run in parallel, then fused): + └─ SearchFilter (scope + valid_at/known_at anchors) core/interfaces.py + └─ 4 retrieval arms (run in parallel, then fused): • vector — VectorIndex.search (cosine) backends/vector_*.py • lexical — Store.fts_search (FTS5/BM25 + LIKE fallback) core/store.py • graph — Personalized PageRank over entities+links core/recall.py + core/graphrank.py (graph_mode="1hop" keeps the old expansion for ablation) + • code — symbols/files/calls with memory bridges core/engine.py └─ RRF fusion + six-term weighted score core/scoring.py └─ rerank top-N backends/reranker.py - └─ context packing (token budget) + reinforce() core/recall.py / core/store.py + └─ context packing (token budget) + optional explicit reinforcement + core/recall.py / core/store.py ``` Backends are selected by `get_embedder()` / `get_vector_index()` / `get_reranker()` and @@ -177,7 +177,7 @@ These are pure, unit-tested functions — change them only with a corresponding --- -## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 4`) +## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 6`) - **Scope hierarchy:** `workspace → repo → session → memory`. Scopes: `session|repo|workspace|user`. - **Bi-temporal validity on every record:** world-time `valid_from/valid_to` + @@ -188,7 +188,8 @@ These are pure, unit-tested functions — change them only with a corresponding Lexicographic sort == chronological. - **Tables:** `workspaces`, `repos`, `sessions`, `memories`, `mem_vectors`, `mem_fts` (FTS5 + plain-table fallback), `entities`, `edges` (bi-temporal), `mem_links`, - `symbols`, `code_edges`, `code_files`, `code_memory_links`, `operation_receipts`, + `memory_entities`, `symbols`, `code_edges`, `code_files`, `code_memory_links`, + `operation_receipts`, `events`, `audit`, `schema_migrations`. - **Vectors are stored L2-normalized** so cosine similarity == dot product. @@ -221,7 +222,7 @@ These are pure, unit-tested functions — change them only with a corresponding - **`README.md`** — installation, product surfaces, configuration, and public API usage. - **`CHANGELOG.md`** — shipped capability and release history. Keep phase/status ledgers out of this operating manual. -- **`docs/SYNC.md`** — cloud sync (Pro): architecture, the convergent merge, CLI usage, the +- **`docs/SYNC.md`** — cloud sync (Pro): architecture, the convergent merge, CLI usage, and the untrusted-bundle security model. - **`AGENTS.md`** (this file) + **`CLAUDE.md`** — how to work in the repo. - **`skills/engraphis-memory/`** — portable Agent Skill (SKILL.md + `references/`) that teaches any diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 1da576fd..3d4f58db 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -19,15 +19,25 @@ and stated everywhere the numbers appear (`eval/external.py`). them through the *real* `MemoryEngine` write path (conflict resolution + evolution) and hybrid recall with a real sentence-transformers embedder. It reports `recall_at_k` / `hit_at_k` / `answer_token_recall` — i.e. *did the evidence come back*, not *did an LLM answer correctly*. + It retains source categories and abstention/no-evidence questions as explicit exclusions from + retrieval-only aggregates rather than silently dropping them. `eval.longmemeval_v2` is a local, + text-only adapter for the official LongMemEval-V2 `insert(trajectory)` / `query(query, + query_image=None)` memory interface; it does not download data or call a model. - **Grounded** — `eval/grounded.py`: answerable → cite, off-topic → abstain. - **Chunking (quality per token)** — `eval/chunking_eval.py` over `eval/datasets/longdoc.jsonl` ingests a multi-topic corpus twice — one memory per document (`whole`) vs. sub-file `ChunkingExtractor` (`chunked`) — and queries both through the real recall pipeline. This is the first cut of the context-reduction metric (item 3 below). On the deterministic embedder: - **recall@5 1.000 for both, at ~73% fewer context tokens (826 → 224) and ~4× smaller + **recall@5 1.000 for both, at ~73% fewer context tokens (809 → 219) and ~4× smaller tokens-to-evidence (162 → 42).** Pass `--embed-model sentence-transformers/all-MiniLM-L6-v2` for a real retrieval number (recall should then favour chunked on larger corpora, not just tie). +- **Full-pipeline latency + quality** — `eval/performance.py` times the shipped semantic + + lexical + graph + fusion + scoring + rerank + packing path after warmup, with reinforcement + disabled so repeated measurements do not mutate their corpus. It reports p50/p95/p99 latency, + retrieval quality, and packed context tokens in one JSON-safe schema. `--filler-memories` + provides deterministic corpus scaling, and every report records the runtime, architecture, + embedder, vector backend, corpus size, warmups, and iteration count. ### Reproduce @@ -38,6 +48,11 @@ python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 python -m eval.harness --dataset eval/datasets/graph_multihop.jsonl --k 5 python -m eval.ablation +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 \ + --iterations 5 --filler-memories 1000 +# Canonical latency/resource protocol: requires >=1,000 queries and five processes. +python -m eval.performance --dataset fixed-1000-plus.jsonl --acceptance-matrix --processes 5 # Real retrieval numbers (downloads all-MiniLM-L6-v2) python -m eval.external --dataset longmemeval_s.json --format longmemeval --k 10 @@ -47,22 +62,86 @@ python -m eval.external --dataset locomo10.json --format locomo --k 10 ## What we do NOT yet claim - **No end-to-end QA accuracy.** Official LoCoMo / LongMemEval QA scores depend on an answering model and evaluator. Engraphis isolates retrieval and does not present that result as end-to-end answer accuracy. -- **No published latency.** There is no measured p50/p95 recall latency in-repo; we have not - measured our equivalent. The Rust hot path (Phase 6) is not started. +- **No hosted-service latency comparison.** The in-repo p50/p95/p99 benchmark covers the local + reference pipeline and records its environment; unlike environments are not compared. - **No neutral third-party ranking.** We have not run an external eval platform. -## Plan to produce publishable numbers +Every publishable run should emit the `engraphis-benchmark/v2` envelope: dataset/config hashes, +per-question records, explicit exclusions, fixed-budget context curves, and deterministic +stratified or paired bootstrap confidence intervals. Every run names its token counter. +Noncanonical offline fixtures may identify a deterministic estimate; canonical public evidence +requires the exact pinned reader tokenizer and immutable model revision. The lightweight CI +fixtures validate that machinery; they are not a claim about external benchmark performance. + +The benchmark context metric reads strict recall usage fields rather than inferring prompt size: +`budget_tokens`, `context_tokens`, `source_tokens`, `saved_tokens`, `savings_ratio`, +`packed_count`, `omitted_count`, and `token_counter`. Use `engraphis_recall_context` for a +hard-budget prompt packet; legacy `engraphis_recall` remains available in full or compact response +mode for compatibility. + +### Canonical public artifacts + +Use `python -m eval.benchmark --input report.json --output artifacts/run.json` to validate a +report and write sorted, immutable JSON plus `run.json.sha256`. The command permits an identical +retry but refuses to replace a different artifact at the same path. For an official +LongMemEval-V2 run, add `--canonical`: this requires a profile with an exact benchmark repository +revision, dataset revision, reader model revision, and embedding model revision. The checked-in +profile pins immutable upstream commits; replacing any revision with a mutable tag fails +validation. Canonical profiles label the baseline (`no_retrieval`, `lexical_only`, `dense_only`, +`dense_lexical_rrf`, `full_hybrid`, `full_history`, `no_graph`, `no_reranker`, +`no_temporal_resolution`, or `whole_document`) and declare the required fixed context-budget +matrix: 256, 512, 1024, 2048, and 4096 tokens. Canonical in-repo reports rerun every question at +all five budgets and validate each aggregate against its per-question evidence. The checked-in +LongMemEval-V2 memory-module configuration sets the official adapter's operating point to 1,024 +tokens; that single official point must not be presented as a five-point curve. + +`eval.external --canonical` refuses `--limit` and rejects a normalized output that omitted source +cases. Retrieval-only abstention/no-evidence records remain visible in the artifact's +`exclusions`; they are not counted as evidence-retrieval scores. + +### LongMemEval-V2 memory-module adapter + +`eval.longmemeval_v2.EngraphisLongMemEvalV2Memory` follows the official +`memory_modules.memory.Memory` interface at LongMemEval-V2 commit +`6f020ac2fc3275e46c706d3406e02c3ed79b7be2`. When imported in that environment, its +`@register_memory` decorator registers `memory_type="engraphis"`; use the checked-in +[`eval/configs/longmemeval_v2_engraphis.json`](eval/configs/longmemeval_v2_engraphis.json) +with the official harness. The config pins `Qwen/Qwen3-Embedding-8B` to revision +`1d8ad4ca9b3dd8059ad90a75d4983776a23d44af`; mutable embedding revisions are rejected, and a +canonical adapter run fails instead of relabeling the deterministic offline fallback as Qwen. +Run `python -m eval.run_longmemeval_v2` with the official harness arguments and the pinned +checkout on `PYTHONPATH`. This wrapper performs the upstream registry import in the required order +before delegating to `evaluation.harness`; a direct upstream invocation must otherwise import +`eval.longmemeval_v2` before calling `build_memory`. + +The checked-in configuration is canonical only when the adapter resolves the pinned Qwen reader +processor at `c202236235762e1c871ad0ccb60c8ee5ba337b9a`. The wrapper also forces the audited +official harness's otherwise-unpinned `AutoProcessor` call to that same revision. It refuses to +start if the optional processor dependency or immutable revision is unavailable; the local regex +counter is never silently relabeled as a reader budget. The recorded budget counts each returned +context item's content with that reader tokenizer (without prompt framing or inter-item +separators), so it is a hard **evidence-item content** budget, not a claim about total chat-prompt +tokens. Packed sources are returned as separate context items, preserving the largest fitting +evidence prefix instead of dropping one oversized monolithic item. The adapter does not download +benchmark data or call the reader/evaluator; the official harness owns those steps. + +## Next steps for external publishable numbers 1. **Add a QA layer to `eval/external.py`.** Optional answering model + judge on top of the - existing retrieval pipeline, so we can report end-to-end accuracy on the same datasets the - field quotes — reusing the retrieval harness underneath. -2. **Measure recall latency.** Instrument `RecallEngine.recall()` end to end (parallel arms → - RRF → score → rerank → pack) and publish p50/p95 on a fixed corpus and machine class. -3. **Adopt a context-reduction metric.** Report **recall@k against tokens injected** — recall - at a fixed token budget, and tokens-to-first-correct-evidence. It is a natural fit: Engraphis - already does token-budget context packing in recall and already reports a **compaction** - number from consolidation (`core/consolidate.py::_compaction`). Wire those two together into - one "quality per token" curve — arguably our strongest story, since decay + consolidation - are built to raise it. -4. **Run an external eval platform** for a neutral comparison once (1)–(3) exist. + existing retrieval pipeline, so the official datasets can report end-to-end accuracy while + reusing the retrieval harness underneath. +2. **Publish production-backend latency.** Run `eval/performance.py` with the real embedder and + sqlite-vec/backend configuration on a fixed machine class and corpus scale. +3. **Run the fixed-budget curve on the complete official datasets.** The v2 harness now measures + every question at 256, 512, 1,024, 2,048, and 4,096 evidence tokens and validates the + per-question records, aggregates, and pinned reader-tokenizer identity. Publish the curve only + after complete official runs produce immutable artifacts for every point. +4. **Run an external evaluation platform** once (1)–(3) exist. + +## Evaluation question +The predeclared question is whether the full vector + lexical/BM25 + sparse PPR graph + calibrated +rerank pipeline, bi-temporal resolution, and grounded abstention produce higher evidence recall +per injected token than the registered baselines. The answer must come from a complete, +machine-readable artifact with paired confidence intervals; otherwise the release reports +“no demonstrated improvement.” diff --git a/CHANGELOG.md b/CHANGELOG.md index ff60e634..48867108 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,41 @@ All notable changes to Engraphis are documented here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/); versions use SemVer. +## Unreleased + +### Added + +- `engraphis_recall_context` brings the MCP surface to 30 tools and is the compact, hard-budget + path for agent prompts. It returns packed context, compact source identities, strict token usage + fields, optional retrieval diagnostics, and preserves `engraphis_recall` as the full-response + compatibility surface. +- Recall and grounded recall now expose `valid_at` (world time) and `known_at` (system time); + `as_of` remains the compatible `valid_at` alias and conflicting anchors are rejected. Retrieval + defaults to the `balanced` profile; `auto` remains explicit opt-in. +- MCP and HTTP remember calls can set a fact's world-time `valid_from`; recall, grounded recall, + and the compatibility answer tool can run a point-in-time `as_of` query. +- `eval.performance` reports full recall-pipeline quality, packed context tokens, and + p50/p95/p99 latency with a reproducible JSON schema and deterministic corpus scaling. +- Schema v5 adds temporal history for symbols, code edges, code-memory links, and persisted + memory-entity incidence. Code retrieval is now a first-class profile, and graph walks use + bounded sparse PageRank instead of a dense quadratic transition matrix. +- Optional `subject_key` and `claim_kind` make mutable claims explicit. Uncertain similar facts + are conservatively related while keyed or strongly evidenced contradictions supersede. +- `engraphis-benchmark/v2`, canonical workspace exports, and release-evidence manifests provide + deterministic hashes, per-question records, fixed token-budget curves, and validation before + public evidence is written. + +### Fixed + +- Supersessions now close the old fact at the replacement's effective world time instead of its + ingestion time. Superseded, corrected, promoted, merged, forgotten, and consolidated source + vectors remain available to historical semantic recall while temporal filters keep them out of + the current view. +- Non-finite write and recall timestamps fail validation instead of entering scoring or SQLite. +- Ordinary recall is observational by default, so weak nearest-neighbor results do not gain + stability merely by being returned. Grounded recall still reinforces only cited evidence, and + Python callers with an explicit use signal can request reinforcement. + ## [1.1.5] - 2026-07-28 ### Changed @@ -277,6 +312,9 @@ authentication, licensing and relay behavior, and the redesigned Knowledge Graph work, and suppress expensive dense-graph effects. - The duplicate global Recall shortcut was removed from the dashboard header. Recall remains available in the Memory Operations sidebar and from contextual page actions. +- The README documentation was expanded to clarify note-link graphs, agent memory, code + awareness, encryption, and sleep-time consolidation without making unmeasured product + comparisons. - The README now documents Command Code CLI as an MCP-native client and includes its verified stdio registration command. @@ -613,9 +651,10 @@ and safe hosted deployment. budget (`ENGRAPHIS_CHUNK_TOKENS`, default 256) with a sentence-level overlap (`ENGRAPHIS_CHUNK_OVERLAP`, default 32); a hard per-document cap (`ENGRAPHIS_CHUNK_MAX`, default 200) bounds amplification. numpy/stdlib only, so it runs - under the offline gate and is byte-identical across runs. This lifts recall on long, - multi-topic documents that previously became one diluted memory. New: `ChunkingExtractor` - in `backends/extractor.py`; `tests/test_chunking_extractor.py`. + under the offline gate and is byte-identical across runs. This gives long, multi-topic + documents finer retrieval units instead of one diluted memory; the bundled evaluation below + preserves Recall@5 while reducing retrieved context. New: `ChunkingExtractor` in + `backends/extractor.py`; `tests/test_chunking_extractor.py`. - **File/folder imports chunk too.** With `ENGRAPHIS_EXTRACTOR=chunk`, `import_folder`/`import_files` split each file into several retrieval-sized memories (each still `trusted:false`, stamped with `metadata.chunk={index,of,heading}`) instead of @@ -625,7 +664,7 @@ and safe hosted deployment. - **Chunking eval + `longdoc` dataset.** `eval/chunking_eval.py` + `eval/datasets/longdoc.jsonl` compare whole-file vs chunked ingestion through the real recall pipeline. On the offline embedder: identical recall@5 (1.000) at **~73% fewer - context tokens** (826 → 224) and ~4× smaller tokens-to-evidence — the "quality per token" + context tokens** (809 → 219) and ~4× smaller tokens-to-evidence (162 → 42) — the "quality per token" number `BENCHMARKS.md` calls for. `tests/test_chunking_eval.py`. - **"Dreaming" trigger for automated maintenance.** `automation.should_dream` / `dream_due` run a consolidation sweep *before* the cadence when enough new episodic memories have diff --git a/Dockerfile b/Dockerfile index c6425a83..4c131703 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,9 @@ ENV PYTHONUNBUFFERED=1 \ # ONCE, not on every cold container. A fresh in-container download blocks startup and # can lose the healthcheck race; caching on the volume makes subsequent boots instant. HF_HOME=/data/.cache/huggingface \ - # Customer license / trial / machine-id / lease state. Kept on the /data volume - # (not the container's ephemeral home) so activation and device binding survive. + # Customer-side cloud session and entitlement display cache. Keep it on /data rather + # than the container's ephemeral home so reconnects do not lose rotated credentials. + # License issuance, trial state, leases, and revocations remain private services. ENGRAPHIS_STATE_DIR=/data/.engraphis WORKDIR /app diff --git a/MANIFEST.in b/MANIFEST.in index 8789a2e6..7ceee97b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,10 +8,13 @@ recursive-include engraphis/classic_assets/vendor * recursive-include engraphis/dashboard_assets *.html *.css *.js *.png *.ico recursive-include engraphis/dashboard_assets/vendor * include engraphis/commercial_manifest.json -include LICENSE NOTICE README.md CHANGELOG.md +include LICENSE NOTICE README.md CHANGELOG.md BENCHMARKS.md include pyproject.toml include .env.example requirements.txt include docker-entrypoint.sh Dockerfile docker-compose.yml include railway.json +recursive-include eval *.py +include eval/BASELINES.md +recursive-include eval/configs *.json recursive-include eval/datasets *.jsonl recursive-include tests *.py diff --git a/README.md b/README.md index fa3313e5..26169e46 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ https://discord.com/invite/Wfr2ejBmY > Auto Dreaming, Auto Consolidation, and Team identity/seat management run only on the > official hosted service; their server implementations are not distributed here. +> **Support continued Engraphis development with Pro.** Your subscription helps cover hosted +> infrastructure and ongoing development while unlocking Cloud Sync, Analytics, Auto Consolidation, +> and Auto Dreaming across your installations. [Start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro&trial=pro#billing) +> or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing). + ## Full Engraphis install: pip install "engraphis[all]" Engraphis-Dashboard opens `http://127.0.0.1:8700` in your browser. No cloud, signup, or API key is required for @@ -119,7 +124,7 @@ embeddings. You bring an LLM only for optional chat, synthesis, structured extra or structured consolidation. - **Local-first & private** — runs offline; the core depends only on `numpy`. -- **MCP-native** — 29 tools for Claude Code, Command Code, Cursor, Cline, Zed, Windsurf. +- **MCP-native** — 30 tools for Claude Code, Command Code, Cursor, Cline, Zed, Windsurf. - **Self-maintaining facts** — writes are deterministically conflict-resolved (no LLM required). - **Advisory retention supervision** — an optional LLM can label writes as ephemeral, normal, or critical; outputs are bounded, clamped, audited, and can never silently drop a write. @@ -175,14 +180,84 @@ memories created before provider/model activity metadata was introduced still ap structured-extraction entries. > Privacy boundary: text sent through structured extraction leaves the local process and is -> handled under the selected provider/model's data terms. Keep extraction off for material that -> must remain entirely local, or use the offline `chunk` extractor instead. +> handled under the selected provider/model's data terms. Turning extraction off disables only +> this transfer. For ingestion that must remain entirely local, also keep +> `ENGRAPHIS_RETENTION_SUPERVISOR=none` (the default) and use the offline `chunk` extractor; +> other explicitly invoked LLM-backed operations have their own transfer boundaries. --- -## Evidence-backed capabilities +## Measured quality and token efficiency + +Engraphis ships typed and scoped memory, bi-temporal history, grounded recall, hybrid +vector/lexical/graph retrieval, deterministic context packing, and MCP-native agent tools. +The current deterministic offline regression fixtures reproduce these quality results: + +| Fixture | Reproduced result | +|---|---| +| CodeMem retrieval — 44 memories, 26 questions | **Recall@5 1.000**, hit@5 1.000, answer-token recall 1.000 | +| Grounded-answer decisions — 10 cases | **10/10 correct**: 5/5 answerable questions cited evidence and 5/5 off-topic questions abstained | + +### Proof at a glance + +| **72.9% less retrieved context** | **3.8× smaller evidence record** | **55.38% smaller MCP response** | +|---|---| +| **808.8 → 219.0** tokens per question | **162.2 → 42.4** tokens to supporting evidence | **17,172 → 7,663** serialized tokens | +| Same Recall@5 **1.000** in the long-document fixture | Same 18 fixture questions returned an evidence-holding memory | Same CodeMem retrieval scores across 260 timed recalls | + +Agents spend less of their context window carrying irrelevant history, leaving more room for the +current task and cited evidence. These are controlled, deterministic fixtures—not model-billing, +task-time, or external benchmark claims. + +#### A controlled before-and-after example + +| Retrieval mode | Mean returned memory content | Recall@5 | +|---|---:|---:| +| Whole documents | 808.8 tokens | 1.000 | +| Engraphis structure-aware chunks | 219.0 tokens | 1.000 | + +The chunked mode returns the relevant passage instead of the whole document: **589.8 fewer tokens +per question**. Under the same model-context budget, that leaves roughly **590 tokens** for task +instructions or other relevant evidence. + +### Measurement details and reproducibility + +The table below records every current token/context efficiency measurement and its counting +boundary. + +| What is counted | Comparison | Measured reduction | Quality held constant | +|---|---|---|---| +| Retrieved top-5 memory content, averaged per question | Whole documents: **808.8** tokens → structure-aware chunks: **219.0** tokens | **589.8 fewer tokens per question** (**72.9% lower**, about **3.7× smaller**) | Recall@5 **1.000** in both modes across 6 documents and 18 questions | +| Smallest returned memory that contains the reference evidence | Whole documents: **162.2** tokens → chunks: **42.4** tokens | **119.8 fewer tokens to evidence** (**73.9% lower**, about **3.8× smaller**) | The same 18 questions had a returned evidence-holding memory in both modes | +| Serialized MCP recall response across 260 timed CodeMem recalls | Full result: **17,172** `engraphis.regex.v1` tokens → compact result: **7,663** tokens | **9,509 response tokens avoided** (**55.38% lower**) | Recall@5, hit@5, and answer-token recall all **1.000** | +| Packed prompt-context usage in the same CodeMem performance fixture | Hard budget: **1,500** tokens; observed mean: **87.73**; observed maximum: **106** | A hard cap prevents a recall from exceeding its configured context budget | This is usage accounting, not a before/after savings comparison | + +The compact MCP response avoids duplicating full memory bodies when the packed context and source +list are enough. That can reduce what an agent must inspect or pass onward, but the fixtures do +**not** measure model-provider charges, end-to-end task time, or customer cost savings. -Engraphis publishes capability descriptions and reproducible benchmark protocols without narrative vendor comparisons. See [`BENCHMARKS.md`](BENCHMARKS.md) for the evidence and claim policy. +The measures are deliberately separate and **must not be added together**: chunking counts the +content of retrieved memory records before `ContextPacker`, whereas compact recall counts the +serialized MCP response returned to a client. “Tokens to evidence” is the size of the smallest +retrieved memory record holding the reference evidence; it is not latency or end-to-end answer +accuracy. Chunking creates more focused stored records (24 chunks rather than 6 whole-document +memories in this fixture), so this is a context-efficiency result—not a storage-reduction claim. + +Reproduce the quality and token/context measurements without a network connection or API key: + +```bash +python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 +python -m eval.grounded +python -m eval.chunking_eval +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 5 --json +``` + +These are small deterministic correctness and efficiency fixtures, not official LoCoMo / +LongMemEval QA scores or a third-party leaderboard result. Compact-response counts use the exact +`engraphis.regex.v1` counter; the chunking evaluation uses its documented deterministic +normalized-character estimator. Chunking measures retrieved memory content, while compact recall +measures serialized MCP response size. See [`BENCHMARKS.md`](BENCHMARKS.md) for definitions, +limitations, canonical external-evaluation requirements, and the no-unsupported-claims policy. --- @@ -281,7 +356,8 @@ claude mcp add engraphis -- engraphis-mcp cmd mcp add engraphis -- engraphis-mcp # Command Code CLI ``` -Your agent now has 29 tools — remember, recall (grounded + proactive), proactive context, +Your agent now has 30 tools — remember, recall context (plus full, grounded, and proactive recall), +proactive context, grounded answer alias, why, timeline, forget, pin, correct, promote, ingest, consolidate, index_repo, search/code path/impact/export, privacy receipts, PostgreSQL schema ingestion, link, record_event, start/end_session, stats, and check_update. See the [MCP tools table](#mcp-tools) below. @@ -342,6 +418,23 @@ print(hit["context"]) The same `MemoryService` backs the dashboard and the MCP server. +For an agent prompt, prefer `engraphis_recall_context`: it returns one hard-budget packed +`context` plus compact `sources`, deterministic `usage` accounting (`budget_tokens`, `context_tokens`, +`source_tokens`, `saved_tokens`, `savings_ratio`, `packed_count`, `omitted_count`, and +`token_counter`), and optional diagnostics. Accounting is exact for the named counter; inject the +reader's tokenizer when reader-model token parity is required. `engraphis_recall` remains the compatible full-recall +surface; use `response_mode="compact"` when the packed context is enough and full memory bodies +would duplicate it. Both default to the `balanced` retrieval profile; `auto` remains opt-in. + +For bi-temporal reads, `valid_at` selects what was true at a Unix timestamp and `known_at` selects +what Engraphis had learned then. `as_of` remains a compatibility alias for `valid_at`; supplying +both is allowed only when they match. + +For a mutable claim, pass a stable `subject_key` and optional `claim_kind`, such as +`subject_key="api.rate_limit", claim_kind="configured_value"`. Matching claim identities make +supersession deterministic; when similarity suggests a relationship but not a contradiction, +Engraphis keeps both memories and returns `op="relate"`. + --- ## Govern memories without losing history @@ -350,7 +443,7 @@ Engraphis separates automatic write resolution from explicit human governance: | Operation | Use it when | What happens to history | |---|---|---| -| `remember` | Adding or restating one fact | Deterministically adds, reinforces, or supersedes a same-scope memory | +| `remember` | Adding or restating one fact | Adds, reinforces, safely supersedes, or relates an uncertain neighbor | | `correct` | Replacing one known-wrong memory | Closes the old validity window and links the replacement | | `promote` | A narrow learning now applies more broadly | Writes a wider-scope successor and closes/links the source instead of editing scope in place | | `merge` | Combining two or more overlapping memories | Retires every source and creates one memory that supersedes all of them | @@ -410,10 +503,14 @@ worker implementations live in a private repository and are not part of this pac See [`docs/LICENSING.md`](docs/LICENSING.md) for the source-license, service, grace, and recovery boundaries. +If Engraphis is useful in your work, a Pro subscription is the simplest way to support the +project while adding hosted sync, analytics, and managed memory maintenance. [Subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_pricing#billing) +($10/month or $100/year; annual billing saves two months). + | | Free (available now) | Pro — $10/mo or $100/yr | Team — $20/seat/mo or $200/seat/yr | |---|---|---|---| | Dashboard WebUI (with built-in inspector) | ✓ | ✓ | ✓ | -| Memory engine + 29 MCP tools | ✓ | ✓ | ✓ | +| Memory engine + 30 MCP tools | ✓ | ✓ | ✓ | | Version-chain diffs, offline knowledge graph | ✓ | ✓ | ✓ | | Manual local consolidation (dry-run by default) | ✓ | ✓ | ✓ | | Local workspace export (JSON: memories, sessions, audit) | ✓ | ✓ | ✓ | @@ -439,7 +536,8 @@ recovery boundaries. | Write | `engraphis_ingest` | Apply the configured extractor (`chunk`, `llm`, or `llm_structured`); `none` stores one verbatim memory | | Write | `engraphis_ingest_postgres_schema` | Store a new PostgreSQL schema snapshot + typed graph per call; DSN is never stored | | Write | `engraphis_consolidate` | Pure dry-run or live sleep-time sweep; a live call can write multiple resolved facts and receipts | -| Stateful read | `engraphis_recall` | Hybrid vector + lexical + graph recall; reinforces returned memories and records a receipt | +| Stateful read | `engraphis_recall_context` | Recommended prompt context: hard-budget packed text, compact sources, strict token usage, and optional diagnostics | +| Stateful read | `engraphis_recall` | Hybrid vector + lexical + graph recall; records a receipt without strengthening weak matches | | Stateful read | `engraphis_recall_grounded` | Cited answer or abstention; records a receipt and reinforces cited memories | | Stateful read | `engraphis_answer` | Backward-compatible grounded-answer alias with the same effects | | Pure read | `engraphis_recall_proactive` | "What should I know right now" — no query, reinforcement, or receipt | @@ -619,10 +717,12 @@ background loop, cron wrapper, or worker. Secret-class and session-scoped memories are excluded before a managed snapshot is serialized; secret-class rows are rejected again by the hosted service. The encoded payload is capped at -16 MiB and travels over HTTPS without end-to-end encryption. Managed compute is enabled by -default once an installation is connected to Engraphis Cloud — connecting accepts the terms -that cover it — and stays off for a local-only installation with no cloud session; cloud -entitlement is also required. `ENGRAPHIS_MANAGED_COMPUTE_CONSENT=0` opts a connected +16 MiB. A connected installation sends that bounded, non-secret snapshot to Engraphis Cloud +over HTTPS, where the hosted service must read it to produce a proposal; this is not +end-to-end-encrypted processing. Local-only installations send nothing. Managed compute is +enabled by default once an installation is connected to Engraphis Cloud — connecting accepts +the terms that cover it — and stays off for a local-only installation with no cloud session; +cloud entitlement is also required. `ENGRAPHIS_MANAGED_COMPUTE_CONSENT=0` opts a connected installation back out. A managed proposal never silently rewrites the local database. Manual consolidation can also use schema-validated LLM output through @@ -686,7 +786,7 @@ engraphis/ │ ├── core/ # v2 engine — interfaces, store, recall, scoring, schema, sync │ ├── backends/ # pluggable embedder / vector index / reranker / codegraph / sync transports / encryption │ ├── service.py # validated MemoryService facade -│ ├── mcp_server.py # MCP server — 29 tools +│ ├── mcp_server.py # MCP server — 30 tools │ ├── dashboard_app.py # dashboard WebUI (FastAPI) │ ├── dashboard_assets/ # primary Ledger interface + graph engine │ ├── classic_assets/ # selectable full operator dashboard backup @@ -726,11 +826,23 @@ ruff check . ``` Numbers, not assertions: the offline harness is a **correctness floor** (deterministic embedder). -LoCoMo / LongMemEval adapters run separately with a real embedder — see +LoCoMo / LongMemEval adapters and the pinned LongMemEval-V2 reader profile are available for +approved official evaluation runs — see [`BENCHMARKS.md`](BENCHMARKS.md). --- +## Release evidence + +Each tagged release includes `release-evidence.json` and a reproducible CycloneDX JSON SBOM as +GitHub Release assets. The evidence binds the matching tag and commit to the built wheel and +source distribution hashes, SBOM hash, source-input hashes, and the completed release-gate checks. +It is intentionally limited: it does not attest to publication, hosted services, payments, +deployments, or runtime data; the SBOM describes the build job's Python environment rather than an +operating-system or container image. + +--- + ## License Apache-2.0 — see [LICENSE](LICENSE) and [NOTICE](NOTICE). "Engraphis" is a trademark of the diff --git a/deploy/railway-template.json b/deploy/railway-template.json index a66fa4bf..077721b7 100644 --- a/deploy/railway-template.json +++ b/deploy/railway-template.json @@ -41,6 +41,11 @@ "value": "*", "required": true }, + "ENGRAPHIS_DASHBOARD_URL": { + "value": "https://${{RAILWAY_PUBLIC_DOMAIN}}", + "prompt": "The Railway public domain is used for the dashboard and MCP HTTP origin allow-list. Override this with your HTTPS custom domain after it is active.", + "required": false + }, "ENGRAPHIS_CLOUD_CONTROL_URL": { "value": "https://api.engraphis.com", "required": false diff --git a/docker-compose.yml b/docker-compose.yml index 19674c71..a023d8eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,11 +32,9 @@ services: # safe only because the published host port above is loopback-only. ENGRAPHIS_LOCAL_TRUSTED_PEERS: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fc00::/7 ENGRAPHIS_DB_PATH: /data/engraphis.db - # Persist license/trial/machine-id/lease + the revocation registry on the volume. + # Persist the customer-side cloud session and non-authoritative entitlement display + # cache on the volume. Issuance, trial state, leases, and revocations stay private. ENGRAPHIS_STATE_DIR: /data/.engraphis - # Relay/registry DB (issued keys + revocations) — on the persistent volume so a - # revoked key STAYS revoked across redeploys. - ENGRAPHIS_RELAY_DB: /data/.engraphis/relay.db volumes: - engraphis-data:/data restart: unless-stopped diff --git a/docs/ARCHITECTURE_V3.md b/docs/ARCHITECTURE_V3.md index 5e1f7291..69c523ce 100644 --- a/docs/ARCHITECTURE_V3.md +++ b/docs/ARCHITECTURE_V3.md @@ -5,9 +5,9 @@ retention-supervision, and privacy-receipt additions introduced with schema vers ```mermaid flowchart LR - Agent["Agent / host LLM"] --> Intent["remember · link · recall"] + Agent["Agent / host LLM"] --> Intent["remember · link · recall_context (compact) · recall"] CLI["engraphis-graph CLI"] --> Service["MemoryService"] - MCP["29 MCP tools"] --> Service + MCP["30 MCP tools"] --> Service HTTP["Dashboard + read-only graph HTTP"] --> Service Import["Local resources / PostgreSQL catalog"] --> Extractors["Optional local extractors"] Extractors --> Service diff --git a/docs/HOSTING_RAILWAY.md b/docs/HOSTING_RAILWAY.md index 5cf510d5..1f119036 100644 --- a/docs/HOSTING_RAILWAY.md +++ b/docs/HOSTING_RAILWAY.md @@ -25,6 +25,16 @@ Set `ENGRAPHIS_FORWARDED_ALLOW_IPS=*` only when the container is reachable exclu Railway's trusted proxy. Set the dashboard's public URL where the runtime supports it, terminate TLS at the platform edge, and keep the volume private. +The published template derives `ENGRAPHIS_DASHBOARD_URL` from Railway's generated public domain. +That lets the dashboard's MCP-over-HTTP endpoint accept the public dashboard origin without +loosening its host/origin allow-list. If you attach a custom domain, override it with that domain's +canonical HTTPS URL after Railway has activated the domain; do not use an internal Railway domain +or a URL containing credentials. + +Do not add Resend (or any other email-provider) credentials to this customer node. The public +runtime has no transactional-email sender, verification, invitation, or billing-email service; +those systems remain in the official hosted control plane. + ## Connect to hosted Pro/Team services Complete onboarding through the official Engraphis Cloud dashboard, then configure only the diff --git a/docs/KILO_CODE_INTEGRATION.md b/docs/KILO_CODE_INTEGRATION.md index 76360b9f..4c30655c 100644 --- a/docs/KILO_CODE_INTEGRATION.md +++ b/docs/KILO_CODE_INTEGRATION.md @@ -10,9 +10,9 @@ This manual is written for someone who wants the full technical picture: what En There are two separate questions hiding inside "connect Kilo Code to Engraphis," and they are usually where people talk past each other: -1. **Transport layer — "get the pipes connected."** This is: install the Engraphis MCP server, tell Kilo Code how to launch it, confirm the tools show up. It's a plumbing task. When it's done, Kilo Code can *see* 29 `engraphis_*` tools. Success here is binary — either the tools appear or they don't. +1. **Transport layer — "get the pipes connected."** This is: install the Engraphis MCP server, tell Kilo Code how to launch it, confirm the tools show up. It's a plumbing task. When it's done, Kilo Code can *see* 30 `engraphis_*` tools. Success here is binary — either the tools appear or they don't. -2. **Orchestration layer — "use the memory well."** This is: *when* should the agent remember vs. recall, how should memories be scoped (`workspace → repo → session`), which of the 29 tools answers which question, and how to keep the store clean over time. This is where the actual value is, and it's a discipline, not a config. +2. **Orchestration layer — "use the memory well."** This is: *when* should the agent remember vs. recall, how should memories be scoped (`workspace → repo → session`), which of the 30 tools answers which question, and how to keep the store clean over time. This is where the actual value is, and it's a discipline, not a config. You need both. A perfect config with no discipline gives you an agent that has memory tools and never uses them correctly. Good discipline with a broken config gives you an agent that wants to remember and can't. **Section 3 is the transport layer. Sections 4–6 are the orchestration layer.** Do them in order. @@ -40,7 +40,7 @@ Everything runs on your machine. The whole store is a single SQLite file. Local You interact with Engraphis through three surfaces, all backed by the *same* engine (`MemoryService`), so they can never drift apart: - **The dashboard WebUI** (`engraphis-dashboard`, `http://127.0.0.1:8700`) — a visual product to see, search, and curate memory. -- **The MCP server** (`engraphis-mcp`) — the 29 tools your coding agent calls. **This is the surface Kilo Code uses.** +- **The MCP server** (`engraphis-mcp`) — the 30 tools your coding agent calls. **This is the surface Kilo Code uses.** - **The Python library** (`from engraphis.service import MemoryService`) — for direct programmatic use. ### 2.1 The five ideas that make it more than a vector store @@ -183,7 +183,7 @@ You can also click **Approve Always** on any tool at runtime to write the same r --- -## 4. The 29 tools — the orchestration surface +## 4. The 30 tools — the orchestration surface Once connected, Kilo Code sees these. Do **not** assume only `remember`/`recall` exist — the value is in the rest. This is the full surface, grouped by what question each one answers. @@ -194,8 +194,9 @@ Once connected, Kilo Code sees these. Do **not** assume only `remember`/`recall` | Write | `engraphis_link` | Explicitly connect two related memories (e.g. a bug ↔ its fix). | | Write | `engraphis_ingest` | Store raw/undistilled text; extracts discrete facts first when an LLM extractor is configured. | | Write | `engraphis_ingest_postgres_schema` | Store a new point-in-time PostgreSQL schema + graph per call; the DSN is never stored. | -| **Stateful recall** | `engraphis_recall` | Hybrid vector + lexical + graph recall; reinforces matches and appends a privacy-safe receipt. | -| Stateful recall | `engraphis_recall_grounded` | Cited answer assembled *only* from retrieved memories — or abstains; records a receipt and reinforces cited memories. | +| **Stateful recall** | `engraphis_recall_context` | Recommended prompt packet: hard-budget context, compact source identities, strict token usage, and optional diagnostics. | +| **Stateful recall** | `engraphis_recall` | Hybrid vector + lexical + graph recall, with independent `valid_at`/`known_at`; appends a privacy-safe receipt without strengthening weak matches. | +| Stateful recall | `engraphis_recall_grounded` | Cited answer assembled *only* from retrieved memories — or abstains — with optional point-in-time `as_of`; records a receipt and reinforces cited memories. | | Stateful recall | `engraphis_answer` | Backward-compatible grounded-answer alias with the same state effects; prefer `engraphis_recall_grounded` for new configs. | | **Read** | `engraphis_recall_proactive` | "What should I know right now" — pure queryless ranking + last-session handoff, with no reinforcement or receipt. | | Stateful recall | `engraphis_proactive_context` | Build a task-aware, cited context packet; task/agent-state recall records a receipt without reinforcement. | @@ -230,10 +231,20 @@ This is how to make the connection actually pay off. The discipline fits on a ca ### 5.1 The core loop for a coding task 1. **Starting work in a repo** → `engraphis_recall_proactive` (loads high-signal context with no query) and, for multi-step work, `engraphis_start_session` (its `bootstrap` hands back the last same-user/agent summary and unresolved `open_threads`, so the agent resumes without crossing an identity boundary). -2. **Before answering or acting**, when prior context would help → `engraphis_recall`. Do this *before* asking you something you may have already said. +2. **Before answering or acting**, when prior context would help → `engraphis_recall_context`. It + supplies one hard-budget prompt packet; retain `engraphis_recall` for full-body compatibility. + Do this *before* asking you something you may have already said. 3. **The moment it learns something durable** → `engraphis_remember` (a convention, a decision *with its rationale*, a bug's cause→fix, a preference, a reusable procedure). 4. **Finishing the task** → `engraphis_end_session` with a `summary` and `open_threads` for the next session in that repo. +`engraphis_recall_context` returns `usage` fields for the declared token counter: `budget_tokens`, +`context_tokens`, `source_tokens`, `saved_tokens`, `savings_ratio`, `packed_count`, +`omitted_count`, and `token_counter`. Recall defaults to the `balanced` profile; set `auto` only +explicitly. For time travel, use `valid_at` for what was true and `known_at` for what was known; +`as_of` remains the `valid_at` alias and must match it when both are provided. `engraphis_recall` +remains the full-response compatibility path, with `response_mode=compact` when duplicate bodies +are unnecessary; both recall surfaces accept `diagnostics=true` for a retrieval trace. + ### 5.2 Scope in one minute `workspace → repo → session → memory`. On every write, choose: @@ -278,8 +289,8 @@ engraphis_start_session(workspace="acme", repo="backend", agent="kilo-code", goal="fix flaky auth tests") → bootstrap.open_threads: ["tests 3-5 still failing after token refactor"] -engraphis_recall(query="how do we handle auth token expiry?", - workspace="acme", repo="backend") +engraphis_recall_context(query="how do we handle auth token expiry?", + workspace="acme", repo="backend", token_budget=1024) → "Access tokens expire in 15m; refresh in Redis keyed by session (PASETO, not JWT)." # ...agent finds and fixes the cause... @@ -328,7 +339,7 @@ Kilo Code is an MCP client; Engraphis ships an MCP server (`engraphis-mcp`, loca `kilo.jsonc` (`["cmd","/c","engraphis-mcp"]` on Windows, `["engraphis-mcp"]` on macOS/Linux), pin `ENGRAPHIS_DB_PATH`, bump `timeout` to 15000, and verify with `engraphis_stats`. That gets the pipes connected. The *value* is the orchestration layer -above it — 27 scoped, typed, bi-temporal memory, code, audit, and maintenance tools plus the +above it — 30 scoped, typed, bi-temporal memory, code, audit, and maintenance tools plus the discipline of "recall before you ask, remember before you move on," with `workspace → repo → session` scoping and periodic `engraphis_consolidate` to keep it clean. diff --git a/docs/RAILWAY_TEMPLATE.md b/docs/RAILWAY_TEMPLATE.md index d9a7000f..f9c4ab5d 100644 --- a/docs/RAILWAY_TEMPLATE.md +++ b/docs/RAILWAY_TEMPLATE.md @@ -10,6 +10,8 @@ issuer, relay, managed compute, Auto Dreaming, Auto Consolidation, or Team ident - Service mode: `customer`. - Persistent volume: `/data`. - Health check: `/api/ready`. +- `ENGRAPHIS_DASHBOARD_URL` derived from Railway's generated public domain (override it with the + canonical HTTPS custom domain once one is active so public MCP origin checks remain strict). - Generated local API bearer supplied as `ENGRAPHIS_API_TOKEN`. - No vendor signer, billing, mail, Team-admin, relay-storage, or worker secrets. diff --git a/docs/SYNC.md b/docs/SYNC.md index 4cd2aa75..986a4f8f 100644 --- a/docs/SYNC.md +++ b/docs/SYNC.md @@ -25,6 +25,10 @@ The split is deliberate. Local checks in Apache-licensed code are not DRM and ca a fork. The paid boundary is authorization to use the official private service and its operated infrastructure. +If you want hosted sync across your installations while helping fund continued Engraphis +development, [start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=sync_doc&trial=pro#billing) +or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=sync_doc#billing). + ## Trial and grace The no-card Pro or Team trial begins after email confirmation and lasts **exactly 3 active @@ -100,14 +104,20 @@ when both endpoints remain in the export. Inbound legacy or untrusted bundles ca relabel, or overwrite session-scoped state because the sync format carries no authenticated session owner or lifecycle contract. +Bundle format v2 preserves durable claim identity and the system-time at which a +world-time invalidation was learned. Current Engraphis accepts inbound v1 bundles for +compatibility but exports v2. Older clients reject v2 instead of silently forwarding a +downgraded bundle that loses those fields. + Bundle input is untrusted. The client validates schema and size limits before applying records, rechecks workspace scope, and retains provenance/audit evidence. A relay cannot inject a record outside the authorized workspace merely by changing bundle fields. ## Security and privacy -- Use HTTPS for every hosted endpoint. The public client rejects redirects, embedded URL - credentials, and unsafe remote targets. +- Local-only installations send no memory content to Engraphis. Cloud Sync and managed compute + send the explicitly eligible records or bounded snapshot to Engraphis Cloud over TLS; the + hosted service can read that submitted content and the transport is not end-to-end encrypted. - Treat cloud session and refresh files as credentials; keep their directory owner-only. - `secret` memories are excluded from managed uploads. Managed compute also rejects secret rows server-side. diff --git a/docs/dashboard-button-qa.md b/docs/dashboard-button-qa.md new file mode 100644 index 00000000..5a75c582 --- /dev/null +++ b/docs/dashboard-button-qa.md @@ -0,0 +1,55 @@ +# Dashboard button QA + +Date: 2026-07-29 +Scope: v2 Ledger (`/`) and legacy Classic (`/classic`) dashboards. + +## Test setup + +The manual pass used four parallel browser lanes and an isolated local v2 server on +`127.0.0.1:8701` with deterministic embeddings. The fixture contained the `demo` and +`beta` workspaces plus representative memories, graph data, provenance, timeline, and +consolidation state. The four lanes covered: + +- primary Ledger navigation, memory creation, grounded Ask, and theme controls; +- Library, import/editor actions, and empty-form behavior; +- Graph & Relations, Provenance, Manage, exports, saved views, and switches; +- broad regression including Classic and responsive/mobile keyboard behavior. + +## Button coverage + +The pass exercised the primary navigation, workspace selector, dashboard/theme switcher, +New memory, Save/Close, memory card actions, Import files, grounded answer, provenance +trace, timeline/history, supersessions, all Provenance tabs, all Graph tabs/styles/layouts/ +palettes/saved views/layers/toggles/actions/exports, all Manage tabs, workspace create and +workspace actions, consolidation preview and commit confirmation, plan comparison, and +Classic navigation/mobile-nav controls. + +## Failures found and fixed + +1. **Empty Save memory was silent.** Native form validation prevented the JavaScript + handler from running, leaving the editor open with no explanation. The editor now uses + explicit validation, an alert-region error, `aria-invalid`, focus on the content field, + and a status announcement. +2. **Closing the modern editor lost focus.** Close now returns focus to the button or card + that opened the editor, with a safe New memory fallback. +3. **Empty Ask, Provenance, Timeline/Supersessions, and workspace-create actions were + silent for the same native-validation reason.** These forms now use custom validation + messages and focus the relevant field. Successful submissions clear the prior status + message so an old validation error cannot remain beside a successful result. +4. **Classic mobile Escape closed the menu without reliably returning focus.** Escape now + closes the menu through the shared focus-restoring path. + +## Environment notes + +- One parallel lane could not start against the repository's default database because that + existing database is schema version 5 while this checkout supports schema version 4. + This is an environment/data compatibility issue, not a dashboard button failure. The + isolated schema-4 fixture started and exercised the UI successfully. +- The browser harness did not expose programmatic download events for the PNG/JSON export + anchors, but the dashboard status confirmed both exports completed. No application + console errors were observed during the manual pass. + +## Regression checks + +The focused static regression checks live in +`tests/test_dashboard_button_regressions.py` and cover each repaired failure mode. diff --git a/engraphis/backends/embedder_st.py b/engraphis/backends/embedder_st.py index aec30a02..bbb1f643 100644 --- a/engraphis/backends/embedder_st.py +++ b/engraphis/backends/embedder_st.py @@ -15,9 +15,15 @@ class SentenceTransformerEmbedder: - def __init__(self, model_name: str) -> None: + def __init__(self, model_name: str, *, revision: Optional[str] = None) -> None: from sentence_transformers import SentenceTransformer # lazy: optional dependency - self.model = SentenceTransformer(model_name) + kwargs = {"revision": revision} if revision else {} + # Keep declared model provenance beside the loaded object. Benchmark + # artifacts must be able to distinguish a pinned model from a mutable + # fallback without inspecting implementation-specific internals. + self.model_name = model_name + self.revision = revision + self.model = SentenceTransformer(model_name, **kwargs) self._dim = int(self.model.get_embedding_dimension()) @property @@ -34,12 +40,17 @@ def embed(self, texts: list[str], *, kind: Literal["text", "code"] = "text") -> LAST_EMBEDDER_ERROR = "" -def get_embedder(model_name: Optional[str] = None, dim: int = 256): +def get_embedder( + model_name: Optional[str] = None, + dim: int = 256, + *, + revision: Optional[str] = None, +): """A real model if available, else the deterministic offline embedder.""" global LAST_EMBEDDER_ERROR if model_name: try: - emb = SentenceTransformerEmbedder(model_name) + emb = SentenceTransformerEmbedder(model_name, revision=revision) LAST_EMBEDDER_ERROR = "" return emb except Exception as exc: # noqa: BLE001 - optional dep; record why we fall back diff --git a/engraphis/backends/extractor.py b/engraphis/backends/extractor.py index 9cc99ce3..39845302 100644 --- a/engraphis/backends/extractor.py +++ b/engraphis/backends/extractor.py @@ -1,8 +1,8 @@ """Fact extractors — implementations of the ``core.interfaces.Extractor`` protocol. -Modern memory systems often auto-distill raw text into discrete -facts before storage; Engraphis makes that step *pluggable and optional* so the core -stays offline-capable (AGENTS.md §3.8): +Fact extraction can distill raw text into discrete records before storage. Engraphis +makes that step *pluggable and optional* so the core stays offline-capable +(AGENTS.md §3.8): * ``PassthroughExtractor`` — the default: the caller's text is stored exactly as given (today's behaviour, zero dependencies, zero network). diff --git a/engraphis/backends/graph_extractor.py b/engraphis/backends/graph_extractor.py index ba99a351..ac1c7b74 100644 --- a/engraphis/backends/graph_extractor.py +++ b/engraphis/backends/graph_extractor.py @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from typing import Any, Optional -from engraphis.core.interfaces import Edge, Node +from engraphis.core.interfaces import Edge, Node, SearchFilter # ── Regex NER (ported from engraphis/engines/ingest.py — the v1 heuristic path) ── # Capitalized multi-word sequences, emails, hashtags, and mentions. Keep the @@ -346,7 +346,9 @@ def get_graph_extractor(kind: str = "none"): def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] = None, title: str = "", extractor: Any = None, provenance: Optional[dict] = None, commit: bool = True, - extraction: Any = None) -> dict: + extraction: Any = None, + valid_from: Optional[float] = None, + ingested_at: Optional[float] = None) -> dict: """Extract entities/relations from free text and write them into the knowledge graph, scoped to ``(workspace_id, repo_id)``. @@ -383,7 +385,10 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] memory_ids.append(memory_id) prov["memory_ids"] = memory_ids - existing_edges = store.neighbors(list(name_to_id.values())) + existing_edges = store.neighbors( + list(name_to_id.values()), + flt=SearchFilter(workspace_id=workspace_id, repo_id=repo_id), + ) edge_by_key = {(e.src, e.dst, e.relation): e for e in existing_edges} specific_pairs: set[frozenset[str]] = set() written_relations = 0 @@ -397,10 +402,17 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] specific_pairs.add(frozenset((sid, did))) existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov, commit=commit) + store.add_edge_support( + existing.id, + prov, + valid_from=valid_from, + ingested_at=ingested_at, + commit=commit, + ) continue eid = store.upsert_edge(Edge(id="", src=sid, dst=did, relation=relation, workspace_id=workspace_id, repo_id=repo_id, + valid_from=valid_from, ingested_at=ingested_at, provenance=prov), commit=commit) edge_by_key[key] = Edge(id=eid, src=sid, dst=did, relation=relation) written_relations += 1 @@ -421,12 +433,19 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] key = (lo, hi, "co_occurs") existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov, commit=commit) + store.add_edge_support( + existing.id, + prov, + valid_from=valid_from, + ingested_at=ingested_at, + commit=commit, + ) continue eid = store.upsert_edge(Edge( id="", src=lo, dst=hi, relation="co_occurs", weight=_COOCCUR_WEIGHT, workspace_id=workspace_id, - repo_id=repo_id, provenance=prov, + repo_id=repo_id, valid_from=valid_from, + ingested_at=ingested_at, provenance=prov, ), commit=commit) edge_by_key[key] = Edge(id=eid, src=lo, dst=hi, relation="co_occurs") written_relations += 1 diff --git a/engraphis/backends/sync_folder.py b/engraphis/backends/sync_folder.py index 53ad1152..8bbd9a46 100644 --- a/engraphis/backends/sync_folder.py +++ b/engraphis/backends/sync_folder.py @@ -3,8 +3,8 @@ The zero-infrastructure, self-hostable tier of cloud sync: point two or more devices at the same folder that is *already* replicated between them — a Dropbox / iCloud Drive / OneDrive folder, a Syncthing share, a mounted network drive, or even -a git repo you push/pull — and Engraphis handles the memory-aware merge on top. -This folder transport provides a free local sync path with deterministic bundle merging. +a git repo you push/pull — and Engraphis handles the memory-aware, deterministic +merge on top. It implements the ``SyncTransport`` Protocol (``core/interfaces.py``): opaque named byte blobs, no knowledge of memory semantics. Each device writes exactly one @@ -13,9 +13,9 @@ (temp file + ``os.replace``) so a half-written bundle is never observed — the same mount-safe discipline the rest of the repo uses (AGENTS.md §7). -The managed TLS relay (the headline Pro upsell) is a different ``SyncTransport`` -implementation that plugs in here unchanged. Client-side end-to-end encryption is a -documented follow-up; today's relay stores opaque but plaintext bundle bytes at rest. +The managed TLS relay is a different ``SyncTransport`` implementation that plugs in +here unchanged. Client-side end-to-end encryption is a documented follow-up; today's +relay stores opaque but plaintext bundle bytes at rest. """ from __future__ import annotations diff --git a/engraphis/backends/vector_numpy.py b/engraphis/backends/vector_numpy.py index 7dc8a01b..d28edfd2 100644 --- a/engraphis/backends/vector_numpy.py +++ b/engraphis/backends/vector_numpy.py @@ -30,6 +30,8 @@ def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = def search(self, vec: np.ndarray, k: int, *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]: + if k <= 0: + return [] q = np.asarray(vec, dtype=np.float32) n = float(np.linalg.norm(q)) if n > 0: @@ -41,9 +43,13 @@ def search(self, vec: np.ndarray, k: int, mat = np.vstack([r[1] for r in rows]) # already normalized on write scores = mat @ q # cosine == dot for unit vectors k = min(k, len(ids)) - top = np.argpartition(-scores, k - 1)[:k] - top = top[np.argsort(-scores[top])] - return [(ids[i], float(scores[i])) for i in top] + # ``argpartition`` does not define which equal-scored rows survive at + # the top-k boundary. Hashing embeddings produce ties frequently, so + # use the memory id as an explicit stable secondary key. + top = sorted( + range(len(ids)), key=lambda index: (-float(scores[index]), ids[index]) + )[:k] + return [(ids[index], float(scores[index])) for index in top] def delete(self, ids: list[str]) -> None: marks = ",".join("?" for _ in ids) diff --git a/engraphis/classic_assets/dashboard.css b/engraphis/classic_assets/dashboard.css index 3d0d2c7d..9d9046db 100644 --- a/engraphis/classic_assets/dashboard.css +++ b/engraphis/classic_assets/dashboard.css @@ -271,6 +271,7 @@ body{ .empty{padding:var(--space-6) 0;color:var(--color-text-dim);line-height:1.5;text-align:left} .empty .btn{margin-top:var(--space-3)} .upgrade-panel{max-width:720px;padding:var(--space-6);border:var(--rule-strong) solid var(--color-accent);background:linear-gradient(135deg,var(--color-accent-bg),transparent 68%)}.upgrade-panel-kicker,.upgrade-panel-benefits-title{color:var(--color-accent);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em}.upgrade-panel h2{margin:var(--space-2) 0;font-size:var(--text-xl);letter-spacing:-.02em;color:var(--color-text)}.upgrade-panel-lede{max-width:620px;margin:0;color:var(--color-text-dim)}.upgrade-panel-price{margin-top:var(--space-4);color:var(--color-text);font-size:var(--text-lg);font-weight:600}.upgrade-panel-benefits{margin-top:var(--space-4);padding-top:var(--space-4);border-top:var(--rule) solid var(--color-border)}.upgrade-panel-benefits ul{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--space-2) var(--space-5);margin:var(--space-3) 0 0;padding:0;list-style:none}.upgrade-panel-benefits li{display:flex;gap:var(--space-2);color:var(--color-text-muted);font-size:var(--text-sm);line-height:1.35}.upgrade-panel-benefits li::before{color:var(--color-accent);content:"✓"}.upgrade-panel-trial{margin:var(--space-4) 0 0;color:var(--color-text-dim);font-size:var(--text-sm)}.upgrade-panel-actions{display:flex;gap:var(--space-2);margin-top:var(--space-4);flex-wrap:wrap}.upgrade-panel-actions .btn{margin-top:0}@media(max-width:640px){.upgrade-panel{padding:var(--space-4)}.upgrade-panel-benefits ul{grid-template-columns:1fr}} +.pro-support-copy{margin-top:10px;padding:10px 12px;border-left:2px solid var(--color-accent);background:var(--color-accent-bg);color:var(--color-text-dim);font-size:var(--text-sm);line-height:1.45}.pro-support-copy strong{color:var(--color-text)} .hosted-opportunity{display:grid;max-width:900px;grid-template-columns:minmax(0,1.2fr) minmax(260px,.8fr);gap:var(--space-5);padding:var(--space-6);border:var(--rule-strong) solid var(--color-accent);background:linear-gradient(135deg,var(--color-accent-bg),transparent 65%)}.hosted-opportunity-kicker,.hosted-opportunity-preview-label,.hosted-opportunity-card span{color:var(--color-accent);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em}.hosted-opportunity-kicker span{color:var(--color-text-dim)}.hosted-opportunity h2{max-width:14ch;margin:var(--space-2) 0 var(--space-3);color:var(--color-text);font-family:var(--font-display);font-size:var(--text-xl);letter-spacing:-.02em;line-height:1.05}.hosted-opportunity-lede{max-width:58ch;margin:0;color:var(--color-text-muted);font-size:var(--text-sm);line-height:1.55}.hosted-opportunity-next{margin:var(--space-4) 0 0;color:var(--color-text);font-size:var(--text-sm);font-weight:600;line-height:1.45}.hosted-opportunity-actions{display:flex;gap:var(--space-2);margin-top:var(--space-4);flex-wrap:wrap}.hosted-opportunity-preview{display:flex;min-width:0;flex-direction:column;gap:var(--space-2);padding:var(--space-4);border:var(--rule) solid var(--color-border);background:color-mix(in srgb,var(--color-raised) 72%,transparent)}.hosted-opportunity-preview-label{margin-bottom:var(--space-1);color:var(--color-text-dim)}.hosted-opportunity-card{padding:var(--space-3);border-left:var(--rule-strong) solid var(--color-accent);background:var(--color-panel)}.hosted-opportunity-card p{margin:var(--space-2) 0 0;color:var(--color-text-muted);font-size:var(--text-xs);line-height:1.45}.hosted-opportunity-privacy{grid-column:1/-1;padding-top:var(--space-3);border-top:var(--rule) solid var(--color-border);color:var(--color-text-dim);font-size:var(--text-xs);line-height:1.5}.hosted-opportunity-privacy strong{color:var(--color-text-muted)}@media(max-width:720px){.hosted-opportunity{grid-template-columns:1fr;padding:var(--space-4)}.hosted-opportunity h2{max-width:none}.hosted-opportunity-privacy{grid-column:auto}} /* Route compositions inherit the same ledger geometry. */ diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js index 6f5f6932..1d885f43 100644 --- a/engraphis/classic_assets/dashboard.js +++ b/engraphis/classic_assets/dashboard.js @@ -192,8 +192,9 @@ function fmtDay(epoch){const n=Number(epoch)||0;if(!(n>0))return '';try{const d= prefers ENGRAPHIS_PRO_UPGRADE_URL, so where the portal and the checkout are configured separately it is the Pro checkout under a neutral name. LIC.account_url resolves the generic value directly; the fallback only matters against a build that predates it. */ -function hostedAccountUrl(){return safeUrl((LIC&&LIC.account_url)||(LIC&&LIC.upgrade_url))} -function hostedPlanUrl(plan,trial,interval){const cadence=interval==='annual'?'annual':'monthly',key=plan+'_'+cadence+'_upgrade_url',raw=(LIC&&(LIC[key]||(plan==='team'?LIC.team_upgrade_url:LIC.pro_upgrade_url)))||(LIC&&LIC.upgrade_url);const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);if(plan==='pro'||plan==='team')url.searchParams.set('plan',plan);url.searchParams.set('interval',cadence);if(!url.hash)url.hash='billing';if(trial)url.searchParams.set('trial',plan);return url.href}catch(e){return safe}} +function withCtaAttribution(raw,content,medium){const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);url.searchParams.set('utm_source','engraphis');url.searchParams.set('utm_medium',medium||'product');url.searchParams.set('utm_campaign','pro_conversion');url.searchParams.set('utm_content',content||'plans');return url.href}catch(e){return safe}} +function hostedAccountUrl(content){return withCtaAttribution((LIC&&LIC.account_url)||(LIC&&LIC.upgrade_url),content||'account','product')} +function hostedPlanUrl(plan,trial,interval,content){const cadence=interval==='annual'?'annual':'monthly',key=plan+'_'+cadence+'_upgrade_url',raw=(LIC&&(LIC[key]||(plan==='team'?LIC.team_upgrade_url:LIC.pro_upgrade_url)))||(LIC&&LIC.upgrade_url);const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);if(plan==='pro'||plan==='team')url.searchParams.set('plan',plan);url.searchParams.set('interval',cadence);if(!url.hash)url.hash='billing';if(trial)url.searchParams.set('trial',plan);return withCtaAttribution(url.href,content||plan,'product')}catch(e){return safe}} /* Why is this feature locked? One sentence per access state, so the panel never claims a trial the customer cannot start nor blames billing for a trial that simply ran out. This is DENIAL copy: every caller reaches it because a hosted request was refused. The @@ -216,13 +217,15 @@ function teamTeaserNote(){const ends=licTrialEnds(); if(licPlanKey()!=='team'||!licAccessLive())return lockReason(true); if(licAccessState()==='trial')return `Your free trial includes Team${ends?` until ${esc(ends)}`:''}. Organizations, roles, and seats are managed in Engraphis Cloud.`; return 'Your TEAM subscription includes this. Organizations, roles, and seats are managed in Engraphis Cloud.'} -function unlockHtml(feature,plan){const url=hostedPlanUrl(plan,false,'monthly'),annualUrl=hostedPlanUrl(plan,false,'annual'),trialUrl=hostedPlanUrl(plan,true,'monthly'),team=plan==='team';const offerTrial=licTrialAvailable();const trial=team?'Start hosted Team trial':'Start hosted Pro trial';const purchase=team?'Purchase Team license':'Purchase Pro license';const price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year';const detail=lockReason(team);const benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'];return `
ENGRAPHIS ${team?'TEAM':'PRO'}

Unlock ${esc(feature)} and more

Make the local memory engine work across your installations—and keep improving without manual upkeep.

${price}
Your license unlocks

${detail}

${offerTrial?`${trial}`:''}${purchase}Annual option
`} +function hostedCta(plan,content,interval){const team=plan==='team',name=team?'Team':'Pro',state=licAccessState(),current=licPlanKey();if(state==='lapsed')return {label:'Update billing',href:hostedAccountUrl(content||'account'),kind:'account'};if(licAccessLive()&&(current===plan||(current==='team'&&plan==='pro')))return {label:current==='team'&&team?'Open Team Cloud':'Open Engraphis Cloud',href:hostedAccountUrl(content||'account'),kind:'account'};const trial=licTrialAvailable()&&state==='inactive';return {label:trial?`Start ${TRIAL_DAYS}-day ${name} trial`:`Subscribe to ${name}`,href:hostedPlanUrl(plan,trial,interval||'monthly',content||plan),kind:trial?'trial':'subscribe'} } +function ctaLinkHtml(cta,className,content){return `${esc(cta.label)}`} +function unlockHtml(feature,plan){const team=plan==='team',name=team?'Team':'Pro',featureKey=`feature_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,primary=hostedCta(plan,featureKey),annual=primary.kind==='account'?'':{label:`Annual ${name} option`,href:hostedPlanUrl(plan,false,'annual',`${featureKey}_annual`),kind:'subscribe'},price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year',detail=lockReason(team),benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'],lede=team?'Team adds shared workspaces, named seats, roles, and remote agent access.':'Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development while unlocking Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming across your installations.';return `
ENGRAPHIS ${name.toUpperCase()}

Unlock ${esc(feature)} and more

${lede}

${price}
Your license unlocks

${detail}

${ctaLinkHtml(primary,'btn btn-primary',name.toLowerCase())}${annual.href&&annual.href!=='#'?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}
`} function startTrialPlan(plan){const url=hostedPlanUrl(plan,true);if(url==='#'){toast('Hosted signup URL is not configured','err');return}const link=document.createElement('a');link.href=url;link.target='_blank';link.rel='noopener';link.click()} function startTrial(){return startTrialPlan('pro')} function startTeamTrial(){return startTrialPlan('team')} /* The badge follows the access state, not the plan name. A plan name alone told a trialist they were a subscriber, and told a lapsed or expired customer nothing was wrong. */ -function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const st=licAccessState(),plan=licPlanName();bd.textContent=st==='trial'?'TRIAL':st==='trial_expired'?'TRIAL ENDED':st==='lapsed'?plan+' INACTIVE':st==='active'?plan:'LOCAL';bd.className='pill '+(licAccessLive()?'pill-accent':'pill-muted')} +function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const st=licAccessState(),plan=licPlanName(),trial=licTrialAvailable(),label=st==='trial'?'TRIAL':st==='trial_expired'?'GET PRO':st==='lapsed'?'BILLING':st==='active'?plan:trial?'TRY PRO':'GET PRO',aria=st==='active'?'Open Engraphis Cloud account':st==='lapsed'?'Update billing in hosted plan settings':trial?'Start the 3-day Pro trial in hosted plan settings':'Subscribe to Pro in hosted plan settings';bd.textContent=label;bd.className='pill '+(licAccessLive()?'pill-accent':'pill-muted');bd.setAttribute('aria-label',aria);bd.title=aria} function updateFeatureLocks(){ const has=f=>LIC&&(LIC.features||[]).includes(f); const apply=(id,feature,label,plan)=>{ @@ -245,8 +248,13 @@ function renderAnalytics(a,isPortfolio){const t=a.totals||{},f=a.decay_forecast| /* A consent-required response is a valuable moment to show the job Pro can do, not a dead end about configuration. A customer with live access must never be offered their own plan again: hosted features are on by default once their account is available. */ -function managedConsentHtml(feature){const automation=/automation/i.test(feature),live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const accountUrl=hostedAccountUrl(),trialUrl=hostedPlanUrl('pro',true),purchaseUrl=hostedPlanUrl('pro'),actions=live?`Open Engraphis Cloud`:`${trial?`Start ${TRIAL_DAYS}-day Pro trial`:''}Purchase Pro license`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Purchase Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} +function managedConsentHtml(feature){const automation=/automation/i.test(feature),featureKey=`managed_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const primary=hostedCta('pro',featureKey),annual=primary.kind==='account'?'':{label:'Annual Pro option',href:hostedPlanUrl('pro',false,'annual',`${featureKey}_annual`),kind:'subscribe'},actions=`${ctaLinkHtml(primary,'btn btn-primary',featureKey)}${annual.href?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':licAccessState()==='lapsed'?'Your subscription needs billing attention. Update billing to restore hosted insights and maintenance.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Subscribe to Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} function managedConsentRequired(error){return error&&error.status===409&&error.detail&&error.detail.code==='consent_required'} +const CLOUD_PRIVACY_COPY='Engraphis Cloud must read the bounded snapshot you submit to produce results. It travels over HTTPS but is not end-to-end encrypted; secret and session-scoped memories stay local.'; +const EXTERNAL_LLM_PRIVACY_COPY='Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; +async function confirmCloudTransfer(title,summary,submit){return confirmAction(title,summary+'\n\nPrivacy: '+CLOUD_PRIVACY_COPY,submit||'Continue')} +const managedConsentHtmlBase=managedConsentHtml; +managedConsentHtml=function(feature){return managedConsentHtmlBase(feature).replace('',`
Privacy, by design. ${esc(CLOUD_PRIVACY_COPY)}
`)}; /* Only an unconfigured local installation may turn a 401 into trial signup. A revoked or expired Cloud session is also a 401, but ``trial.available`` is false there and it must remain a reconnect error instead of offering a trial the control plane rejects. */ @@ -276,9 +284,12 @@ async function saveAutomation(){const body={enabled:document.getElementById('au- async function runMaintenance(){const el=document.getElementById('au-result');if(el)el.innerHTML='';try{const d=await api('/maintenance/run?workspace='+encodeURIComponent(WS||''),{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({dry_run:true})});if(el)el.innerHTML=`PROPOSAL Hosted work was submitted for review.
${esc(JSON.stringify(d,null,2))}
`;toast('Managed proposal requested','ok')}catch(e){if(el)el.innerHTML=managedConsentRequired(e)?managedConsentHtml('Hosted Automation'):'
'+esc(e.message)+'
';toast(managedConsentRequired(e)?'Hosted Automation starts automatically with Pro':((e.status===402||e.status===501)?'Hosted Automation requires Pro or Team':e.message),'err')}} const runMaintenanceBase=runMaintenance; +const saveAutomationBase=saveAutomation; +saveAutomation=async function(){const enabled=document.getElementById('au-enabled');if(enabled&&enabled.checked&&!await confirmCloudTransfer('Save hosted policy','Saving this enabled policy uploads this workspace’s normal and sensitive memory content to Engraphis Cloud; secret and session-scoped rows stay local.','Save policy'))return;return saveAutomationBase()} let MAINTENANCE_PENDING=false; runMaintenance=async function(dry){ if(MAINTENANCE_PENDING)return; + if(!await confirmCloudTransfer('Request hosted proposal','This sends this workspace’s normal and sensitive memory content to Engraphis Cloud for a reviewable proposal; secret and session-scoped rows stay local.','Request proposal'))return; const buttons=Array.from(document.querySelectorAll('#automation-body button[data-onclick="h90"]')),labels=buttons.map(button=>button.textContent); MAINTENANCE_PENDING=true; buttons.forEach(button=>{button.disabled=true}); @@ -505,17 +516,7 @@ function licStateBanner(state,plan,ends,status){ if(state==='lapsed'){const note=LIC_STATUS_NOTE[status];return `
Your ${esc(plan||'hosted')} subscription is no longer active${note?esc(note.charAt(0).toUpperCase()+note.slice(1))+', so hosted':'Hosted'} features are locked until billing is up to date. Your local memories are unaffected. Open the account portal to restore access.
`} if(state==='inactive')return `
No hosted plan on this installationThe local memory engine is free and complete on its own. Cloud Sync, Analytics, Automation, and Team administration run in Engraphis Cloud.
`; return ''} -function licActionsHtml(state){ - if(licTrialAvailable())return `
`; - /* A lapsed customer is fixing an existing subscription, not shopping. Both actions go - to the plan-neutral account portal so a payment-method problem is never reframed as a - new Pro or Team purchase. */ - if(state==='lapsed')return `
Update billingOpen account portal
`; - const buy=state==='trial_expired'; - if(state==='active'){const label=licPlanKey()==='team'?'Open Team Cloud':'Open Pro Cloud';return `
${label}
`} - const primary=buy?'Subscribe to Pro':'Open Pro Cloud'; - const secondary=buy?'Subscribe to Team':'Open Team Cloud'; - return `
${primary}${secondary}
`} +function licActionsHtml(state){const pro=hostedCta('pro','license');if(state==='active'||state==='lapsed')return `
${ctaLinkHtml(pro,'btn btn-primary btn-sm','license')}
`;const team=hostedCta('team','license_team');return `
${ctaLinkHtml(pro,'btn btn-primary btn-sm','license')}${ctaLinkHtml(team,'btn btn-ghost btn-sm','license_team')}
`} function renderLicense(d){ const el=document.getElementById('lic-body');if(!el)return; const state=licAccessState(),raw=String(d.plan||'local').toLowerCase(); @@ -535,6 +536,8 @@ function renderLicense(d){ it. Emitted by /api/license since the plan resolver landed, and never shown until now — so "the dashboard says PRO" and "the cloud says PRO" could not be told apart. */ if(d.plan_source)h+=`
Plan source${esc(LIC_SOURCE_LABEL[d.plan_source]||d.plan_source)}${d.plan_checked_at?' · confirmed '+esc(fmtRel(d.plan_checked_at)):''}
`; + if(state==='active')h+=`
Thank you for supporting Engraphis. Your subscription helps fund hosted infrastructure and ongoing development.
`; + else if(state!=='lapsed')h+=`
Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development while unlocking Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming.
`; h+=`
The local core remains free. Pro and Team capabilities execute in Engraphis Cloud. The email-confirmed, no-card trial lasts exactly ${TRIAL_DAYS} active days; private-service account grace is separate, capped at 24 hours, and never extends cloud access or restricts local MCP and dashboard use.
`; h+=licActionsHtml(state); el.innerHTML=h; @@ -542,20 +545,22 @@ function renderLicense(d){ async function exportWorkspace(){try{const d=await api('/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-export-'+Date.now()+'.json';a.click();URL.revokeObjectURL(a.href);toast('Exported','ok')}catch(e){toast(e.message,'err')}} /* Hosted Team is a service CTA; local identity and seat administration are not shipped. */ -async function loadTeam(){const el=document.getElementById('team-body');let url=licPlanKey()==='team'&&licAccessLive()?hostedAccountUrl():hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}const trialUrl=hostedPlanUrl('team',true);el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
${licTrialAvailable()?`Start hosted Team trial`:''}Open Team Cloud
`} +async function loadTeam(){const el=document.getElementById('team-body'),teamCta=hostedCta('team','team_tab');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
${ctaLinkHtml(teamCta,'btn btn-primary btn-sm','team_tab')}
`} /* health + settings */ function connectionContext(){const host=(location.hostname||'').toLowerCase();return host==='localhost'||host==='127.0.0.1'||host==='::1'||host.endsWith('.localhost')?'Local engine':'Remote customer node'} async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}} function loadSettings(){loadLicense();loadSyncStatus();loadHostedAgentAccess();loadLlmStatus();const s=document.getElementById('cfg-store');if(s)s.textContent=location.host} -async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF keeps everything on this machine.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} +async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} function onLlmProvChange(){const p=document.getElementById('llm-prov').value;const sel=document.getElementById('llm-model');const defs={openai:'gpt-4o-mini',anthropic:'claude-3-5-sonnet-20241022',google:'gemini-1.5-flash',openrouter:'openai/gpt-4o-mini'};if(sel&&defs[p]){sel.value=defs[p]}updateLlmSnippet()} function updateLlmSnippet(){const p=(document.getElementById('llm-prov')||{}).value||'openai';const m=(document.getElementById('llm-model')||{}).value||'';const ta=document.getElementById('llm-snippet');if(!ta)return;ta.value='ENGRAPHIS_LLM_PROVIDER='+p+'\nENGRAPHIS_LLM_MODEL='+m+'\nENGRAPHIS_LLM_API_KEY=\nENGRAPHIS_EXTRACTOR=llm_structured\n'} function copyLlmSnippet(){const ta=document.getElementById('llm-snippet');if(!ta)return;ta.select();try{navigator.clipboard.writeText(ta.value);toast('Copied .env snippet','ok')}catch(e){toast('Copy failed — select and Ctrl+C','err')}} -async function setLlmExtractor(on){try{const d=await api('/llm/extractor',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:!!on})});const ok=!!d.extractor_enabled;toast(ok?'LLM extraction turned on — new memories will be sent to your provider':'LLM extraction turned off — memories stay on this machine'+(d.persisted===false?' (could not save for restart)':''),ok?'ok':'muted');loadLlmStatus()}catch(e){toast(e.message,'err')}} +async function setLlmExtractor(on){try{const d=await api('/llm/extractor',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:!!on})});const ok=!!d.extractor_enabled;toast(ok?'LLM extraction turned on — new memories will be sent to your provider':'LLM extraction turned off — extractor transfers are disabled'+(d.persisted===false?' (could not save for restart)':''),ok?'ok':'muted');loadLlmStatus()}catch(e){toast(e.message,'err')}} async function testLlm(){const r=document.getElementById('llm-test-result');if(r){r.textContent='Testing…';setTone(r,'muted')}try{const d=await api('/llm/test',{method:'POST'});if(r){if(d.ok){const transient=d.auto_enabled&&d.persisted===false;r.textContent=(transient?'⚠ ':'✓ ')+'Connected — '+esc(d.provider)+'/'+esc(d.model)+(transient?' Extraction is active for this process, but the setting could not be saved for restart. Set ENGRAPHIS_EXTRACTOR=llm_structured and ENGRAPHIS_LLM_AUTO_EXTRACT=1 in the deployment environment.':'');setTone(r,transient?'red':'green')}else{r.textContent='✗ '+(d.error||'failed');setTone(r,'red')}}}catch(e){if(r){r.textContent='✗ '+esc(e.message);setTone(r,'red')}}} -async function loadHostedAgentAccess(){const el=document.getElementById('tokens-body');if(!el)return;let url=licPlanKey()==='team'&&licAccessLive()?hostedAccountUrl():hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Per-member agent accounts, roles, named seats, and rotating device credentials are managed in Team Cloud, not by this local dashboard.
Open Team CloudAgent Connect guide
`} +async function loadHostedAgentAccess(){const el=document.getElementById('tokens-body');if(!el)return;const teamCta=hostedCta('team','agent_access');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Per-member agent accounts, roles, named seats, and rotating device credentials are managed in Team Cloud, not by this local dashboard.
${ctaLinkHtml(teamCta,'btn btn-primary btn-sm','agent_access')}Agent Connect guide
`} +const setLlmExtractorBase=setLlmExtractor; +setLlmExtractor=async function(on){if(on&&!await confirmAction('Turn on LLM extraction',EXTERNAL_LLM_PRIVACY_COPY,'Turn on'))return;return setLlmExtractorBase(on)}; /* Route by cause, exactly like loadAnalytics/loadAutomation. Rendering the purchase panel for every failure told a paying customer to buy the plan they already own whenever the network blipped or the cloud answered 5xx. Only 401/402/501 are billing answers. */ @@ -565,6 +570,9 @@ function syncRecoveryHtml(){return unlockHtml('Cloud Sync','pro')+`
Hosted relayCONNECTED
Relay storage and authorization run in Engraphis Cloud. This package contains only the customer client; it does not run a local relay or background scheduler.
${esc(status)}
`} async function syncNow(){const b=document.getElementById('sync-btn')||document.getElementById('sync-retry-btn');const original=b&&b.textContent;const s=document.getElementById('sync-status');if(b){b.disabled=true;b.textContent='Syncing…'}if(s)s.textContent='Contacting the cloud…';try{const d=await api('/sync/run',{method:'POST',headers:{'Content-Type':'application/json'},body:'{}'});const su=d.summary||{};toast('Synced — pushed '+(su.exported||0)+', '+(su.added||0)+' new from other devices','ok');await loadSyncStatus()}catch(e){if(e.status===401||e.status===402||e.status===403){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncRecoveryHtml();toast(e.status===402?'Cloud Sync requires an active Pro or Team entitlement — open Engraphis Cloud to upgrade or renew.':'Cloud Sync authorization is no longer active — reconnect in Engraphis Cloud.','err');return}toast('Sync failed: '+e.message,'err');if(b){b.disabled=false;b.textContent=original||'Sync now'}if(s)s.textContent='Sync failed — try again.'}} +const syncNowBase=syncNow; +syncNow=async function(){if(!await confirmCloudTransfer('Sync shared workspaces','Cloud Sync sends eligible changes from your shared workspaces to Engraphis Cloud and receives authorized changes from your other installations; secret and session-scoped rows stay local.','Sync now'))return;return syncNowBase()} + /* ─── knowledge graph (force-graph + d3-force: compact defaults and selectable layouts) ─── */ let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null; const GRAPH_PRESETS={ @@ -576,6 +584,16 @@ const GRAPH_PRESETS={ custom:{label:'Custom tuning',curve:.1,particles:0} }; window.GSET=window.GSET||{mode:'communities',font:12,size:3,repel:48,link:16,gravity:48,labels:false,linkw:.72,labelDensity:24,flow:true,frozen:false}; +/* Keep legacy Classic geometry in the same compact world-space range as Ledger. The old + `size * sqrt(1 + degree)` rule let a highly connected entity become a giant disc, then + zoom-to-fit magnified that disc again. Degree still adds a restrained emphasis, but it is + normalized and bounded so material/theme painters cannot change node geometry. */ +function graphNodeRadius(node,base,metric){ + const size=Number.isFinite(+base)&&+base>0?+base:3; + const normalized=Math.max(0,Math.min(1,Number(metric)||0)); + const radius=size*.45*(.55+Math.min(1.6,normalized*1.9)); + return Math.max(.8,Math.min(size*1.1,radius)); +} const ETYPE_TOKEN={person_or_concept:'--entity-concept',mention:'--entity-mention',hashtag:'--entity-hashtag',email:'--entity-email',organization:'--entity-organization',location:'--entity-location'}; const GRAPH_PALETTES={ theme:null, @@ -651,15 +669,14 @@ function graphUpdateHud(data){ if(count&&data)count.textContent=data.nodes.length.toLocaleString()+' entities · '+data.links.length.toLocaleString()+' relations'; if(badge)badge.textContent=GPERF.large?'Large graph mode':'Adaptive rendering'; } -/* ── opt-in next-generation renderer (`?graph-engine=next`) ────────────────────────────── - The classic renderer stays the default and the rollback path. Everything below is written - so that any failure in the opt-in engine degrades to classic rather than taking the graph - view down: one throw sets GRAPH_ENGINE_FAILED and the flag is never honoured again for the - life of the page. */ +/* ── canonical graph renderer ──────────────────────────────────────────────────────────── + Classic now uses the same renderer as Ledger for its normal graph view. The legacy canvas + remains the rollback path: one engine failure latches GRAPH_ENGINE_FAILED and the graph + degrades to the legacy renderer instead of taking the view down. */ let GRAPH_ENGINE_FAILED=false; function graphEngineEnabled(){ if(GRAPH_ENGINE_FAILED)return false; - try{return new URLSearchParams(window.location.search).get('graph-engine')==='next'}catch(e){return false} + try{return new URLSearchParams(window.location.search).get('graph-engine')==='next'||/(^|\/)classic\/?$/.test(window.location.pathname)}catch(e){return false} } function graphEngineFallback(error){ GRAPH_ENGINE_FAILED=true; @@ -762,9 +779,9 @@ async function loadLegacyGraph(){ GRESIZEFRAME=requestAnimationFrame(()=>{GRESIZEFRAME=0;const element=document.getElementById('graph-net');if(GRAPH_ENGINE)GRAPH_ENGINE.resize();else if(FG&&element)FG.width(element.clientWidth).height(element.clientHeight)}); }); } - const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),fullGraph=GRAPH_FULL?'&full=true&limit=20000':''; + const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),showUnlinked=GRAPH_FULL||!!document.getElementById('graph-show-iso').checked,graphLimit=GRAPH_FULL?20000:320,graphScope=GRAPH_FULL?'&full=true':(showUnlinked?'':'&connected_only=true'); try{ - GRAPH=await api('/graph?workspace='+encodeURIComponent(WS||'')+layerFilter+'&include_code='+(includeCode?'true':'false')+fullGraph+(repo?'&repo='+encodeURIComponent(repo):'')); + GRAPH=await api('/graph?workspace='+encodeURIComponent(WS||'')+layerFilter+'&include_code='+(includeCode?'true':'false')+'&limit='+graphLimit+graphScope+(repo?'&repo='+encodeURIComponent(repo):'')); renderGraphSide();graphRender(); }catch(error){ showAs(empty,true,'flex');empty.textContent='Graph failed: '+error.message;graphSetLayoutStatus('Load failed',false); @@ -795,7 +812,8 @@ function graphData(){ let sourceNodes=GRAPH.nodes;if(hideIso)sourceNodes=sourceNodes.filter(node=>node.degree>0); const names=new Set(sourceNodes.map(node=>node.id)); const nodes=sourceNodes.map(node=>({id:node.id,label:node.label||node.id,displayLabel:(node.label||node.id).length>30?(node.label||node.id).slice(0,29)+'…':(node.label||node.id),etype:node.etype,degree:node.degree||0,val:1+(node.degree||0)})); - nodes.sort((a,b)=>b.degree-a.degree).forEach((node,index)=>{node.rank=index;node.hub=index<24;node.radius=Math.max(1.6,window.GSET.size*Math.sqrt(node.val)*.45);node.color=graphTypeColor(node.etype);node.stroke=graphContrastColor(node.color)}); + const maxDegree=Math.max(1,...nodes.map(node=>node.degree||0)); + nodes.sort((a,b)=>b.degree-a.degree).forEach((node,index)=>{node.rank=index;node.hub=index<24;node.radius=graphNodeRadius(node,window.GSET.size,(node.degree||0)/maxDegree);node.color=graphTypeColor(node.etype);node.stroke=graphContrastColor(node.color)}); const links=GRAPH.edges.filter(edge=>names.has(edge.from)&&names.has(edge.to)).map(edge=>({source:edge.from,target:edge.to,label:edge.label,layer:edge.layer||'semantic'})); const data={nodes,links};GDATA_CACHE={graph:GRAPH,hideIso,data};return data; } @@ -957,8 +975,14 @@ function graphMaterialSprite(p,tier){ var spriteCtx=canvas.getContext('2d');if(!spriteCtx)return null;if(typeof spriteCtx.scale==='function'){spriteCtx.scale(dpr,dpr);graphPaintMaterialDirect(spriteCtx,half,half,radius,p,tier)}else graphPaintMaterialDirect(spriteCtx,half*dpr,half*dpr,radius*dpr,p,tier); var value={canvas:canvas,half:half,radius:radius};GRAPH_MATERIAL_CACHE.set(key,value);if(GRAPH_MATERIAL_CACHE.size>GRAPH_MATERIAL_CACHE_LIMIT)GRAPH_MATERIAL_CACHE.delete(GRAPH_MATERIAL_CACHE.keys().next().value);return value; } -function graphPaintMaterialSurface(ctx,x,y,r,scale,profile,large){ - var tier=graphMaterialTier(r*Math.max(.01,scale),large),sprite=graphMaterialSprite(profile,tier); +function graphPaintMaterialSurface(ctx,x,y,r,scale,profile,large,paintDirect){ + var screenRadius=r*Math.max(.01,scale),tier=graphMaterialTier(screenRadius,large); + /* The full material sprite is intentionally bounded to 40 CSS pixels. On a focused or + high-rank node, enlarging that raster sprite is what produces the blocky/pixelated blob + users see in Classic. Paint only that exceptional node directly at its final size; keep + ordinary nodes on the cache so the large graph remains responsive. */ + if(paintDirect&&tier==='full'&&screenRadius>GRAPH_MATERIAL_RADIUS.full){graphPaintMaterialDirect(ctx,x,y,r,profile,tier);return tier} + var sprite=graphMaterialSprite(profile,tier); if(sprite&&typeof ctx.drawImage==='function'){var half=r*sprite.half/sprite.radius;ctx.drawImage(sprite.canvas,x-half,y-half,half*2,half*2)}else graphPaintMaterialDirect(ctx,x,y,r,profile,tier); return tier; } @@ -971,20 +995,25 @@ function graphStyleBackground(ctx,scale){ ctx.strokeStyle='rgba(255,190,120,.10)';ctx.lineWidth=1/scale;var RR=[72,132,200,286,384];for(var k=0;k1,neighbor=focus&&GHOVERSET.has(node.id),dim=focus&&!neighbor; - var r=node.radius,col=node.color,profile; + var r=node.radius,col=node.color,profile,directMaterial=node.id===GHILITE||node.rank===0; ctx.globalAlpha=dim?.12:1; if(GSTYLE==='galaxy'){ - profile=graphMaterialProfile('galaxy',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large); + profile=graphMaterialProfile('galaxy',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large,directMaterial); }else if(GSTYLE==='solar'){ - var sun=node.rank===0;if(sun)r*=1.7; - profile=graphMaterialProfile('solar',sun?graphMix(col,'#d38b43',.46):col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large); + var sun=node.rank===0; + profile=graphMaterialProfile('solar',sun?graphMix(col,'#d38b43',.46):col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large,directMaterial); }else if(GSTYLE==='cyber'){ - profile=graphMaterialProfile('cyber',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large); + profile=graphMaterialProfile('cyber',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large,directMaterial); }else{ - profile=graphMaterialProfile('classic',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large); + profile=graphMaterialProfile('classic',col);graphPaintMaterialSurface(ctx,node.x,node.y,r,scale,profile,GPERF.large,directMaterial); } if(node.id===GHILITE){ graphMaterialFill(ctx,node.x,node.y,r*.76,graphAlpha('#ffffff',.065)); @@ -1096,7 +1125,8 @@ function graphSetHighlight(id){ } function graphRefreshNodeMetrics(){ const nodes=FG&&FG.graphData?FG.graphData().nodes||[]:[]; - nodes.forEach(node=>{node.radius=Math.max(1.6,window.GSET.size*Math.sqrt(node.val)*.45)}); + const maxDegree=Math.max(1,...nodes.map(node=>node.degree||0)); + nodes.forEach(node=>{node.radius=graphNodeRadius(node,window.GSET.size,(node.degree||0)/maxDegree)}); } function graphRedraw(){ if(!FG||GREDRAWFRAME)return; @@ -1235,8 +1265,8 @@ function graphRender(fit=true,reheat=true){ FG.linkWidth(link=>{const width=window.GSET.linkw||1,focus=GHOVERSET&&GHOVERSET.size>1,bridge=link.label==='influences';if(!focus)return (bridge?.45:(GPERF.dense?.62:.82))*width;const source=(link.source&&link.source.id)||link.source,target=(link.target&&link.target.id)||link.target;return (source===GHILITE||target===GHILITE)?(bridge?1.0:1.8)*width:.25*width}); if(FG.linkLineDash)FG.linkLineDash(GPERF.dense?null:(link=>link.layer==='temporal'?[4,3]:(link.layer==='causal'?[2,2]:null))); if(FG.linkCurvature)FG.linkCurvature(GPERF.dense?0:mode.curve); - FG.linkDirectionalArrowLength(GPERF.dense?0:2.5).linkDirectionalArrowRelPos(1); - if(FG.linkDirectionalParticles){FG.linkDirectionalParticles((reduced||data.links.length>800||window.GSET.flow===false)?0:(GSTYLE==='cyber'?2:(mode.particles||2))).linkDirectionalParticleWidth(1.7).linkDirectionalParticleSpeed(.004)} + FG.linkDirectionalArrowLength(GPERF.dense?0:.625).linkDirectionalArrowRelPos(1); + if(FG.linkDirectionalParticles){FG.linkDirectionalParticles((reduced||data.links.length>800||window.GSET.flow===false)?0:(GSTYLE==='cyber'?2:(mode.particles||2))).linkDirectionalParticleWidth(.85).linkDirectionalParticleCanvasObject(graphPaintFlowArrow).linkDirectionalParticleSpeed(.004)} if(settings.labels){ FG.linkCanvasObjectMode(()=>'after').linkCanvasObject((link,ctx,scale)=>{ if(scale<2.4||!link.label||!link.source.x||(GPERF.dense&&!GHILITE))return; @@ -1545,7 +1575,7 @@ document.addEventListener('keydown',event=>{ if(memories&&memories.classList.contains('show')){closeEntityMems();return} const theme=document.getElementById('theme-menu'); if(theme&&theme.classList.contains('is-open')){closeThemeMenu();document.getElementById('theme-btn').focus();return} - if(document.querySelector('.app').classList.contains('mobile-nav-open')){closeMobileNav();document.getElementById('mobile-nav-toggle').focus();return} + if(document.querySelector('.app').classList.contains('mobile-nav-open')){closeMobileNav(true);return} if(document.getElementById('view-mem-editor').classList.contains('active'))closeMem(); }); diff --git a/engraphis/cloud_features.py b/engraphis/cloud_features.py index c2239f30..7d90e06a 100644 --- a/engraphis/cloud_features.py +++ b/engraphis/cloud_features.py @@ -351,7 +351,8 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, status=413) rows = service.store.conn.execute( "SELECT id, title, content, mtype, scope, ingested_at, last_access, valid_from, " - "valid_to, expired_at, stability, importance, pinned, sensitivity, metadata " + "valid_to, valid_to_recorded_at, expired_at, subject_key, claim_kind, " + "stability, importance, pinned, sensitivity, metadata " "FROM memories WHERE workspace_id=? AND COALESCE(scope, 'workspace')!='session' " "ORDER BY ingested_at, id", (workspace_id,), @@ -397,7 +398,10 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, "last_access": float(item.get("last_access") or item.get("ingested_at") or 0), "valid_from": float(item.get("valid_from") or 0), "valid_to": item.get("valid_to"), + "valid_to_recorded_at": item.get("valid_to_recorded_at"), "expired_at": item.get("expired_at"), + "subject_key": str(item.get("subject_key") or ""), + "claim_kind": str(item.get("claim_kind") or ""), "stability": float(item.get("stability") or 1), "importance": float(item.get("importance") or 0.5), "pinned": bool(item.get("pinned")), diff --git a/engraphis/cloud_session.py b/engraphis/cloud_session.py index ca555011..62fec3e4 100644 --- a/engraphis/cloud_session.py +++ b/engraphis/cloud_session.py @@ -440,6 +440,12 @@ def _declared_entitlement(response: object) -> dict: return {} plan = plan.strip().lower()[:_MAX_PLAN_CHARS] active = response.get("cloud_access_active") + # Compatibility is for an *omitted* field from a control plane that predates this + # disclosure. An explicitly malformed field is not an older-server response: treating + # ``"false"`` (or ``0``) as absent would take the optimistic compatibility path and + # render a paid entitlement live. Keep the last good persisted answer instead. + if "cloud_access_active" in response and not isinstance(active, bool): + return {} named = response.get("status") named = named.strip().lower() if isinstance(named, str) else "" declared = { @@ -916,5 +922,25 @@ def access_for_workspace( if key not in declared: updated.pop(key, None) updated.update(declared) - _save(updated) + try: + _save(updated) + except (OSError, RuntimeError) as exc: + # The control plane has already consumed ``refresh``. Leaving that stale + # value usable after a local write fault makes the next request replay it, + # which can revoke the credential family. Retire it in memory first (so this + # process cannot replay it even when the state mount remains broken), then + # make a best-effort persisted retirement for a fault that was transient. + # The original write is deliberately never retried: it contains a replacement + # credential that may have reached disk only partially on an exotic mount. + _UNUSABLE_REFRESHES.add(_refresh_identity(refresh)) + try: + _mark_refresh_unusable(saved, refresh) + except Exception: # noqa: BLE001 - the state store is already failing + pass + raise CloudSessionError( + "Engraphis Cloud refreshed this session but the rotated credential " + "could not be saved. Connect this installation again.", + status=409, + refresh_unusable=True, + ) from exc return access, organization_id, compute diff --git a/engraphis/core/consolidate.py b/engraphis/core/consolidate.py index 6d9f7e1c..796cf57e 100644 --- a/engraphis/core/consolidate.py +++ b/engraphis/core/consolidate.py @@ -1,7 +1,7 @@ """Sleep-time consolidation (episodic→semantic distillation). -Some systems ship "sleep-time compute" as a cloud service; the local-first equivalent is a -background job the *user* schedules (cron / Windows Task Scheduler / a session hook): +The local-first implementation is a background job the *user* schedules +(cron / Windows Task Scheduler / a session hook): python -m scripts.consolidate --db engraphis.db --workspace acme @@ -127,7 +127,9 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, episodic = store.list_memories( _replace(flt, mtypes=[MemoryType.EPISODIC]), limit=DISTILL_SCAN_LIMIT) - clusters = _cluster_by_subject(episodic, threshold=subject_jaccard) + clusters = _cluster_by_subject( + episodic, threshold=subject_jaccard, store=store, flt=flt, + ) report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, "clusters_found": 0, "digests_created": [], "archived": [], @@ -223,12 +225,9 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, store.close_validity( m.id, actor="consolidation", reason=f"retention {r:.4f} below {archive_below} (consolidation sweep)") - try: - engine.index.delete([m.id]) - except Exception as exc: - logger.warning( - "index delete failed for memory %s (%s)", m.id, type(exc).__name__ - ) + # Preserve the vector as historical evidence. Temporal filtering keeps the + # archived row out of current recall while allowing an explicit ``as_of`` + # query to reproduce the semantic result from when it was live. # ── compaction summary: the payoff of the sweep, as a number ───────────── report["compaction"] = { @@ -250,10 +249,74 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, # ── internals ───────────────────────────────────────────────────────────────── def _cluster_by_subject( - memories: list[MemoryRecord], *, threshold: float + memories: list[MemoryRecord], *, threshold: float, store=None, + flt: Optional[SearchFilter] = None, ) -> list[list[MemoryRecord]]: - """Greedy single-link clustering on token Jaccard — deterministic, order-stable - (memories arrive newest-first from the store; clusters keep that order).""" + """Cluster claim/entity evidence before falling back to token similarity. + + Explicit claim identity is the strongest signal. Persisted memory↔entity + incidence is next, and only records lacking either key take the older + deterministic Jaccard path. Each memory appears in at most one cluster. + """ + keyed: dict[tuple[str, str], list[MemoryRecord]] = {} + assigned: set[str] = set() + for memory in memories: + subject = (memory.subject_key or "").strip() + if subject: + keyed.setdefault((subject, (memory.claim_kind or "").strip()), []).append(memory) + assigned.add(memory.id) + + entity_groups: list[list[MemoryRecord]] = [] + if store is not None: + by_id = {memory.id: memory for memory in memories if memory.id not in assigned} + parent = {memory_id: memory_id for memory_id in by_id} + first_for_entity: dict[str, str] = {} + linked: set[str] = set() + + def find(memory_id: str) -> str: + while parent[memory_id] != memory_id: + parent[memory_id] = parent[parent[memory_id]] + memory_id = parent[memory_id] + return memory_id + + def union(left: str, right: str) -> None: + left_root, right_root = find(left), find(right) + if left_root == right_root: + return + # Stable root selection makes component construction independent of + # incidence-row order and therefore canonical-export friendly. + if left_root > right_root: + left_root, right_root = right_root, left_root + parent[right_root] = left_root + + # Only the bounded consolidation scan can participate in these clusters. + # Restrict the database query too, rather than materializing all workspace + # incidence rows and discarding the unrelated majority in Python. + for link in store.list_memory_entities(flt, memory_ids=list(by_id)): + memory = by_id.get(link.get("memory_id")) + entity_id = str(link.get("entity_id") or "") + if memory is None or not entity_id: + continue + linked.add(memory.id) + existing = first_for_entity.setdefault(entity_id, memory.id) + union(existing, memory.id) + + components: dict[str, list[MemoryRecord]] = {} + for memory in memories: + if memory.id in linked: + components.setdefault(find(memory.id), []).append(memory) + assigned.add(memory.id) + entity_groups = list(components.values()) + + remainder = [memory for memory in memories if memory.id not in assigned] + similarity = _cluster_by_similarity(remainder, threshold=threshold) + return [*keyed.values(), *entity_groups, *similarity] + + +def _cluster_by_similarity( + memories: list[MemoryRecord], *, threshold: float, +) -> list[list[MemoryRecord]]: + """Greedy deterministic fallback for memories without durable identity.""" token_sets = [tokenize(f"{m.title} {m.content}") for m in memories] n = len(memories) parent = list(range(n)) @@ -647,14 +710,7 @@ def _write_structured_digests(engine, cluster: list[MemoryRecord], facts: list[d continue engine.store.close_validity( memory.id, at=now, actor="consolidation", reason=reason) - try: - engine.index.delete([memory.id]) - except Exception as exc: - logger.warning( - "index delete failed for memory %s (%s)", - memory.id, - type(exc).__name__, - ) + # Preserve the source vector for historical/as_of retrieval. return ids diff --git a/engraphis/core/context.py b/engraphis/core/context.py new file mode 100644 index 00000000..04c20722 --- /dev/null +++ b/engraphis/core/context.py @@ -0,0 +1,473 @@ +"""Deterministic, token-budgeted context packing. + +The default packer deliberately has no model or tokenizer dependency. It uses a +small, named regex tokenizer so its accounting is exact for the counter it +declares, reproducible offline, and replaceable by benchmark/provider-specific +token counters at the composition boundary. +""" +from __future__ import annotations + +import math +import re +from collections.abc import Callable +from typing import Optional + +from engraphis.core.interfaces import ( + Candidate, + ContextUsage, + PackedChunk, +) + + +_TOKEN_RE = re.compile(r"\w+|[^\w\s]", re.UNICODE) +_SENTENCE_RE = re.compile(r"(?<=[.!?])(?:[\"')\]]*)\s+|\n+") +_WORD_RE = re.compile(r"\w+", re.UNICODE) +_BRIDGE_TERMS = frozenset({ + "call", "calls", "called", "caller", "dependency", "depends", "flow", + "graph", "impact", "path", "related", "relationship", "why", +}) +_QUALIFIER_TERMS = frozenset({ + "cannot", "except", "if", "must", "never", "no", "not", "only", + "unless", "until", "when", "without", +}) + + +class RegexTokenCounter: + """Exact counter for Engraphis' dependency-free tokenization contract.""" + + identity = "engraphis.regex.v1" + + def __call__(self, text: str) -> int: + return len(_TOKEN_RE.findall(text or "")) + + +class DeterministicContextPacker: + """Pack diverse, relevant evidence into a strict token budget. + + Selection is stable for identical inputs. A supersession/consolidation + family contributes at most one member, summaries are preferred when they + retain query evidence, and oversized sources are reduced at sentence + boundaries before a final token-boundary fallback. + """ + + def __init__( + self, + token_counter: Optional[Callable[[str], int]] = None, + *, + token_counter_identity: Optional[str] = None, + ) -> None: + self._count = token_counter or RegexTokenCounter() + self.token_counter_identity = ( + token_counter_identity + or getattr(self._count, "identity", None) + or getattr(self._count, "__name__", None) + or type(self._count).__name__ + ) + + def pack( + self, + query: str, + candidates: list[Candidate], + token_budget: int, + ) -> tuple[str, list[PackedChunk], ContextUsage]: + budget = max(0, int(token_budget)) + source_tokens = sum(self._source_tokens(candidate) for candidate in candidates) + if budget == 0 or not candidates: + return "", [], self._usage( + budget, 0, source_tokens, 0, len(candidates) + ) + + representatives, duplicate_count = _family_representatives(candidates) + query_terms = _terms(query) + needs_bridge = bool(query_terms & _BRIDGE_TERMS) or bool( + re.search(r"(?:\w+[./\\])+\w+|::|->|\b[A-Za-z_]\w*\(\)", query) + ) + ordered = self._selection_order( + representatives, query_terms=query_terms, needs_bridge=needs_bridge + ) + + context = "" + packed: list[PackedChunk] = [] + covered: set[str] = set() + remaining = list(ordered) + + while remaining: + # Re-evaluate novelty after every selection. This gives compact, + # complementary evidence preference over repeated keyword matches. + remaining.sort( + key=lambda candidate: self._utility( + candidate, + query_terms=query_terms, + covered=covered, + needs_bridge=needs_bridge, + ), + reverse=True, + ) + candidate = remaining.pop(0) + record = candidate.record + if record is None: + continue + + prefix = "\n\n" if context else "" + header = self._header(candidate, len(packed) + 1) + base = f"{context}{prefix}{header}\n" + if self._count(base) >= budget: + continue + + available = budget - self._count(base) + excerpt, truncated, reason = self._excerpt( + query, candidate, available + ) + if not excerpt: + continue + proposed = f"{base}{excerpt}" + if self._count(proposed) > budget: + # A custom tokenizer need not be additive. Fit against the + # complete proposed context so the public hard-budget contract + # still holds. + excerpt = self._fit_text( + excerpt, + max_tokens=available, + prefix=base, + total_budget=budget, + ) + truncated = True + reason = "token_boundary_excerpt" + if not excerpt: + continue + proposed = f"{base}{excerpt}" + + context = proposed + packed.append(PackedChunk( + id=candidate.id, + excerpt=excerpt, + tokens=self._count(excerpt), + truncated=truncated, + reason=reason, + )) + covered.update(_terms(excerpt) & query_terms) + + context_tokens = self._count(context) + omitted = len(candidates) - len(packed) + # ``duplicate_count`` is intentionally folded into omitted_count; keep + # the local name to make the family-diversity policy explicit. + omitted = max(omitted, duplicate_count) + return context, packed, self._usage( + budget, context_tokens, source_tokens, len(packed), omitted + ) + + def count_tokens(self, text: str) -> int: + """Count answer text with the exact counter declared by this packer.""" + return int(self._count(text or "")) + + def _selection_order( + self, + candidates: list[Candidate], + *, + query_terms: set[str], + needs_bridge: bool, + ) -> list[Candidate]: + return sorted( + candidates, + key=lambda candidate: self._utility( + candidate, + query_terms=query_terms, + covered=set(), + needs_bridge=needs_bridge, + ), + reverse=True, + ) + + def _utility( + self, + candidate: Candidate, + *, + query_terms: set[str], + covered: set[str], + needs_bridge: bool, + ) -> tuple[float, float, str]: + record = candidate.record + if record is None: + return (-math.inf, -math.inf, candidate.id) + text = f"{record.title} {record.summary or record.content}" + terms = _terms(text) + overlap = terms & query_terms + novelty = len(overlap - covered) / max(1, len(query_terms)) + relevance = max(0.0, float(candidate.score)) + bridge = 0.2 if needs_bridge and candidate.arm in {"graph", "code"} else 0.0 + compactness = 1.0 / math.sqrt(max(1, self._count(text))) + utility = (0.7 * relevance) + (0.25 * novelty) + bridge + (0.05 * compactness) + # Negate the lexical id tie-break while sorting reverse by using a + # stable ordinal derived from the original id separately below. + return (utility, relevance, _reverse_text(candidate.id)) + + def _excerpt( + self, + query: str, + candidate: Candidate, + max_tokens: int, + ) -> tuple[str, bool, str]: + record = candidate.record + if record is None or max_tokens <= 0: + return "", False, "" + full = (record.content or "").strip() + summary = (record.summary or "").strip() + query_terms = _terms(query) + + if summary and self._summary_is_useful(summary, full, query_terms): + if self._count(summary) <= max_tokens: + return summary, summary != full, "summary" + + if full and self._count(full) <= max_tokens: + return full, False, ( + "bridge_evidence" if candidate.arm in {"graph", "code"} else "full" + ) + + excerpt = self._sentence_excerpt(full or summary, query_terms, max_tokens) + if excerpt: + return excerpt, True, ( + "bridge_excerpt" + if candidate.arm in {"graph", "code"} + else "relevant_sentence_excerpt" + ) + fitted = self._fit_text(full or summary, max_tokens=max_tokens) + return fitted, bool(fitted), "token_boundary_excerpt" + + def _summary_is_useful( + self, + summary: str, + full: str, + query_terms: set[str], + ) -> bool: + if not full: + return True + full_overlap = _terms(full) & query_terms + summary_terms = _terms(summary) + preserves_query = not full_overlap or bool(summary_terms & full_overlap) + qualifiers = _terms(full) & _QUALIFIER_TERMS + preserves_qualifiers = qualifiers.issubset(summary_terms) + return preserves_query and preserves_qualifiers + + def _sentence_excerpt( + self, + text: str, + query_terms: set[str], + max_tokens: int, + ) -> str: + sentences = [part.strip() for part in _SENTENCE_RE.split(text) if part.strip()] + if not sentences: + return "" + ranked = sorted( + enumerate(sentences), + key=lambda item: ( + -len(_terms(item[1]) & query_terms), + -len(_terms(item[1]) & _QUALIFIER_TERMS), + item[0], + ), + ) + chosen: list[tuple[int, str]] = [] + qualifier_sentences = [ + item for item in ranked if _terms(item[1]) & _QUALIFIER_TERMS + ] + # A relevant positive sentence without a separate ``unless``/``except``/ + # ``not`` clause can reverse the source's meaning. Admit qualifying + # sentences first; only then spend remaining budget on other evidence. + def admit(items: list[tuple[int, str]]) -> None: + nonlocal chosen + for index, sentence in items: + proposed = " ".join( + value for _, value in sorted(chosen + [(index, sentence)]) + ) + marker = " […]" if len(chosen) + 1 < len(sentences) else "" + if self._count(proposed + marker) <= max_tokens: + chosen.append((index, sentence)) + + admit(qualifier_sentences) + if len(chosen) == len(qualifier_sentences): + admit([item for item in ranked if item not in qualifier_sentences]) + if not chosen: + preferred = qualifier_sentences[0] if qualifier_sentences else ranked[0] + return self._fit_text(preferred[1], max_tokens=max_tokens) + excerpt = " ".join(value for _, value in sorted(chosen)) + if len(chosen) < len(sentences): + marked = f"{excerpt} […]" + if self._count(marked) <= max_tokens: + excerpt = marked + return excerpt + + def _fit_text( + self, + text: str, + *, + max_tokens: int, + prefix: str = "", + total_budget: Optional[int] = None, + ) -> str: + if max_tokens <= 0: + return "" + required_qualifiers = _terms(text) & _QUALIFIER_TERMS + + def semantically_safe(excerpt: str) -> bool: + return required_qualifiers.issubset(_terms(excerpt)) + + tokens = list(_TOKEN_RE.finditer(text)) + if not tokens: + return "" + limit = min(len(tokens), max_tokens) + while limit > 0: + end = tokens[limit - 1].end() + excerpt = text[:end].rstrip() + if limit < len(tokens) and max_tokens > 1: + marked = f"{excerpt} […]" + if self._count(marked) <= max_tokens: + excerpt = marked + within_local = self._count(excerpt) <= max_tokens + within_total = ( + total_budget is None + or self._count(f"{prefix}{excerpt}") <= total_budget + ) + if within_local and within_total and semantically_safe(excerpt): + return excerpt + limit -= 1 + # A custom token counter may split a single regex token (for example a + # character counter or provider tokenizer). In that case there is no + # shorter regex boundary to try, even though a character prefix fits. + # Find the longest safe prefix against the declared counter so tight + # budgets are still used without violating the hard ceiling. + low, high = 1, len(text) + best = "" + while low <= high: + middle = (low + high) // 2 + excerpt = text[:middle].rstrip() + if not excerpt: + low = middle + 1 + continue + marked = f"{excerpt} […]" if middle < len(text) else excerpt + candidate = marked if self._count(marked) <= max_tokens else excerpt + fits = ( + self._count(candidate) <= max_tokens + and ( + total_budget is None + or self._count(f"{prefix}{candidate}") <= total_budget + ) + ) + if fits: + if semantically_safe(candidate): + best = candidate + low = middle + 1 + else: + high = middle - 1 + return best + + def _header(self, candidate: Candidate, ordinal: int) -> str: + record = candidate.record + if record is None: + return f"[{ordinal}]" + # The compact source list carries identity/scope. Repeating ULIDs and + # scope labels inside the context spends reader tokens without adding + # evidence; the ordinal is the citation bridge. + header = f"[{ordinal}]" + if record.title: + title = " ".join(record.title.split())[:120] + header += f" {title}" + return header + + def _source_tokens(self, candidate: Candidate) -> int: + record = candidate.record + if record is None: + return 0 + return self._count(f"{record.title}\n{record.content}") + + def _usage( + self, + budget: int, + context_tokens: int, + source_tokens: int, + packed_count: int, + omitted_count: int, + ) -> ContextUsage: + saved = max(0, source_tokens - context_tokens) + ratio = (saved / source_tokens) if source_tokens else 0.0 + return ContextUsage( + budget_tokens=budget, + context_tokens=context_tokens, + source_tokens=source_tokens, + saved_tokens=saved, + savings_ratio=ratio, + packed_count=packed_count, + omitted_count=max(0, omitted_count), + token_counter=self.token_counter_identity, + ) + + +def _terms(text: str) -> set[str]: + return {match.group(0).casefold() for match in _WORD_RE.finditer(text or "")} + + +def _family_representatives( + candidates: list[Candidate], +) -> tuple[list[Candidate], int]: + """Keep the highest-ranked member of each supersession/consolidation family.""" + parents: dict[str, str] = {} + + def find(value: str) -> str: + parents.setdefault(value, value) + while parents[value] != value: + parents[value] = parents[parents[value]] + value = parents[value] + return value + + def union(left: str, right: str) -> None: + left_root, right_root = find(left), find(right) + if left_root != right_root: + parents[max(left_root, right_root)] = min(left_root, right_root) + + by_claim: dict[str, str] = {} + for candidate in candidates: + find(candidate.id) + record = candidate.record + metadata = record.metadata if record and isinstance(record.metadata, dict) else {} + direct_subject = str(getattr(record, "subject_key", "") or "").strip() + direct_kind = str(getattr(record, "claim_kind", "") or "").strip() + if direct_subject: + claim_identity = f"{direct_subject}\0{direct_kind}" + prior = by_claim.setdefault( + f"subject_key:{claim_identity}", candidate.id + ) + union(candidate.id, prior) + for field in ("subject_key", "claim_key", "consolidation_family"): + value = str(metadata.get(field) or "").strip() + if value: + if field == "subject_key": + # Legacy rows may carry their claim identity solely in metadata. + # Preserve independently relevant kinds for the same subject. + claim_kind = str(metadata.get("claim_kind") or direct_kind).strip() + value = f"{value}\0{claim_kind}" + prior = by_claim.setdefault(f"{field}:{value}", candidate.id) + union(candidate.id, prior) + related = metadata.get("supersedes") or metadata.get("source_ids") or [] + if isinstance(related, str): + related = [related] + if isinstance(related, list): + for item in related: + if isinstance(item, str) and item: + union(candidate.id, item) + + selected: dict[str, Candidate] = {} + for candidate in candidates: + root = find(candidate.id) + current = selected.get(root) + if current is None or (candidate.score, candidate.id) > ( + current.score, + current.id, + ): + selected[root] = candidate + representatives = sorted( + selected.values(), key=lambda candidate: (-candidate.score, candidate.id) + ) + return representatives, len(candidates) - len(representatives) + + +def _reverse_text(value: str) -> str: + # Stable reverse-sort helper without relying on process-randomized hashes. + return "".join(chr(0x10FFFF - ord(char)) for char in value) diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index a3b20def..cf7fb818 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -37,8 +37,8 @@ SearchFilter, ) from engraphis.core.recall import RecallEngine, RecallResult -from engraphis.core.resolve import RELATED_SIM_FLOOR, ResolutionOp, resolve -from engraphis.core.store import Store, now_ts +from engraphis.core.resolve import RELATED_SIM_FLOOR, Resolution, ResolutionOp, resolve +from engraphis.core.store import Store, memory_matches_filter, now_ts from engraphis.core.textutil import estimate_tokens, jaccard, tokenize logger = logging.getLogger("engraphis.core.engine") @@ -310,6 +310,7 @@ def __init__(self, store: Store, embedder, vector_index, reranker=None, @classmethod def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, + embed_revision: Optional[str] = None, embed_dim: int = 384, vector_backend: str = "auto", rerank_model: Optional[str] = None, extractor: str = "none", graph_extractor: str = "none", @@ -319,7 +320,7 @@ def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, from engraphis.backends.graph_extractor import get_graph_extractor as _get_ge from engraphis.backends.retention import get_retention_supervisor store = Store(db_path, connect=connect) - embedder = get_embedder(embed_model, embed_dim) + embedder = get_embedder(embed_model, embed_dim, revision=embed_revision) index = get_vector_index(store, dim=embedder.dim, prefer=vector_backend) reranker = get_reranker(rerank_model) ext = get_extractor(extractor) @@ -337,7 +338,7 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = scope: Optional[Scope] = None, title: str = "", importance: float = 0.0, keywords: Optional[list] = None, metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, - candidate_k: int = 5, + candidate_k: int = 5, subject_key: str = "", claim_kind: str = "", _trusted_graph_keys: Optional[frozenset] = None) -> str: """Store one memory. Returns the id of the *live* record: a new id for ADD/ INVALIDATE, or the existing memory's id if this was resolved as a NOOP @@ -347,7 +348,8 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = content, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, metadata=metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, - candidate_k=candidate_k, _trusted_graph_keys=_trusted_graph_keys, + candidate_k=candidate_k, subject_key=subject_key, claim_kind=claim_kind, + _trusted_graph_keys=_trusted_graph_keys, )["id"] def remember_with_resolution(self, content: str, *, workspace_id: str, @@ -356,6 +358,7 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, title: str = "", importance: float = 0.0, keywords: Optional[list] = None, metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, candidate_k: int = 5, + subject_key: str = "", claim_kind: str = "", _trusted_graph_keys: Optional[frozenset] = None) -> dict: """Store one memory with deterministic conflict resolution. @@ -367,7 +370,20 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, * ``"invalidate"`` — same subject as an existing memory but new content; the old one's validity was closed (never deleted) and this was inserted. ``superseded`` lists the closed id(s). + * ``"relate"`` — evidence shows a nearby claim but not a safe contradiction; + both remain live and a semantic relation is persisted. """ + if valid_from is not None: + if isinstance(valid_from, bool): + raise ValueError("valid_from must be a finite timestamp") + try: + valid_from = float(valid_from) + except (TypeError, ValueError) as exc: + raise ValueError("valid_from must be a finite timestamp") from exc + if not math.isfinite(valid_from): + raise ValueError("valid_from must be a finite timestamp") + subject_key = str(subject_key or "").strip() + claim_kind = str(claim_kind or "").strip() scope_was_omitted = scope is None scope = ( Scope.REPO if (repo_id or session_id) else Scope.WORKSPACE @@ -413,7 +429,8 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, session_id=session_id, mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, metadata=metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, - candidate_k=candidate_k, trusted_graph_keys=_trusted_graph_keys, + candidate_k=candidate_k, subject_key=subject_key, + claim_kind=claim_kind, trusted_graph_keys=_trusted_graph_keys, ) except BaseException: if (owns_session_transaction @@ -427,6 +444,7 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, title: str, importance: float, keywords: Optional[list], metadata: Optional[dict], valid_from: Optional[float], resolve_conflicts: bool, candidate_k: int, + subject_key: str, claim_kind: str, trusted_graph_keys: Optional[frozenset] = None) -> dict: """The resolve→insert body of ``remember_with_resolution``. The caller holds ``self._write_lock`` for the whole call (atomicity of the resolve decision). @@ -439,8 +457,55 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, decision, neighbors = self._resolve_against_neighbors( text, vec, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, scope=scope, mtype=mtype, - candidate_k=candidate_k, + candidate_k=candidate_k, subject_key=subject_key, + claim_kind=claim_kind, valid_at=valid_from, content=content, + ) + if resolve_conflicts and subject_key and valid_from is not None: + # A durable claim has a temporal identity in addition to its text. A + # scheduled successor can be a better prose match than the version visible + # at this write's effective time, but it is not the version being replaced. + # Select that visible predecessor directly so a backfill is spliced into the + # recorded chain rather than rejected for predating a future match. + claim_history = self.store.list_claim_history( + workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id if scope == Scope.SESSION else None, + scope=scope, mtype=mtype, subject_key=subject_key, + claim_kind=claim_kind, ) + predecessors = [ + record for record in claim_history + if record.valid_from is not None and record.valid_from <= valid_from + and (record.valid_to is None or valid_from < record.valid_to) + ] + if predecessors: + predecessor = max( + predecessors, + key=lambda record: (record.valid_from or float("-inf"), record.id), + ) + if " ".join(content.split()).casefold() == ( + " ".join(predecessor.content.split()).casefold() + ): + decision = Resolution( + ResolutionOp.NOOP, target_id=predecessor.id, + reason=f"exact duplicate of keyed claim {predecessor.id}", + ) + else: + decision = Resolution( + ResolutionOp.INVALIDATE, target_id=predecessor.id, + reason=(f"supersedes temporal predecessor {predecessor.id} " + f"for keyed claim"), + ) + if (decision is not None + and decision.op == ResolutionOp.INVALIDATE + and valid_from is not None): + previous = self.store.get_memory(decision.target_id) + if (previous is not None + and previous.valid_from is not None + and valid_from < previous.valid_from): + raise ValueError( + "valid_from cannot predate the memory it supersedes; " + "record the historical interval separately or correct the older memory" + ) if decision is not None and decision.op == ResolutionOp.NOOP: self.store.reinforce(decision.target_id, boost=scoring.INTERACTION_BOOST["create"]) @@ -469,7 +534,7 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, rec = MemoryRecord( id="", content=content, mtype=mtype, scope=scope, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, title=title, importance=importance, - stability=stability, + stability=stability, subject_key=subject_key, claim_kind=claim_kind, keywords=keywords or [], metadata=meta, valid_from=valid_from, # Lift provenance into its dedicated field/column so recall/why/timeline # surface it (copied, not popped: consolidate.py still reads @@ -518,7 +583,8 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, _graph_feed(self.store, content, workspace_id=workspace_id, repo_id=repo_id, title=title, extractor=StructuredMetadataGraphExtractor(meta), - provenance={"source": "structured_extractor", "memory_id": mid}) + provenance={"source": "structured_extractor", "memory_id": mid}, + valid_from=rec.valid_from, ingested_at=rec.ingested_at) except Exception: pass if scope != Scope.SESSION and self.graph_extractor is not None: @@ -526,18 +592,46 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, from engraphis.backends.graph_extractor import feed as _graph_feed _graph_feed(self.store, content, workspace_id=workspace_id, repo_id=repo_id, title=title, extractor=self.graph_extractor, - provenance={"source": "graph_extractor", "memory_id": mid}) + provenance={"source": "graph_extractor", "memory_id": mid}, + valid_from=rec.valid_from, ingested_at=rec.ingested_at) except Exception: pass + if scope != Scope.SESSION: + self._link_memory_entities( + mid, f"{title}\n{content}", workspace_id=workspace_id, repo_id=repo_id, + valid_from=rec.valid_from, + ) if decision is not None and decision.op == ResolutionOp.INVALIDATE: - self.store.close_validity(decision.target_id, reason=decision.reason) - try: - self.index.delete([decision.target_id]) - except Exception as exc: # noqa: BLE001 — merely stale in the index; recall - # re-checks validity on read, so log (don't audit) and continue. - logger.warning("vector-index delete failed for %s (%s)", - decision.target_id, type(exc).__name__) + # World time closes when the replacement becomes true, not when this process + # happened to ingest it. This keeps backdated and scheduled facts queryable at + # the correct ``as_of`` anchor. + predecessor = self.store.get_memory(decision.target_id) + predecessor_end = predecessor.valid_to if predecessor is not None else None + if (predecessor_end is not None and rec.valid_from is not None + and rec.valid_from < predecessor_end): + # The target was already retired by its recorded successor. Splicing an + # intermediate version must shorten that historical interval, not leave + # the old end in place (``close_validity`` intentionally only closes live + # rows). The new row inherits the old boundary below. + self.store.conn.execute( + "UPDATE memories SET valid_to=?, valid_to_recorded_at=? WHERE id=?", + (rec.valid_from, now_ts(), decision.target_id), + ) + self.store.audit("system", "invalidate", decision.target_id, + decision.reason) + self.store.close_validity( + mid, at=predecessor_end, + reason="bounded by the recorded successor interval", + ) + else: + self.store.close_validity( + decision.target_id, at=rec.valid_from, reason=decision.reason + ) + # Keep the superseded vector. Every vector backend applies the same temporal + # SearchFilter as lexical/graph retrieval, so it is hidden from current recall + # but remains available for historical ``as_of`` queries. Deleting it made + # time travel silently lose the semantic arm. self.store.audit("resolver", "invalidate", decision.target_id, decision.reason) linked = self._evolve(mid, neighbors, exclude={decision.target_id}) out = {"id": mid, "op": "invalidate", "superseded": [decision.target_id], @@ -547,7 +641,16 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, return out linked = self._evolve(mid, neighbors) - out = {"id": mid, "op": "add", "reason": decision.reason if decision else ""} + if decision is not None and decision.op == ResolutionOp.RELATE: + related_to = decision.target_id + if related_to and not self.store.has_link(mid, related_to): + self.store.add_link(mid, related_to, "related", reason=decision.reason) + out = { + "id": mid, "op": "relate", "related_to": related_to, + "reason": decision.reason, + } + else: + out = {"id": mid, "op": "add", "reason": decision.reason if decision else ""} if linked: out["linked"] = linked return out @@ -630,6 +733,38 @@ def _has_structured_graph_metadata(self, metadata: dict) -> bool: or isinstance(structured.get("relations"), list) ) + def _link_memory_entities(self, memory_id: str, content: str, *, + workspace_id: str, repo_id: Optional[str], + valid_from: Optional[float]) -> None: + """Persist edge-derived and exact textual entity evidence for one memory.""" + owns_transaction = not self.store.conn.transaction_owned_by_current_thread() + try: + self.store.backfill_memory_entities_for_memory(memory_id) + entities = self.store.list_entities(SearchFilter( + # New repo memories must attach to workspace entities already + # visible to that repo, not only entities owned by the repo. + workspace_id=workspace_id, repo_id=repo_id, include_ancestors=True, + )) + for entity in entities: + name = (entity.name or "").strip() + if len(name) < 2: + continue + if re.search(r"(? list[str]: """A-MEM-style memory evolution on write: a new memory auto-links to its closest still-live neighbors and gives them a small @@ -663,7 +798,10 @@ def _evolve(self, new_id: str, neighbors: list, *, exclude: Optional[set] = None def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id: str, repo_id: Optional[str], session_id: Optional[str], - scope: Scope, mtype: MemoryType, candidate_k: int): + scope: Scope, mtype: MemoryType, candidate_k: int, + subject_key: str = "", claim_kind: str = "", + valid_at: Optional[float] = None, + content: Optional[str] = None): """Fetch same-scope neighbors via the vector index and run the deterministic resolver (``core.resolve``). Returns ``(decision, neighbors)`` so the caller can also evolve the neighborhood. Never raises — a broken/missing index degrades to @@ -671,23 +809,58 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id flt = SearchFilter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id if scope == Scope.SESSION else None, - scopes=[scope], mtypes=[mtype], + scopes=[scope], mtypes=[mtype], valid_at=valid_at, ) try: hits = self.index.search(vec, candidate_k, filter=flt) except Exception: - return None, [] - now = now_ts() + hits = [] + current_fallback = False + if not hits and valid_at is not None: + # A candidate may be backdated before an already-recorded claim. That claim + # is intentionally outside the candidate's valid-time view, but it still + # has to be found so the caller can reject an impossible supersession rather + # than silently creating overlapping history. This fallback is only a guard + # for an otherwise-empty temporal neighborhood; normal scheduled resolution + # remains anchored at the candidate's validity time above. + current_filter = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id if scope == Scope.SESSION else None, + scopes=[scope], mtypes=[mtype], + ) + try: + hits = self.index.search(vec, candidate_k, filter=current_filter) + current_fallback = True + except Exception: + pass neighbors = [] for nid, sim in hits: nrec = self.store.get_memory(nid) if (nrec and nrec.workspace_id == workspace_id and nrec.repo_id == repo_id and nrec.scope == scope and nrec.mtype == mtype and (scope != Scope.SESSION or nrec.session_id == session_id) - and nrec.expired_at is None - and (nrec.valid_to is None or nrec.valid_to > now)): + and (memory_matches_filter(nrec, flt) + or (current_fallback and nrec.expired_at is None + and nrec.valid_to is None))): neighbors.append((sim, nrec)) - return resolve(text, neighbors), neighbors + if valid_at is not None and subject_key: + # The vector search above is intentionally anchored at the candidate's world + # time. Its top-K may still be non-empty with unrelated facts, so a fallback + # conditioned on ``not hits`` is not sufficient for a keyed claim: always add + # the exact current identity as a chronology guard. + known_ids = {rec.id for _, rec in neighbors} + for record in self.store.list_live_claims( + workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id if scope == Scope.SESSION else None, + scope=scope, mtype=mtype, subject_key=subject_key, + claim_kind=claim_kind, + ): + if record.id not in known_ids: + neighbors.append((1.0, record)) + return resolve( + text, neighbors, subject_key=subject_key, claim_kind=claim_kind, + candidate_content=content, + ), neighbors # ── ingest: extract-then-remember ─────────────────────────────────────────── def ingest(self, text: str, *, workspace_id: str, repo_id: Optional[str] = None, @@ -767,7 +940,9 @@ def consolidate(self, *, workspace_id: str, repo_id: Optional[str] = None, # ── read ────────────────────────────────────────────────────────────────── def _recall_filter(self, *, workspace_id: Optional[str], repo_id: Optional[str], session_id: Optional[str], scopes: Optional[list], - mtypes: Optional[list], as_of: Optional[float]) -> SearchFilter: + mtypes: Optional[list], as_of: Optional[float], + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> SearchFilter: """Build an ancestor-aware filter, resolving a session's parent repo in core. The service performs the same validation for friendly error payloads, but direct @@ -785,25 +960,40 @@ def _recall_filter(self, *, workspace_id: Optional[str], repo_id: Optional[str], repo_id = repo_id or session.get("repo_id") return SearchFilter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, include_ancestors=True, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, include_ancestors=True, ) def recall(self, query: str, *, workspace_id: Optional[str] = None, repo_id: Optional[str] = None, session_id: Optional[str] = None, scopes: Optional[list] = None, mtypes: Optional[list] = None, as_of: Optional[float] = None, - k: int = 8) -> RecallResult: + valid_at: Optional[float] = None, known_at: Optional[float] = None, + k: int = 8, token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", diagnostics: bool = False, + reinforce: bool = False) -> RecallResult: flt = self._recall_filter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, + ) + # Recall is observational unless the caller has an explicit use signal. + # Historical inspection is always observational: reinforcement would make a + # past reconstruction alter future ranking. + return self.recall_engine.recall( + query, flt, k=k, reinforce=bool(reinforce) and not flt.historical, + token_budget=token_budget, retrieval_profile=retrieval_profile, + diagnostics=diagnostics, ) - return self.recall_engine.recall(query, flt, k=k) def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, repo_id: Optional[str] = None, session_id: Optional[str] = None, scopes: Optional[list] = None, mtypes: Optional[list] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, known_at: Optional[float] = None, k: int = 8, llm=None, min_support: Optional[float] = None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", diagnostics: bool = False, max_citations: int = 5, reinforce: bool = True): """Recall, then answer *strictly from* what was recalled — with citations and an explicit abstain when the evidence is too weak (``core.grounded``). Offline and @@ -817,23 +1007,28 @@ def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, from engraphis.core import grounded as _grounded flt = self._recall_filter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, ) # Recall without reinforcing here: a grounded read should reward only the memories # it actually cites, and an abstain should reward nothing — don't reinforce the # irrelevant nearest-neighbours an off-topic query happened to surface. - result = self.recall_engine.recall(query, flt, k=k, reinforce=False) + result = self.recall_engine.recall( + query, flt, k=k, reinforce=False, token_budget=token_budget, + retrieval_profile=retrieval_profile, diagnostics=diagnostics, + ) floor = _grounded.GROUNDED_SUPPORT_FLOOR if min_support is None else min_support answer = _grounded.build_grounded_answer(query, result, self.embedder, llm=llm, min_support=floor, max_citations=max_citations) - if reinforce and answer.grounded: + if reinforce and not flt.historical and answer.grounded: for cite in answer.citations: if cite.get("id"): self.store.reinforce(cite["id"], boost=scoring.INTERACTION_BOOST["recall"]) return answer def why(self, query: str, *, workspace_id: str, repo_id: Optional[str] = None, - k: int = 5) -> dict: + k: int = 5, valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: """Rationale + history for a decision or fact: the live answer, plus whatever it superseded, if anything. This is the bi-temporal "why" that a flat-namespace store (or a plain vector store) cannot answer — the @@ -841,6 +1036,7 @@ def why(self, query: str, *, workspace_id: str, repo_id: Optional[str] = None, """ flt = SearchFilter( workspace_id=workspace_id, repo_id=repo_id, include_ancestors=True, + valid_at=valid_at, known_at=known_at, ) live = [r for _, r in self._relatedness(query, flt, include_invalid=False)[:k]] history: list[MemoryRecord] = [] @@ -857,12 +1053,14 @@ def why(self, query: str, *, workspace_id: str, repo_id: Optional[str] = None, return {"answer": live, "supersedes": history} def timeline(self, query: str, *, workspace_id: str, repo_id: Optional[str] = None, - limit: int = 20) -> list[MemoryRecord]: + limit: int = 20, valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> list[MemoryRecord]: """Chronological, bi-temporal history of a fact: what we believed and when. Includes invalidated versions; sorted by ``valid_from``. """ flt = SearchFilter( workspace_id=workspace_id, repo_id=repo_id, include_ancestors=True, + valid_at=valid_at, known_at=known_at, ) recs = [r for _, r in self._relatedness(query, flt, include_invalid=True)[:limit]] recs.sort(key=lambda r: r.valid_from or r.ingested_at or 0.0) @@ -884,7 +1082,16 @@ def _relatedness(self, query: str, flt: SearchFilter, *, sem[mid] = float(np.dot(qn, vec)) q_tokens = tokenize(query) out: list[tuple[float, MemoryRecord]] = [] - for rec in self.store.list_memories(flt, include_invalid=include_invalid, limit=500): + records = self.store.list_memories(flt, include_invalid=include_invalid, limit=500) + if include_invalid and flt.known_at is not None: + # History must retain closed valid-time intervals, but cannot expose a + # record that was not known at the requested system-time snapshot. + records = [ + rec for rec in records + if (rec.ingested_at is None or rec.ingested_at <= flt.known_at) + and (rec.expired_at is None or flt.known_at < rec.expired_at) + ] + for rec in records: lex = jaccard(q_tokens, tokenize(f"{rec.title} {rec.content}")) score = max(sem.get(rec.id, 0.0), lex) if score > 0.05: @@ -931,10 +1138,8 @@ def forget(self, memory_id: str, *, reason: str = "", actor: str = "user") -> di if self.store.get_memory(memory_id) is None: raise KeyError(f"no memory with id '{memory_id}'") self.store.close_validity(memory_id, actor=actor, reason=reason or "forgotten by request") - try: - self.index.delete([memory_id]) - except Exception: - pass + # Preserve the vector for explicit historical/as_of recall. Temporal filtering + # keeps this retired row out of the current live view. return {"id": memory_id, "status": "forgotten", "reason": reason} def pin(self, memory_id: str, *, pinned: bool = True, actor: str = "user") -> dict: @@ -980,10 +1185,8 @@ def correct(self, memory_id: str, new_content: str, *, reason: str = "", if old.pinned: self.store.set_pinned(new_id, True) self.store.close_validity(memory_id, actor=actor, reason=reason or "corrected") - try: - self.index.delete([memory_id]) - except Exception: - pass + # The old vector is historical evidence; SearchFilter validity hides it from + # current recall while keeping semantic time travel complete. return {"id": new_id, "superseded": [memory_id], "reason": reason} def promote(self, memory_id: str, target_scope: Scope, *, reason: str = "", @@ -1096,10 +1299,7 @@ def promote(self, memory_id: str, target_scope: Scope, *, reason: str = "", old.id, actor=actor, reason=reason or f"promoted from {old.scope.value} to {target_scope.value}", ) - try: - self.index.delete([old.id]) - except Exception: - pass + # Preserve the source vector for historical/as_of inspection. if not self.store.has_link(promoted_id, old.id, relation="promotes"): self.store.add_link( promoted_id, old.id, "promotes", reason=reason or "scope promotion" @@ -1207,10 +1407,7 @@ def merge(self, source_ids: list, merged_content: str, *, for r in sources: self.store.close_validity(r.id, actor=actor, reason=reason or "merged into a combined memory") - try: - self.index.delete([r.id]) - except Exception: - pass + # Preserve source vectors for historical/as_of retrieval. # Linking/auditing stays a separate pass so the audit trail keeps its original # shape: every source's invalidate entry, then every source's merge entry. for r in sources: @@ -1434,9 +1631,12 @@ def search_code(self, query: str, *, repo_id: str, limit: int = 20, """Symbol-graph + lexical code search — far cheaper than dumping files for structural questions, and (via ``called_by``) answers "what breaks if I change X" directly from the call graph.""" - symbols = self.store.search_symbols(repo_id, query, limit=limit) + self._validate_code_filter(repo_id, flt) + symbols = self.store.search_symbols(repo_id, query, limit=limit, flt=flt) for s in symbols: - s["called_by"] = self.store.get_symbol_callers(repo_id, s["name"], limit=10) + s["called_by"] = self.store.get_symbol_callers( + repo_id, s["name"], limit=10, flt=flt + ) s["linked_memories"] = self.store.memories_for_symbol( repo_id, s["id"], flt=flt, limit=10 ) @@ -1523,10 +1723,6 @@ def rebuild_code_memory_links(self, *, repo_id: str) -> int: ) if not records: break - memory_ids = [record.id for record in records] - self.store.clear_code_memory_links_for_memories( - repo_id, memory_ids, commit=False, - ) linked_per_memory = {record.id: 0 for record in records} symbol_cursor: Optional[tuple[str, str, str]] = None while True: @@ -1562,8 +1758,9 @@ def rebuild_code_memory_links(self, *, repo_id: str) -> int: def code_path(self, source: str, target: str, *, repo_id: str, max_depth: int = 8, flt: Optional[SearchFilter] = None) -> dict: """Shortest path across definitions, calls, imports, and symbol aliases.""" - symbols = self.store.list_symbols(repo_id) - stored_edges = self.store.list_code_edges(repo_id) + self._validate_code_filter(repo_id, flt) + symbols = self.store.list_symbols(repo_id, flt=flt) + stored_edges = self.store.list_code_edges(repo_id, flt=flt) adjacency: dict[str, list[tuple[str, dict, bool]]] = defaultdict(list) node_meta: dict[str, dict] = {} for sym in symbols: @@ -1586,13 +1783,7 @@ def code_path(self, source: str, target: str, *, repo_id: str, node_meta.setdefault(src, {"kind": "code", "name": src}) node_meta.setdefault(dst, {"kind": "code", "name": dst}) symbol_by_id = {symbol["id"]: symbol for symbol in symbols} - now = now_ts() for link in self.store.list_code_memory_links(repo_id, flt=flt): - if link.get("expired_at") is not None: - continue - valid_to = link.get("valid_to") - if valid_to is not None and now >= float(valid_to): - continue symbol = symbol_by_id.get(link.get("symbol_id")) if not symbol or not link.get("memory_id"): continue @@ -1705,15 +1896,17 @@ def _resolve_code_node(query: str, symbols: list[dict], def analyze_code_graph(self, *, repo_id: str, limit: Optional[int] = None, - edge_limit: Optional[int] = None) -> dict: + edge_limit: Optional[int] = None, + flt: Optional[SearchFilter] = None) -> dict: """Deterministic weighted communities, hotspots, and cross-file connections. ``limit``/``edge_limit`` bound the symbol/edge fetch. They default to ``None`` (unbounded) so ``analyze_impact`` keeps today's exact answer; ``export_code_graph`` passes its own caps because that payload is reachable by a ``viewer``. """ - edges = self.store.list_code_edges(repo_id, limit=edge_limit) - symbols = self.store.list_symbols(repo_id, limit=limit) + self._validate_code_filter(repo_id, flt) + edges = self.store.list_code_edges(repo_id, limit=edge_limit, flt=flt) + symbols = self.store.list_symbols(repo_id, limit=limit, flt=flt) adjacency: dict[str, dict[str, float]] = defaultdict(dict) degree: dict[str, int] = defaultdict(int) for edge in edges: @@ -1814,6 +2007,7 @@ def analyze_code_graph(self, *, repo_id: str, def analyze_impact(self, changed_files: list[str], *, repo_id: str, flt: Optional[SearchFilter] = None) -> dict: """Estimate graph and memory impact for a git diff / PR file list.""" + self._validate_code_filter(repo_id, flt) normalized = [] seen = set() for file in changed_files: @@ -1825,12 +2019,12 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, if rel and rel not in seen: seen.add(rel) normalized.append(rel) - symbols = self.store.symbols_for_files(repo_id, normalized) + symbols = self.store.symbols_for_files(repo_id, normalized, flt=flt) touched_names = { name for sym in symbols for name in (sym.get("name"), sym.get("fqname")) if name } touched_leaf_names = {str(name).split(".")[-1] for name in touched_names} - edges = self.store.list_code_edges(repo_id) + edges = self.store.list_code_edges(repo_id, flt=flt) inbound = [ edge for edge in edges if edge.get("dst") in touched_names @@ -1844,13 +2038,7 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, memory_mentions: dict[str, dict] = {} touched_symbol_ids = {symbol["id"] for symbol in symbols} - now = now_ts() for link in self.store.list_code_memory_links(repo_id, flt=flt): - if link.get("expired_at") is not None: - continue - valid_to = link.get("valid_to") - if valid_to is not None and now >= float(valid_to): - continue if link.get("symbol_id") not in touched_symbol_ids: continue item = memory_mentions.setdefault( @@ -1881,7 +2069,7 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, ) item["symbols"].append(name) - analysis = self.analyze_code_graph(repo_id=repo_id) + analysis = self.analyze_code_graph(repo_id=repo_id, flt=flt) node_community = analysis.pop("_node_community") communities_affected = sorted({ node_community[name] for name in touched_names if name in node_community @@ -1940,16 +2128,17 @@ def export_code_graph(self, *, repo_id: str, """ limit = max(1, min(CODE_EXPORT_MAX_LIMIT, int(limit))) edge_cap = max(limit * 8, 2_000) + self._validate_code_filter(repo_id, flt) analysis = self.analyze_code_graph(repo_id=repo_id, limit=limit, - edge_limit=edge_cap) + edge_limit=edge_cap, flt=flt) analysis.pop("_node_community", None) # Fetch one sentinel row beyond the payload cap so truncation stays observable # without materializing every indexed file in a large repository. - files = self.store.list_code_files(repo_id, limit=limit + 1) + files = self.store.list_code_files(repo_id, flt=flt, limit=limit + 1) truncated_files = len(files) > limit files = files[:limit] - nodes = self.store.list_symbols(repo_id, limit=limit) - edges = self.store.list_code_edges(repo_id, limit=edge_cap) + nodes = self.store.list_symbols(repo_id, limit=limit, flt=flt) + edges = self.store.list_code_edges(repo_id, limit=edge_cap, flt=flt) memory_links = self.store.list_code_memory_links( repo_id, flt=flt, limit=edge_cap ) @@ -1970,6 +2159,28 @@ def export_code_graph(self, *, repo_id: str, "analysis": analysis, } + def _validate_code_filter( + self, repo_id: str, flt: Optional[SearchFilter] + ) -> None: + """Reject inconsistent repo/workspace filters before any code row is read. + + Code-history tables are keyed by ``repo_id`` rather than duplicating a + workspace column. Without this check, a direct engine caller could pair a + workspace-A filter with a workspace-B repo id and receive B's symbols even + though memory reads correctly returned nothing. + """ + if flt is None: + return + if flt.repo_id is not None and flt.repo_id != repo_id: + raise ValueError("code filter repo_id does not match the requested repo") + if flt.workspace_id is None: + return + row = self.store.conn.execute( + "SELECT workspace_id FROM repos WHERE id=?", (repo_id,) + ).fetchone() + if row is None or row["workspace_id"] != flt.workspace_id: + raise ValueError("code filter workspace_id does not own the requested repo") + def code_graph_report(self, *, repo_id: str, payload: Optional[dict] = None, flt: Optional[SearchFilter] = None) -> str: """Human-readable GRAPH_REPORT.md companion to :meth:`export_code_graph`. diff --git a/engraphis/core/graphrank.py b/engraphis/core/graphrank.py index 34d73325..38a1e744 100644 --- a/engraphis/core/graphrank.py +++ b/engraphis/core/graphrank.py @@ -1,20 +1,24 @@ -"""Personalized PageRank over the memory/entity graph. +"""Deterministic sparse personalized PageRank for local memory graphs. -HippoRAG-style single-step graph retrieval: seed the walk at the query's entities and -let the stationary distribution rank everything reachable — multi-hop associations -included — instead of expanding a fixed number of hops. Pure NumPy (AGENTS.md §3.8), -deterministic, and sized for the local-first reality: the adjacency is built per query -from the scoped store (hundreds to low thousands of nodes), where a dense power -iteration is both exact and fast. A sparse/persistent implementation can replace this -behind the same function signature when scale demands it. +The graph arm can contain thousands of entities, memories, and links. A dense +``N × N`` transition matrix turns an otherwise modest local graph into quadratic +memory pressure, so this implementation stores only normalized outgoing edges +and walks them directly. It deliberately depends on no sparse-matrix package. """ from __future__ import annotations -import numpy as np +import math + DAMPING = 0.85 ITERATIONS = 30 TOL = 1e-9 +# Safety limits for direct callers. Recall already builds a bounded scoped graph; +# these make a malformed local/plugin adjacency fail deterministically rather than +# allocating unbounded state. They are comfortably above normal local graph arms. +MAX_NODES = 100_000 +MAX_EDGES = 1_000_000 +MAX_ITERATIONS = 100 def personalized_pagerank( @@ -25,49 +29,82 @@ def personalized_pagerank( iterations: int = ITERATIONS, tol: float = TOL, ) -> dict[str, float]: - """Rank nodes by their stationary probability under a random walk with restart. + """Rank nodes by a sparse random walk with restart. - ``adjacency`` maps node -> [(neighbor, weight), ...]; pass both directions for an - undirected graph. ``seeds`` are the restart set (unknown seeds are ignored). Nodes - unreachable from every seed score 0. Returns {} when there is nothing to walk. + ``adjacency`` maps node -> ``[(neighbor, weight), ...]``; pass both + directions for an undirected graph. Unknown seed ids retain the legacy + restart behavior when at least one seed has outgoing adjacency. Oversized + inputs return ``{}`` deterministically instead of attempting an unbounded + local computation. """ if not adjacency or not seeds: return {} - nodes: list[str] = sorted( - set(adjacency) - | {dst for nbrs in adjacency.values() for dst, _ in nbrs} - | set(seeds) - ) - idx = {n: i for i, n in enumerate(nodes)} - n = len(nodes) + nodes = set(adjacency) + edge_count = 0 + for neighbors in adjacency.values(): + edge_count += len(neighbors) + if edge_count > MAX_EDGES: + return {} + nodes.update(dst for dst, _ in neighbors) + nodes.update(seeds) + if len(nodes) > MAX_NODES: + return {} - seed_ids = [idx[s] for s in seeds if s in idx] - live_seeds = [s for s in seeds if s in adjacency and adjacency[s]] + ordered_nodes = sorted(nodes) + node_index = {node: index for index, node in enumerate(ordered_nodes)} + n_nodes = len(ordered_nodes) + seed_ids = [node_index[seed] for seed in seeds if seed in node_index] + live_seeds = [seed for seed in seeds if seed in adjacency and adjacency[seed]] if not seed_ids or not live_seeds: return {} - # Column-stochastic transition matrix; dangling nodes restart to the seeds. - M = np.zeros((n, n), dtype=np.float64) - for src, nbrs in adjacency.items(): - col = idx[src] - total = float(sum(max(w, 0.0) for _, w in nbrs)) - if total <= 0.0: + # Aggregate duplicate destinations before applying a source's mass. This + # matches the old dense matrix's ``M[dst, src] += ...`` semantics while + # keeping the storage and each iteration O(nodes + edges). + outgoing: list[list[tuple[int, float]]] = [[] for _ in range(n_nodes)] + for source in ordered_nodes: + neighbors = adjacency.get(source, []) + total = sum(max(float(weight), 0.0) for _, weight in neighbors) + if total <= 0.0 or not math.isfinite(total): continue - for dst, w in nbrs: - if w > 0.0: - M[idx[dst], col] += w / total - - restart = np.zeros(n, dtype=np.float64) - restart[seed_ids] = 1.0 / len(seed_ids) - dangling = M.sum(axis=0) == 0.0 + destination_weights: dict[int, float] = {} + for destination, weight in neighbors: + if weight > 0.0: + destination_id = node_index[destination] + destination_weights[destination_id] = ( + destination_weights.get(destination_id, 0.0) + float(weight) / total + ) + outgoing[node_index[source]] = list(destination_weights.items()) - p = restart.copy() - for _ in range(iterations): - spread = M @ p + p[dangling].sum() * restart - p_next = (1.0 - damping) * restart + damping * spread - if float(np.abs(p_next - p).sum()) < tol: - p = p_next + restart = [0.0] * n_nodes + for seed_id in seed_ids: + restart[seed_id] = 1.0 / len(seed_ids) + dangling = [index for index, neighbors in enumerate(outgoing) if not neighbors] + probability = restart[:] + iteration_limit = max(0, min(int(iterations), MAX_ITERATIONS)) + for _ in range(iteration_limit): + spread = [0.0] * n_nodes + for source_id, edges in enumerate(outgoing): + if probability[source_id] == 0.0: + continue + for destination_id, weight in edges: + spread[destination_id] += probability[source_id] * weight + dangling_mass = sum(probability[index] for index in dangling) + if dangling_mass: + for index, weight in enumerate(restart): + if weight: + spread[index] += dangling_mass * weight + next_probability = [ + (1.0 - damping) * restart[index] + damping * spread[index] + for index in range(n_nodes) + ] + if sum(abs(after - before) for after, before in zip(next_probability, probability)) < tol: + probability = next_probability break - p = p_next + probability = next_probability - return {nodes[i]: float(p[i]) for i in range(n) if p[i] > 0.0} + return { + ordered_nodes[index]: score + for index, score in enumerate(probability) + if score > 0.0 + } diff --git a/engraphis/core/grounded.py b/engraphis/core/grounded.py index b49a8664..179c3b59 100644 --- a/engraphis/core/grounded.py +++ b/engraphis/core/grounded.py @@ -11,10 +11,10 @@ * **Deterministic (offline default).** No LLM. The answer is an *extractive* stitch of the cited memories — it never introduces a claim that is not in a source. The - groundedness verdict is computed from an absolute query-memory support signal (the - max of semantic cosine and lexical Jaccard), independent of the relative, per-query - recall score, so "insufficient evidence" is a real threshold rather than a ranking - artefact. + groundedness verdict is computed from an absolute query-memory support signal + (semantic cosine plus lexical/predicate agreement), independent of the relative, + per-query recall score, so "insufficient evidence" is a real threshold rather than + a ranking artefact. * **Synthesised (opt-in).** If an object implementing ``core.interfaces.LLM`` is injected, it may write prose — but constrained to the same numbered sources and the same abstain sentinel, and it degrades to the extractive answer on any error. @@ -27,12 +27,14 @@ """ from __future__ import annotations +import math import re -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from typing import Optional import numpy as np +from engraphis.core.context import RegexTokenCounter from engraphis.core.interfaces import LLM from engraphis.core.recall import RecallResult from engraphis.core.textutil import jaccard, tokenize @@ -46,6 +48,16 @@ GROUNDED_SUPPORT_FLOOR = 0.25 ABSTAIN_SENTINEL = "INSUFFICIENT_EVIDENCE" _CITE_RE = re.compile(r"\[(\d+)\]") +_QUERY_FRAMING_TERMS = { + "what", "which", "who", "where", "when", "why", "how", "scheme", "format", +} +# Words which can make a citation grammatical without making an additional factual +# claim. The LLM verifier below deliberately permits only these words in addition +# to source tokens. Unknown paraphrases safely fall back to extractive evidence. +_SYNTHESIS_GLUE_TERMS = { + "according", "answer", "answers", "based", "evidence", "indicates", "per", + "provided", "said", "says", "source", "sources", "states", "supports", +} @dataclass @@ -63,9 +75,16 @@ class GroundedAnswer: support: float = 0.0 synthesized: bool = False citations: list[dict] = field(default_factory=list) + usage: dict = field(default_factory=dict) + packed_sources: list[dict] = field(default_factory=list) + valid_at: Optional[float] = None + known_at: Optional[float] = None + historical: bool = False + retrieval_profile: str = "balanced" + retrieval_trace: Optional[list[dict]] = None def to_dict(self) -> dict: - return { + payload = { "answer": self.answer, "grounded": self.grounded, "abstained": self.abstained, @@ -73,7 +92,16 @@ def to_dict(self) -> dict: "support": round(self.support, 4), "synthesized": self.synthesized, "citations": self.citations, + "usage": self.usage, + "packed_sources": self.packed_sources, + "valid_at": self.valid_at, + "known_at": self.known_at, + "historical": self.historical, + "retrieval_profile": self.retrieval_profile, } + if self.retrieval_trace is not None: + payload["retrieval_trace"] = self.retrieval_trace + return payload def _filtered_text(text: str) -> str: @@ -85,37 +113,129 @@ def _filtered_text(text: str) -> str: return " ".join(sorted(toks)) if toks else (text or "") +def _related_term_count(query_tokens: set[str], content_tokens: set[str]) -> int: + """Count conservative exact/morphological term matches. + + A single shared topic word is not evidence for the query's predicate + (``bake sourdough`` versus ``orders sourdough``). Prefix agreement also + recognizes ordinary inflections such as ``token``/``tokens`` and + ``standardise``/``standardised`` without a language model. + """ + matched = 0 + for query_term in query_tokens: + for content_term in content_tokens: + if query_term == content_term: + matched += 1 + break + shorter = min(len(query_term), len(content_term)) + if shorter < 5: + continue + common = 0 + for left, right in zip(query_term, content_term): + if left != right: + break + common += 1 + if common >= max(5, min(7, shorter)): + matched += 1 + break + return matched + + def _support_scores(query: str, contents: list[str], embedder) -> list[float]: - """Absolute per-source support = max(semantic cosine, lexical Jaccard), in [0, 1]. + """Absolute per-source support from semantic, lexical, and predicate agreement. Both arms are query-independent in scale — unlike the recall score, which is min-max normalised *per query* and so cannot be compared against a fixed threshold. That is why groundedness is recomputed here rather than read off ``chunk["score"]``. The - cosine is taken over *stopword-filtered* text so shared filler words don't register - as evidence. + cosine is taken over *stopword-filtered* text, then conservatively discounted when + a multi-term query and source share only one topic term. """ if not contents: return [] - q_tokens = tokenize(query) + q_tokens = tokenize(query) - _QUERY_FRAMING_TERMS texts = [_filtered_text(query)] + [_filtered_text(c) for c in contents] vecs = embedder.embed(texts) qn = np.asarray(vecs[0], dtype=float) qn = qn / (float(np.linalg.norm(qn)) or 1.0) out: list[float] = [] for i, content in enumerate(contents): + content_tokens = tokenize(content) cv = np.asarray(vecs[i + 1], dtype=float) cn = cv / (float(np.linalg.norm(cv)) or 1.0) - cos = float(np.dot(qn, cn)) - lex = jaccard(q_tokens, tokenize(content)) + cos = max(0.0, float(np.dot(qn, cn))) + lex = jaccard(q_tokens, content_tokens) + related_terms = _related_term_count(q_tokens, content_tokens) + # Hashing and dense embedders can consider two texts topically similar + # when they share one salient noun but make unrelated claims. Require a + # second predicate/qualifier match for ordinary multi-term questions, + # while allowing genuinely strong semantic paraphrases to stand alone. + if len(q_tokens) >= 3 and related_terms < 2 and cos < 0.6: + cos *= related_terms / 2.0 out.append(max(cos, lex)) return out -def _cites_a_source(text: str, n_citations: int) -> bool: - """True if ``text`` has at least one ``[i]`` marker with ``1 <= i <= n_citations``. - Guards the synthesised path: prose that cites nothing may have introduced an uncited - (possibly fabricated) claim, so it is rejected in favour of the extractive answer.""" - return any(1 <= int(m) <= n_citations for m in _CITE_RE.findall(text)) +def _citations_are_valid(text: str, n_citations: int) -> bool: + """Require at least one citation and reject every out-of-range marker. + + Accepting prose merely because *one* marker was valid let an answer combine + ``[1]`` with fabricated ``[99]`` evidence. Structural citation integrity is + fail-closed: every numbered source reference must resolve to a supplied source. + """ + markers = [int(marker) for marker in _CITE_RE.findall(text)] + return bool(markers) and all(1 <= marker <= n_citations for marker in markers) + + +def _ordered_tokens(text: str) -> list[str]: + """Case-folded lexical tokens with order and small numbers preserved.""" + return re.findall(r"[^\W_]+", text.casefold(), flags=re.UNICODE) + + +def _contains_span(source: list[str], claim: list[str]) -> bool: + """Whether ``claim`` is one exact contiguous lexical span of ``source``.""" + width = len(claim) + return bool(width) and any( + source[start:start + width] == claim + for start in range(0, len(source) - width + 1) + ) + + +def _synthesis_is_source_bounded(text: str, citations: list[dict]) -> bool: + """Return whether each cited synthesis clause is extractive from one source. + + Citation syntax alone cannot prove a generated claim is present in its source: + ``Invented fact [1]`` has a valid marker but no evidence. A vocabulary-set check + is also insufficient: ``Alice approved alpha, not beta`` reuses every token in + ``Alice approved beta, not alpha`` while reversing its meaning. A general + entailment checker would require another fallible model, so the safe offline + verifier accepts only an exact ordered source span after removing a narrow set + of citation glue words. Legitimate paraphrases that fail this conservative check + degrade to the deterministic extractive answer instead of being labelled grounded. + """ + if not _citations_are_valid(text, len(citations)): + return False + sources = { + int(citation["n"]): _ordered_tokens(str(citation.get("content", ""))) + for citation in citations + if isinstance(citation.get("n"), int) + } + clauses = [clause.strip() for clause in re.split(r"(?<=[.!?])\s+", text) if clause.strip()] + if not clauses: + return False + for clause in clauses: + markers = [int(marker) for marker in _CITE_RE.findall(clause)] + if not markers: + return False + claim_tokens = [ + token for token in _ordered_tokens(_CITE_RE.sub("", clause)) + if token not in _SYNTHESIS_GLUE_TERMS + ] + if not claim_tokens or not any( + _contains_span(sources.get(marker, []), claim_tokens) + for marker in markers + ): + return False + return True def build_grounded_answer(query: str, result: RecallResult, embedder, *, @@ -127,12 +247,49 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, Deterministic and offline unless an ``LLM`` is injected. Never raises on LLM failure — it degrades to the extractive answer. """ - # Score support over ALL retrieved memories (not just the first max_citations) so a - # strongly-supporting memory ranked lower by the fused recall score still counts. - chunks = list(result.chunks) + try: + min_support = float(min_support) + except (TypeError, ValueError) as exc: + raise ValueError("min_support must be a finite number between 0 and 1") from exc + if not math.isfinite(min_support) or not 0.0 <= min_support <= 1.0: + raise ValueError("min_support must be a finite number between 0 and 1") + try: + max_citations = int(max_citations) + except (TypeError, ValueError) as exc: + raise ValueError("max_citations must be a positive integer") from exc + if max_citations < 1: + raise ValueError("max_citations must be a positive integer") + + # Grounding may use only evidence the ContextPacker actually admitted. Raw retrieval + # candidates can be omitted or truncated by the caller's token budget and therefore + # are not evidence available to the answerer. + raw_by_id = {str(chunk.get("id")): chunk for chunk in result.chunks} + chunks = [] + for packed in result.packed_chunks: + raw = raw_by_id.get(str(packed.id)) + if raw is None or not packed.excerpt: + continue + chunks.append({**raw, "content": packed.excerpt}) contents = [str(c.get("content", "")) for c in chunks] per = _support_scores(query, contents, embedder) support = max(per) if per else 0.0 + count_answer_tokens = result.token_counter or RegexTokenCounter() + budget_tokens = result.usage.budget_tokens if result.usage is not None else 0 + recall_metadata = { + "usage": asdict(result.usage) if result.usage is not None else {}, + "packed_sources": [{ + "id": packed.id, + "tokens": packed.tokens, + "truncated": packed.truncated, + "reason": packed.reason, + } for packed in result.packed_chunks], + "valid_at": result.valid_at, + "known_at": result.known_at, + "historical": result.historical, + "retrieval_profile": result.retrieval_profile, + "retrieval_trace": result.retrieval_trace, + } + recall_metadata["usage"]["answer_tokens"] = 0 if not chunks or support < min_support: return GroundedAnswer( @@ -140,6 +297,7 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, reason=(f"no memory in scope sufficiently supports this query " f"(support {support:.3f} < floor {min_support:.3f}); " f"not answering rather than guessing"), + **recall_metadata, ) # Cite the sources that individually clear the floor, strongest evidence first, capped @@ -159,20 +317,36 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, stripped = (prose or "").strip() if stripped == ABSTAIN_SENTINEL: return GroundedAnswer(grounded=False, abstained=True, support=support, - reason="synthesiser judged the sources insufficient") - # Accept synthesised prose only if it actually cites a source; otherwise it may - # have introduced an uncited (possibly fabricated) claim, so fall back to the - # deterministic extractive answer — grounded and cited by construction. - if stripped and _cites_a_source(stripped, len(citations)): + reason="synthesiser judged the sources insufficient", + **recall_metadata) + # Markers alone are not evidence: an LLM can write "Invented fact [1]". + # Accept prose only after the deterministic, citation-specific source + # vocabulary check; otherwise return extractive evidence by construction. + answer_tokens = count_answer_tokens(stripped) + if ( + stripped + and answer_tokens <= budget_tokens + and _synthesis_is_source_bounded(stripped, citations) + ): + recall_metadata["usage"]["answer_tokens"] = answer_tokens return GroundedAnswer(answer=stripped, grounded=True, abstained=False, support=support, synthesized=True, - citations=citations) + citations=citations, **recall_metadata) except Exception: pass # any LLM failure -> fall through to the deterministic answer - return GroundedAnswer(answer=_extractive_answer(citations), grounded=True, + extractive = _extractive_answer(citations) + answer_tokens = count_answer_tokens(extractive) + if answer_tokens > budget_tokens: + return GroundedAnswer( + grounded=False, abstained=True, support=support, + reason="packed evidence cannot fit a cited answer within the token budget", + **recall_metadata, + ) + recall_metadata["usage"]["answer_tokens"] = answer_tokens + return GroundedAnswer(answer=extractive, grounded=True, abstained=False, support=support, synthesized=False, - citations=citations) + citations=citations, **recall_metadata) def _extractive_answer(citations: list[dict]) -> str: @@ -182,8 +356,10 @@ def _extractive_answer(citations: list[dict]) -> str: for c in citations: text = " ".join(str(c.get("content", "")).split()) title = str(c.get("title", "")).strip() - prefix = f"{title}: " if title else "" - lines.append(f"[{c['n']}] {prefix}{text}") + header = f"[{c['n']}]" + if title: + header += " " + " ".join(title.split())[:120] + lines.append(f"{header}\n{text}") return "\n".join(lines) diff --git a/engraphis/core/interfaces.py b/engraphis/core/interfaces.py index b0596d12..847161e6 100644 --- a/engraphis/core/interfaces.py +++ b/engraphis/core/interfaces.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import math from dataclasses import dataclass, field from enum import Enum from typing import Any, Iterable, Literal, Optional, Protocol, runtime_checkable @@ -40,6 +41,21 @@ class GraphLayer(str, Enum): SEMANTIC = "semantic" +def _finite_timestamp(value: Optional[float], name: str) -> Optional[float]: + """Normalize public temporal anchors and reject SQLite's non-finite values.""" + if value is None: + return None + if isinstance(value, bool): + raise ValueError(f"{name} must be a finite timestamp") + try: + timestamp = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a finite timestamp") from exc + if not math.isfinite(timestamp): + raise ValueError(f"{name} must be a finite timestamp") + return timestamp + + # ── Records ────────────────────────────────────────────────────────────────── @dataclass @@ -65,10 +81,13 @@ class MemoryRecord: valid_to: Optional[float] = None # world-time: when it stopped being true ingested_at: Optional[float] = None # system-time: when we learned it expired_at: Optional[float] = None # system-time: when we retired it + subject_key: str = "" # stable optional claim subject + claim_kind: str = "" # optional claim predicate/category pinned: bool = False sensitivity: str = "normal" # normal | sensitive | secret provenance: dict[str, Any] = field(default_factory=dict) embedding: Optional[np.ndarray] = None + valid_to_recorded_at: Optional[float] = None # when valid_to was learned @dataclass @@ -80,11 +99,34 @@ class SearchFilter: scopes: Optional[list[Scope]] = None mtypes: Optional[list[MemoryType]] = None graph_layers: Optional[list[GraphLayer]] = None - as_of: Optional[float] = None # bi-temporal time anchor; None = now + # ``as_of`` remains a compatibility alias for the world-time ``valid_at`` + # anchor. New callers can independently select what was true and what + # had been learned at that time. + as_of: Optional[float] = None # Contextual recall sees broader scopes as ancestors: a repo read can see that # repo plus workspace/user memories, and a session read can additionally see its # exact session. Storage/governance queries stay exact unless they opt in. include_ancestors: bool = False + # Appended after every 1.x field so positional construction remains compatible. + valid_at: Optional[float] = None + known_at: Optional[float] = None + + def __post_init__(self) -> None: + self.as_of = _finite_timestamp(self.as_of, "as_of") + self.valid_at = _finite_timestamp(self.valid_at, "valid_at") + self.known_at = _finite_timestamp(self.known_at, "known_at") + if self.as_of is not None and self.valid_at is not None: + if self.as_of != self.valid_at: + raise ValueError("as_of and valid_at must match when both are supplied") + # Keep legacy backends that read ``as_of`` correct as callers move to + # the less ambiguous ``valid_at`` name. + self.valid_at = self.valid_at if self.valid_at is not None else self.as_of + self.as_of = self.valid_at + + @property + def historical(self) -> bool: + """Whether either time axis was explicitly anchored by the caller.""" + return self.valid_at is not None or self.known_at is not None @dataclass @@ -96,6 +138,29 @@ class Candidate: record: Optional[MemoryRecord] = None +@dataclass +class PackedChunk: + """One source excerpt selected by a context-packing implementation.""" + id: str + excerpt: str + tokens: int + truncated: bool = False + reason: str = "" + + +@dataclass +class ContextUsage: + """Token accounting emitted by a context-packing implementation.""" + budget_tokens: int + context_tokens: int + source_tokens: int + saved_tokens: int + savings_ratio: float + packed_count: int + omitted_count: int + token_counter: str = "estimate_tokens" + + @dataclass class Node: """A knowledge-graph node (entity or concept).""" @@ -123,6 +188,7 @@ class Edge: ingested_at: Optional[float] = None expired_at: Optional[float] = None provenance: dict[str, Any] = field(default_factory=dict) + valid_to_recorded_at: Optional[float] = None @dataclass @@ -219,6 +285,20 @@ class Reranker(Protocol): def rerank(self, query: str, candidates: list[Candidate], k: int) -> list[Candidate]: ... +@runtime_checkable +class ContextPacker(Protocol): + """Choose budgeted, explainable source excerpts for an agent context.""" + def pack(self, query: str, candidates: list[Candidate], token_budget: int + ) -> tuple[str, list[PackedChunk], ContextUsage]: ... + def count_tokens(self, text: str) -> int: ... + + +@runtime_checkable +class RetrievalPolicy(Protocol): + """Select a named retrieval profile without coupling core to a backend.""" + def profile(self, query: str) -> str: ... + + @runtime_checkable class LLM(Protocol): """External or local model for synthesis and structured extraction (§8.2).""" diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py index 468c57a6..77bce3f9 100644 --- a/engraphis/core/recall.py +++ b/engraphis/core/recall.py @@ -12,18 +12,30 @@ """ from __future__ import annotations +import inspect import re -from dataclasses import dataclass, field -from typing import Optional +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Optional from engraphis.core import scoring +from engraphis.core.context import DeterministicContextPacker from engraphis.core.graphrank import personalized_pagerank from engraphis.core.interfaces import ( Candidate, + ContextPacker, + ContextUsage, MemoryRecord, + PackedChunk, Reranker, + RetrievalPolicy, SearchFilter, ) +from engraphis.core.retrieval_policy import ( + DeterministicRetrievalPolicy, + ProfileConfig, + RETRIEVAL_PROFILES, + profile_config, +) from engraphis.core.store import Store, memory_matches_filter, now_ts @@ -32,12 +44,22 @@ class RecallResult: chunks: list[dict] = field(default_factory=list) context: str = "" count: int = 0 + packed_chunks: list[PackedChunk] = field(default_factory=list) + usage: Optional[ContextUsage] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None + historical: bool = False + retrieval_profile: str = "balanced" + retrieval_trace: Optional[list[dict[str, Any]]] = None + token_counter: Optional[Callable[[str], int]] = field(default=None, repr=False) class RecallEngine: def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Reranker] = None, *, weights: Optional[dict] = None, recency_tau_days: float = 30.0, - token_budget: int = 1500, graph_mode: str = "ppr") -> None: + token_budget: int = 1500, graph_mode: str = "ppr", + context_packer: Optional[ContextPacker] = None, + retrieval_policy: Optional[RetrievalPolicy] = None) -> None: self.store = store self.embedder = embedder self.index = vector_index @@ -45,27 +67,73 @@ def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Rera self.weights = weights or scoring.DEFAULT_WEIGHTS self.recency_tau_days = recency_tau_days self.token_budget = token_budget + self.context_packer = context_packer or DeterministicContextPacker() + self.retrieval_policy = retrieval_policy or DeterministicRetrievalPolicy() # "ppr" (default) = Personalized PageRank over entities+links (multi-hop); # "1hop" = the Phase-1 entity expansion, kept for fallback and ablation. self.graph_mode = graph_mode def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, - candidate_k: int = 50, reinforce: bool = True) -> RecallResult: + candidate_k: int = 50, reinforce: bool = False, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + diagnostics: bool = False, + arm_config: Optional[ProfileConfig] = None) -> RecallResult: flt = flt or SearchFilter() - now = flt.as_of if flt.as_of is not None else now_ts() + requested_historical = flt.historical + snapshot = now_ts() + effective_valid_at = ( + flt.valid_at if flt.valid_at is not None else snapshot + ) + effective_known_at = ( + flt.known_at if flt.known_at is not None else snapshot + ) + flt = replace( + flt, + as_of=effective_valid_at, + valid_at=effective_valid_at, + known_at=effective_known_at, + ) + now = effective_valid_at + budget = self.token_budget if token_budget is None else max(0, int(token_budget)) + requested_profile = str(retrieval_profile or "balanced").strip().casefold() + if requested_profile not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValueError(f"retrieval_profile must be one of: {choices}") + selected_profile = ( + self.retrieval_policy.profile(query) + if requested_profile == "auto" + else requested_profile + ) + # ``arm_config`` is a composition-time override for controlled offline + # ablations. Normal callers still use only named RetrievalPolicy profiles, + # so benchmark labels do not expand the public routing contract. + config = arm_config or profile_config(selected_profile) # ── arms ───────────────────────────────────────────────────────────── - qvec = self.embedder.embed([query])[0] - vec = dict(self.index.search(qvec, candidate_k, filter=flt)) # id -> cosine - lex = dict(self.store.fts_search(query, candidate_k, filter=flt)) # id -> lexical - graph = self._graph_arm(query, flt, now) # id -> weight + if config.vector: + qvec = self.embedder.embed([query])[0] + vec = dict(self.index.search(qvec, candidate_k, filter=flt)) + else: + vec = {} + lex = ( + dict(self.store.fts_search(query, candidate_k, filter=flt)) + if config.lexical else {} + ) + graph = self._graph_arm(query, flt, now, candidate_k=candidate_k) if config.graph else {} + code = ( + self._code_arm( + query, flt, candidate_k, historical=requested_historical + ) + if config.code else {} + ) # ── gather candidates and enforce visibility defensively ───────────── # Sorted, not raw set order: a set of ids iterates in hash order, which varies with # PYTHONHASHSEED, so equal-scored results used to come back in a different order in # every process. Sorting here (and on the final sort below) makes recall reproducible. # One batched lookup replaces ~150 single-row get_memory() calls per recall. - candidate_ids = sorted(set(vec) | set(lex) | set(graph)) + candidate_ids = sorted(set(vec) | set(lex) | set(graph) | set(code)) fetched = self.store.get_memories(candidate_ids) recs: dict[str, MemoryRecord] = {} for mid in candidate_ids: @@ -73,35 +141,132 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, if rec and memory_matches_filter(rec, flt, at=now): recs[mid] = rec if not recs: - return RecallResult() + context, packed, usage = self.context_packer.pack(query, [], budget) + return RecallResult( + context=context, + packed_chunks=packed, + usage=usage, + valid_at=flt.valid_at, + known_at=flt.known_at, + historical=requested_historical, + retrieval_profile=selected_profile, + retrieval_trace=[] if diagnostics else None, + token_counter=getattr(self.context_packer, "count_tokens", None), + ) sem_n = scoring.normalize({i: vec[i] for i in vec if i in recs}) lex_n = scoring.normalize({i: lex[i] for i in lex if i in recs}) grp_n = scoring.normalize({i: graph[i] for i in graph if i in recs}) + code_n = scoring.normalize({i: code[i] for i in code if i in recs}) rrf = scoring.reciprocal_rank_fusion([ - _ranked(vec, recs), _ranked(lex, recs), _ranked(graph, recs), + ranked for ranked in ( + _ranked(vec, recs), + _ranked(lex, recs), + _ranked(graph, recs), + _ranked(code, recs), + ) if ranked ]) # ── six-term weighted score (+ small RRF nudge for cross-arm agreement) ── scored: list[Candidate] = [] + score_details: dict[str, dict[str, Any]] = {} for mid, rec in recs.items(): w = self.weights.get(rec.mtype, scoring.Weights()) + adjusted_semantic = sem_n.get(mid, 0.0) * config.semantic_scale + adjusted_lexical = lex_n.get(mid, 0.0) * config.lexical_scale + adjusted_graph = ( + grp_n.get(mid, 0.0) * config.graph_scale + + (config.graph_presence_bonus if mid in graph else 0.0) + ) + adjusted_code = ( + code_n.get(mid, 0.0) * config.code_scale + + (config.code_presence_bonus if mid in code else 0.0) + ) + semantic_score = max(adjusted_semantic, adjusted_code) base = scoring.score_memory( rec, now=now, weights=w, - semantic=sem_n.get(mid, 0.0), lexical=lex_n.get(mid, 0.0), - graph=grp_n.get(mid, 0.0), recency_tau_days=self.recency_tau_days, + semantic=semantic_score, lexical=adjusted_lexical, + graph=adjusted_graph, recency_tau_days=self.recency_tau_days, + ) + arms = [ + name for name, values in ( + ("semantic", vec), + ("lexical", lex), + ("graph", graph), + ("code", code), + ) if mid in values + ] + fusion_score = base + 0.5 * rrf.get(mid, 0.0) + arm = ( + "code" if "code" in arms + else (arms[0] if len(arms) == 1 else ("hybrid" if arms else "fused")) ) - arm = "semantic" if mid in vec else ("lexical" if mid in lex else "graph") - scored.append(Candidate(id=mid, score=base + 0.5 * rrf.get(mid, 0.0), - arm=arm, record=rec)) + scored.append(Candidate( + id=mid, score=fusion_score, arm=arm, record=rec + )) + score_details[mid] = { + "raw": { + "semantic": vec.get(mid), + "lexical": lex.get(mid), + "graph": graph.get(mid), + "code": code.get(mid), + }, + "normalized": { + "semantic": sem_n.get(mid, 0.0), + "lexical": lex_n.get(mid, 0.0), + "graph": grp_n.get(mid, 0.0), + "code": code_n.get(mid, 0.0), + }, + "profile_adjusted": { + "semantic": adjusted_semantic, + "lexical": adjusted_lexical, + "graph": adjusted_graph, + "code": adjusted_code, + }, + "six_term_score": base, + "rrf_score": rrf.get(mid, 0.0), + "fusion_score": fusion_score, + "rerank_score": None, + "calibrated_score": fusion_score, + "arm_agreement": len(arms), + "arms": arms, + } # Tie-break on id so equal scores get a stable, process-independent order. scored.sort(key=lambda c: (-c.score, c.id)) # ── rerank top-N, keep k ───────────────────────────────────────────── pool = scored[: max(k * 4, k)] - final = self.reranker.rerank(query, pool, k) if self.reranker else pool[:k] - - if reinforce: + if self.reranker: + fused_before = {candidate.id: candidate.score for candidate in pool} + reranked = self.reranker.rerank(query, pool, k) + rerank_raw = { + candidate.id: float(candidate.score) for candidate in reranked + } + changed = any( + abs(rerank_raw[candidate.id] - fused_before.get(candidate.id, 0.0)) > 1e-12 + for candidate in reranked + ) + if changed: + fusion_norm = scoring.normalize({ + candidate.id: fused_before.get(candidate.id, 0.0) + for candidate in reranked + }) + rerank_norm = scoring.normalize(rerank_raw) + for candidate in reranked: + candidate.score = ( + 0.7 * fusion_norm.get(candidate.id, 0.0) + + 0.3 * rerank_norm.get(candidate.id, 0.0) + ) + reranked.sort(key=lambda candidate: (-candidate.score, candidate.id)) + final = reranked[:k] + for candidate in final: + detail = score_details[candidate.id] + detail["rerank_score"] = rerank_raw.get(candidate.id) + detail["calibrated_score"] = candidate.score + else: + final = pool[:k] + + if reinforce and not requested_historical: for c in final: self.store.reinforce(c.id, boost=scoring.INTERACTION_BOOST["recall"]) @@ -109,25 +274,241 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, "id": c.id, "title": c.record.title, "content": c.record.content, "scope": c.record.scope.value, "mtype": c.record.mtype.value, "repo_id": c.record.repo_id, "score": round(c.score, 4), "arm": c.arm, + "subject_key": c.record.subject_key, + "claim_kind": c.record.claim_kind, "retention": round(scoring.retention(c.record.stability, c.record.last_access, now), 4), "provenance": c.record.provenance, } for c in final] - return RecallResult(chunks=chunks, context=self._pack(final), count=len(final)) + context, packed_chunks, usage = self.context_packer.pack(query, final, budget) + trace = None + if diagnostics: + trace = [ + {"id": candidate.id, **score_details[candidate.id]} + for candidate in final + ] + return RecallResult( + chunks=chunks, + context=context, + count=len(final), + packed_chunks=packed_chunks, + usage=usage, + valid_at=flt.valid_at, + known_at=flt.known_at, + historical=requested_historical, + retrieval_profile=selected_profile, + retrieval_trace=trace, + token_counter=getattr(self.context_packer, "count_tokens", None), + ) # ── arms / helpers ──────────────────────────────────────────────────────── - def _graph_arm(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: - if self.graph_mode == "1hop": - return self._graph_arm_1hop(query, flt, now) - return self._graph_arm_ppr(query, flt, now) + def _code_arm( + self, + query: str, + flt: SearchFilter, + candidate_k: int, + *, + historical: Optional[bool] = None, + ) -> dict[str, float]: + """Bridge code-symbol matches to scoped memories with bounded work. + + The symbol graph remains optional: an unindexed repo simply contributes + no candidates. Query fan-out, matched symbols, graph edges, and linked + memories are all capped so code recall cannot degrade into a repository + scan. + """ + if not flt.repo_id: + return {} + identifiers = [] + seen_identifiers = set() + stop = { + "about", "called", "class", "code", "does", "file", "from", + "function", "into", "module", "that", "this", "what", "where", + "which", "with", + } + for value in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", query): + folded = value.casefold() + if folded in stop or folded in seen_identifiers: + continue + seen_identifiers.add(folded) + identifiers.append(value) + if len(identifiers) >= 8: + break + if not identifiers: + return {} + + symbols: dict[str, dict] = {} + symbol_strength: dict[str, float] = {} + per_term = max(2, min(12, candidate_k // max(1, len(identifiers)))) + for identifier in identifiers: + matches = _call_temporal_store( + self.store.search_symbols, + flt, + flt.repo_id, + identifier, + limit=per_term, + requested_historical=historical, + ) + for rank, symbol in enumerate(matches): + symbol_id = symbol.get("id") + if not symbol_id: + continue + exact = identifier.casefold() in { + str(symbol.get("name") or "").casefold(), + str(symbol.get("fqname") or "").casefold(), + } + strength = (1.0 if exact else 0.75) / (rank + 1) + symbols[symbol_id] = symbol + symbol_strength[symbol_id] = max( + symbol_strength.get(symbol_id, 0.0), strength + ) + if not symbols: + return {} - def _graph_arm_ppr(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: + aliases: dict[str, str] = {} + for symbol_id, symbol in symbols.items(): + for key in ("id", "name", "fqname"): + value = str(symbol.get(key) or "") + if value: + aliases[value] = symbol_id + # Expand one stored code edge to capture callers/callees, bounded by a + # multiple of candidate_k. Query only edges incident to matched aliases + # before applying that cap, so later files cannot be hidden by a global prefix. + edge_kwargs = { + "limit": max(100, min(2000, candidate_k * 20)), + "layers": flt.graph_layers, + } + # ``endpoints`` is a v2 Store optimization. Preserve compatibility with + # external code stores that have not added the optional filter yet. + try: + edge_parameters = inspect.signature(self.store.list_code_edges).parameters.values() + supports_endpoints = any( + parameter.name == "endpoints" + or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in edge_parameters + ) + except (TypeError, ValueError): + supports_endpoints = False + if supports_endpoints: + edge_kwargs["endpoints"] = list(aliases) + code_edges = _call_temporal_store( + self.store.list_code_edges, + flt, + flt.repo_id, + requested_historical=historical, + **edge_kwargs, + ) + related_names: dict[str, float] = {} + for edge in code_edges: + src, dst = str(edge.get("src") or ""), str(edge.get("dst") or "") + if src in aliases: + related_names[dst] = max( + related_names.get(dst, 0.0), + symbol_strength[aliases[src]] * 0.55, + ) + if dst in aliases: + related_names[src] = max( + related_names.get(src, 0.0), + symbol_strength[aliases[dst]] * 0.55, + ) + if related_names: + symbol_kwargs = { + "limit": max(100, min(2000, candidate_k * 20)), + } + # Like code edges, direct symbol resolution is an optional Store + # optimization. When it is available, apply it before the cap so + # a caller/callee in a later file is still eligible for recall. + try: + symbol_parameters = inspect.signature(self.store.list_symbols).parameters.values() + supports_identifiers = any( + parameter.name == "identifiers" + or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in symbol_parameters + ) + except (TypeError, ValueError): + supports_identifiers = False + if supports_identifiers: + symbol_kwargs["identifiers"] = list(related_names) + else: + # External legacy stores cannot filter this lookup. Do not + # reintroduce the incorrect global prefix cap for them. + symbol_kwargs["limit"] = None + all_symbols = _call_temporal_store( + self.store.list_symbols, + flt, + flt.repo_id, + requested_historical=historical, + **symbol_kwargs, + ) + for symbol in all_symbols: + matched_strength = max( + ( + related_names.get(str(symbol.get(key) or ""), 0.0) + for key in ("id", "name", "fqname") + ), + default=0.0, + ) + symbol_id = symbol.get("id") + if matched_strength > 0.0 and symbol_id: + symbols[symbol_id] = symbol + symbol_strength[symbol_id] = max( + symbol_strength.get(symbol_id, 0.0), matched_strength + ) + + selected_symbol_ids = sorted( + symbols, + key=lambda value: (-symbol_strength.get(value, 0.0), value), + )[:max(10, min(100, candidate_k * 2))] + rows_by_symbol = _call_temporal_store( + self.store.memories_for_symbols, + flt, + flt.repo_id, + selected_symbol_ids, + limit=max(2, min(10, candidate_k)), + requested_historical=historical, + ) + out: dict[str, float] = {} + for symbol_id in selected_symbol_ids: + rows = rows_by_symbol.get(symbol_id, []) + for rank, row in enumerate(rows): + memory_id = row.get("id") + if not memory_id: + continue + confidence = max(0.0, min(1.0, float(row.get("confidence") or 0.0))) + score = symbol_strength[symbol_id] * confidence / (rank + 1) + out[memory_id] = max(out.get(memory_id, 0.0), score) + return dict( + sorted(out.items(), key=lambda item: (-item[1], item[0]))[:candidate_k] + ) + + def _graph_arm( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: + if flt.graph_layers is not None and not flt.graph_layers: + return {} + if self.graph_mode == "1hop": + return self._graph_arm_1hop(query, flt, now, candidate_k=candidate_k) + return self._graph_arm_ppr(query, flt, now, candidate_k=candidate_k) + + def _graph_arm_ppr( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: """Personalized PageRank arm: build the scoped entity/memory graph — entity↔entity edges (bi-temporal), memory↔entity mentions, memory↔memory links — seed at the query's entities, and rank memories by walk probability. Multi-hop associations surface without expanding an explicit hop count; entity nodes are prefixed so names can never collide with memory ids.""" - entity_map = self._entity_map(flt) + entity_map = self._seed_entity_map(query, flt) patterns = { eid: (name.casefold(), _entity_pattern(name)) for eid, name in entity_map.items() @@ -149,37 +530,108 @@ def connect(a: str, b: str, w: float) -> None: adj.setdefault(a, []).append((b, w)) adj.setdefault(b, []).append((a, w)) - for e in self.store.edges_in_scope(flt, at=now): + # Build a bounded edge set outward from the query entities. A global + # ULID-ordered cap would let old unrelated edges crowd out a new relation + # required by this query before PPR sees it. + edge_cap = 4000 + edges_by_id = {} + frontier = set(seeds) + expanded: set[str] = set() + while frontier and len(edges_by_id) < edge_cap: + batch = sorted(frontier - expanded)[:400] + if not batch: + break + frontier.difference_update(batch) + expanded.update(batch) + next_frontier: set[str] = set() + for edge in self.store.neighbors( + batch, at=now, layers=flt.graph_layers, flt=flt, + limit=edge_cap - len(edges_by_id)): + if edge.id in edges_by_id: + continue + edges_by_id[edge.id] = edge + next_frontier.update((edge.src, edge.dst)) + if len(edges_by_id) >= edge_cap: + break + frontier.update(next_frontier - expanded) + for e in edges_by_id.values(): connect(ent(e.src), ent(e.dst), max(float(e.weight or 1.0), 1e-6)) - # Past this cap, PPR would be rejected below anyway. Fall back before - # scanning every memory against every entity and then repeating that - # work in the 1-hop arm. - if len(adj) > 4000: - return self._graph_arm_1hop(query, flt, now) - - recs = self.store.list_memories(flt, limit=500) - for rec in recs: - hay = f"{rec.title} {rec.content}" - hay_folded = hay.casefold() - for eid, (needle, pattern) in patterns.items(): - # Most entity names are absent. The C-level substring guard avoids - # millions of comparatively expensive regex searches while the - # regex retains exact token-boundary semantics for actual matches. - if needle in hay_folded and pattern.search(hay): - connect(rec.id, ent(eid), 1.0) + # Query only the entity frontier before applying the incidence cap. A + # global confidence/ID prefix can otherwise omit a memory attached to a + # seeded or reached entity in a large scope. + incidence_entity_ids = sorted({ + *seeds, + *(endpoint for edge in edges_by_id.values() for endpoint in (edge.src, edge.dst)), + }) + incidence = self.store.list_memory_entities( + flt, entity_ids=incidence_entity_ids, limit=12_000, + ) + # Links are graph evidence in their own right. Restricting their endpoints + # to incidence rows silently drops a linked memory which has no entity + # mention, even when its peer is reachable from a seeded entity. Use the + # same bounded, scoped, bi-temporally visible memory universe as the other + # retrieval arms so PPR can traverse that edge without widening scope. Keep + # the incidence frontier as well when independent caps choose a different + # subset of the scoped memory universe. + incidence_memory_ids = { + str(row.get("memory_id") or "") + for row in incidence if row.get("memory_id") + } + frontier_links = self.store.links_touching( + sorted(incidence_memory_ids), + layers=flt.graph_layers, + flt=flt, + limit=20_000, + ) + # Expand from the entity-incidence frontier before adding the bounded newest + # memory window. An older unmentioned endpoint can then participate in PPR + # through its visible link instead of being silently dropped by that window. + memory_ids = sorted(incidence_memory_ids | { + endpoint + for link in frontier_links + for endpoint in (link["a"], link["b"]) + } | { + memory.id for memory in self.store.list_memories(flt, limit=12_000) + }) + incidence_strength: dict[tuple[str, str], float] = {} + for row in incidence: + memory_id = str(row.get("memory_id") or "") + entity_id = str(row.get("entity_id") or "") + if memory_id and entity_id: + key = (memory_id, entity_id) + incidence_strength[key] = max( + incidence_strength.get(key, 0.0), + max(float(row.get("confidence") or 0.0), 1e-6), + ) + for (memory_id, entity_id), confidence in incidence_strength.items(): + connect(memory_id, ent(entity_id), confidence) for link in self.store.links_among( - [r.id for r in recs], layers=flt.graph_layers + memory_ids, + layers=flt.graph_layers, + flt=flt, + limit=20_000, ): connect(link["a"], link["b"], 1.0) ranked = personalized_pagerank(adj, [ent(eid) for eid in seeds]) - return {nid: score for nid, score in ranked.items() - if not nid.startswith("ent::") and score > 0.0} - - def _graph_arm_1hop(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: - entity_map = self._entity_map(flt) + memory_scores = [ + (nid, score) for nid, score in ranked.items() + if not nid.startswith("ent::") and score > 0.0 + ] + memory_scores.sort(key=lambda item: (-item[1], item[0])) + return dict(memory_scores[:max(0, int(candidate_k))]) + + def _graph_arm_1hop( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: + entity_map = self._seed_entity_map(query, flt) patterns = { eid: (name.casefold(), _entity_pattern(name)) for eid, name in entity_map.items() @@ -193,31 +645,42 @@ def _graph_arm_1hop(self, query: str, flt: SearchFilter, now: float) -> dict[str ] if not seed_ids: return {} - names = {entity_map[eid] for eid in seed_ids if entity_map.get(eid)} - for e in self.store.neighbors(seed_ids, at=now, layers=flt.graph_layers): - if e.src in entity_map: - names.add(entity_map[e.src]) - if e.dst in entity_map: - names.add(entity_map[e.dst]) + related_ids = set(seed_ids) + for edge in self.store.neighbors( + seed_ids, at=now, layers=flt.graph_layers, flt=flt + ): + related_ids.add(edge.src) + related_ids.add(edge.dst) + rows = self.store.list_memory_entities( + flt, entity_ids=sorted(related_ids), limit=12_000 + ) out: dict[str, float] = {} - name_patterns = [ - (name.casefold(), _entity_pattern(name)) - for name in names - if name - ] - for rec in self.store.list_memories(flt, limit=500): - hay = f"{rec.title} {rec.content}" - hay_folded = hay.casefold() - hits = sum( - 1 - for needle, pattern in name_patterns - if needle in hay_folded and pattern.search(hay) - ) - if hits: - out[rec.id] = float(hits) - return out - - def _entity_map(self, flt: SearchFilter) -> dict[str, str]: + if rows: + for row in rows: + memory_id = str(row.get("memory_id") or "") + if memory_id: + out[memory_id] = ( + out.get(memory_id, 0.0) + + max(0.0, float(row.get("confidence") or 0.0)) + ) + return dict(sorted( + out.items(), key=lambda item: (-item[1], item[0]) + )[:max(0, int(candidate_k))]) + + return dict(sorted( + out.items(), key=lambda item: (-item[1], item[0]) + )[:max(0, int(candidate_k))]) + + def _seed_entity_map( + self, query: str, flt: SearchFilter, *, limit: int = 2048, + ) -> dict[str, str]: + """Return a bounded, scoped set of entity names that may occur in ``query``.""" + terms = sorted({ + term.casefold() for term in re.findall(r"[\w@#.+-]+", query) + if len(term) >= 2 + })[:16] + if not terms: + return {} sql = "SELECT DISTINCT id, name FROM entities" clauses, params = [], [] if flt.workspace_id: @@ -235,23 +698,53 @@ def _entity_map(self, flt: SearchFilter) -> dict[str, str]: else: clauses.append("repo_id=?") params.append(flt.repo_id) + clauses.append( + "(" + " OR ".join("instr(lower(name), ?) > 0" for _ in terms) + ")" + ) + params.extend(terms) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY id LIMIT ?" + params.append(max(0, int(limit))) + return { + r["id"]: r["name"] + for r in self.store.conn.execute(sql, params).fetchall() + } + + def _entity_map(self, flt: SearchFilter, *, limit: int = 2048) -> dict[str, str]: + """Compatibility view of scoped entities without restoring unbounded recall scans. + + The retrieval pipeline uses :meth:`_seed_entity_map` so graph seeding remains + query-directed. Older integrations and scope-invariant tests exercised this private + helper directly, so retain its original semantics behind an explicit safety bound. + """ + sql = "SELECT DISTINCT id, name FROM entities" + clauses, params = [], [] + if flt.workspace_id: + if flt.include_ancestors: + clauses.append("(workspace_id=? OR workspace_id IS NULL)") + else: + clauses.append("workspace_id=?") + params.append(flt.workspace_id) + if flt.repo_id: + if flt.include_ancestors: + clauses.append("(repo_id=? OR repo_id IS NULL)") + else: + clauses.append("repo_id=?") + params.append(flt.repo_id) if clauses: sql += " WHERE " + " AND ".join(clauses) - return {r["id"]: r["name"] for r in self.store.conn.execute(sql, params).fetchall()} + sql += " ORDER BY id LIMIT ?" + params.append(max(0, int(limit))) + return { + row["id"]: row["name"] + for row in self.store.conn.execute(sql, params).fetchall() + } def _pack(self, cands: list[Candidate]) -> str: - parts, used = [], 0 - for c in cands: - r = c.record - header = f"[{r.scope.value}:{r.repo_id or '-'}]" - if r.title: - header += f" {r.title}" - block = f"{header}\n{r.summary or r.content}" - used += len(block) // 4 - if used > self.token_budget and parts: - break - parts.append(block) - return "\n\n".join(parts) + """Compatibility helper for callers that exercised the old private method.""" + context, _, _ = self.context_packer.pack("", cands, self.token_budget) + return context def _entity_pattern(name: str) -> re.Pattern[str]: @@ -263,3 +756,34 @@ def _ranked(arm: dict[str, float], recs: dict) -> list[str]: # Tie-break on id: RRF depends on rank position, so equal arm scores must not order # differently between runs (they feed the final score). return [i for i, _ in sorted(arm.items(), key=lambda x: (-x[1], x[0])) if i in recs] + + +def _call_temporal_store( + method, + flt: SearchFilter, + *args, + requested_historical: Optional[bool] = None, + **kwargs, +): + """Call an optional code-store extension without masking implementation bugs. + + Older third-party stores may not expose the v5 ``flt`` keyword. Current reads can + retain their legacy behavior, but historical reads must fail closed: retrying a + method without the filter would silently substitute present-day code evidence. + Signature inspection distinguishes an unsupported keyword from a genuine + ``TypeError`` raised inside the implementation, which is allowed to propagate. + """ + try: + parameters = inspect.signature(method).parameters.values() + supports_filter = any( + parameter.name == "flt" + or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + supports_filter = False + if supports_filter: + return method(*args, flt=flt, **kwargs) + if flt.historical if requested_historical is None else requested_historical: + return [] + return method(*args, **kwargs) diff --git a/engraphis/core/resolve.py b/engraphis/core/resolve.py index 5d8afe96..7b818f23 100644 --- a/engraphis/core/resolve.py +++ b/engraphis/core/resolve.py @@ -39,12 +39,18 @@ # the new fact and reinforce the stale one, while a wrongly-INVALIDATE'd restatement just # refreshes the phrasing and keeps the old version readable in history. PARAPHRASE_EMBED_SIM = 0.90 +# Supersession without an explicit claim key is intentionally stricter than a +# generic "related" judgment. Both independent signals must agree before a +# write hides a currently-live fact from ordinary recall. +STRONG_SUBJECT_TOKEN_JACCARD = 0.55 +STRONG_JOINT_EMBED_SIM = 0.45 class ResolutionOp(str, Enum): ADD = "add" # genuinely new -> insert NOOP = "noop" # already known -> reinforce the existing memory, don't insert INVALIDATE = "invalidate" # same subject, new content -> close old, insert new + RELATE = "relate" # retain both facts and persist a semantic relation @dataclass(frozen=True) @@ -54,7 +60,9 @@ class Resolution: reason: str = "" -def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> Resolution: +def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]], *, + subject_key: str = "", claim_kind: str = "", + candidate_content: Optional[str] = None) -> Resolution: """Decide ADD / NOOP / INVALIDATE for new content against its nearest neighbors. ``neighbors`` are ``(embedding_similarity, MemoryRecord)`` pairs that the caller has @@ -65,11 +73,29 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> the embedding cosine as a second signal for paraphrased restatements/contradictions. """ cand_tokens = tokenize(candidate_text) - best: Optional[tuple[float, MemoryRecord, float]] = None # (overlap, rec, sim) - best_sim: Optional[tuple[float, MemoryRecord]] = None # highest-cosine neighbor + candidate_subject = str(subject_key or "").strip() + candidate_kind = str(claim_kind or "").strip() + exact_claim_neighbors: list[tuple[float, MemoryRecord]] = [] + fallback_neighbors: list[tuple[float, MemoryRecord]] = [] for sim, rec in neighbors: + record_subject = str(rec.subject_key or "").strip() + record_kind = str(rec.claim_kind or "").strip() + # Explicit claim identities outrank similarity. Two keyed records that + # disagree on subject or predicate cannot be duplicate/supersession + # candidates merely because their prose happens to be similar. + if candidate_subject and record_subject: + if candidate_subject != record_subject or candidate_kind != record_kind: + continue + exact_claim_neighbors.append((sim, rec)) + continue if sim < RELATED_SIM_FLOOR: continue + fallback_neighbors.append((sim, rec)) + + considered = exact_claim_neighbors or fallback_neighbors + best: Optional[tuple[float, MemoryRecord, float]] = None # (overlap, rec, sim) + best_sim: Optional[tuple[float, MemoryRecord]] = None # highest-cosine neighbor + for sim, rec in considered: overlap = jaccard(cand_tokens, tokenize(f"{rec.title} {rec.content}")) if best is None or overlap > best[0]: best = (overlap, rec, sim) @@ -80,20 +106,51 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> return Resolution(ResolutionOp.ADD, reason="no related memory in scope") overlap, rec, sim = best + same_subject = bool(candidate_subject) and candidate_subject == ( + str(rec.subject_key or "").strip() + ) + same_claim = same_subject and candidate_kind == str(rec.claim_kind or "").strip() + if same_claim: + # Candidate embeddings and overlap include a display title, but durable + # claim equality is about the stored content. Comparing title+content to + # content would turn an identical titled write into a false supersession. + duplicate_text = candidate_content if candidate_content is not None else candidate_text + candidate_normalized = " ".join(duplicate_text.split()).casefold() + record_normalized = " ".join(rec.content.split()).casefold() + if candidate_normalized == record_normalized: + return Resolution( + ResolutionOp.NOOP, + target_id=rec.id, + reason=f"exact duplicate of keyed claim {rec.id}", + ) + return Resolution(ResolutionOp.INVALIDATE, target_id=rec.id, + reason=f"supersedes {rec.id} (shared claim key, " + f"token overlap={overlap:.2f}, similarity={sim:.2f})") if overlap >= DUP_TOKEN_JACCARD: + if candidate_subject: + return Resolution( + ResolutionOp.INVALIDATE, + target_id=rec.id, + reason=f"replaces unkeyed duplicate {rec.id} with durable claim identity " + f"(token overlap={overlap:.2f})", + ) return Resolution(ResolutionOp.NOOP, target_id=rec.id, reason=f"near-duplicate of {rec.id} (token overlap={overlap:.2f})") - if overlap >= SUBJECT_TOKEN_JACCARD: + # Without an explicit claim key, invalidation needs strong agreement from + # lexical and semantic signals. A high cosine alone can be a topical + # paraphrase rather than a contradiction, so it becomes a relation instead. + if overlap >= STRONG_SUBJECT_TOKEN_JACCARD and sim >= STRONG_JOINT_EMBED_SIM: return Resolution(ResolutionOp.INVALIDATE, target_id=rec.id, - reason=f"supersedes {rec.id} (same subject, " + reason=f"supersedes {rec.id} (strong joint evidence: " f"token overlap={overlap:.2f}, similarity={sim:.2f})") - # Token overlap says "distinct", but a high-enough embedding cosine says "same fact - # in different words" — the paraphrase case token Jaccard cannot see (the known - # ceiling this second signal exists to close). if best_sim is not None and best_sim[0] >= PARAPHRASE_EMBED_SIM: psim, prec = best_sim povl = jaccard(cand_tokens, tokenize(f"{prec.title} {prec.content}")) - return Resolution(ResolutionOp.INVALIDATE, target_id=prec.id, - reason=f"supersedes {prec.id} (paraphrase: cosine={psim:.2f}, " + return Resolution(ResolutionOp.RELATE, target_id=prec.id, + reason=f"related to {prec.id} (paraphrase-like: cosine={psim:.2f}, " f"token overlap={povl:.2f})") + if overlap >= SUBJECT_TOKEN_JACCARD: + return Resolution(ResolutionOp.RELATE, target_id=rec.id, + reason=f"related to {rec.id} (same topic, " + f"token overlap={overlap:.2f}, similarity={sim:.2f})") return Resolution(ResolutionOp.ADD, reason=f"related but distinct (best overlap={overlap:.2f})") diff --git a/engraphis/core/retrieval_policy.py b/engraphis/core/retrieval_policy.py new file mode 100644 index 00000000..dbf93750 --- /dev/null +++ b/engraphis/core/retrieval_policy.py @@ -0,0 +1,92 @@ +"""Deterministic retrieval-profile selection. + +``balanced`` preserves the established hybrid path. ``auto`` is explicit and +conservative: it only selects a specialized profile when the query has a strong, +locally-observable signal. This keeps automatic routing measurable and prevents +an unbenchmarked policy change from silently altering existing callers. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + + +RETRIEVAL_PROFILES = frozenset({"balanced", "auto", "lexical", "graph", "code"}) + +_CODE_RE = re.compile( + r"(?:\w+[./\\])+\w+|::|->|\b(?:class|def|function|import|module)\b|" + r"\b[A-Za-z_]\w*\([^)]*\)", + re.IGNORECASE, +) +_GRAPH_RE = re.compile( + r"\b(?:calls?|causes?|depends?|impact|path|related|relationship|why)\b", + re.IGNORECASE, +) +_LEXICAL_RE = re.compile( + r"\"[^\"]+\"|'[^']+'|\b[A-Z][A-Z0-9_]{2,}\b|" + r"\b(?:exact|identifier|literal|named|spelled)\b", +) + + +@dataclass(frozen=True) +class ProfileConfig: + name: str + vector: bool + lexical: bool + graph: bool + code: bool + semantic_scale: float = 1.0 + lexical_scale: float = 1.0 + graph_scale: float = 1.0 + code_scale: float = 1.0 + graph_presence_bonus: float = 0.0 + code_presence_bonus: float = 0.0 + + +_CONFIGS = { + "balanced": ProfileConfig("balanced", True, True, True, False), + "lexical": ProfileConfig("lexical", False, True, False, False), + # Specialized profiles retain supporting arms but make their declared + # evidence type decisive. ``balanced`` stays byte-for-byte equivalent to + # the established scoring behavior, and ``auto`` remains opt-in. + "graph": ProfileConfig( + "graph", True, True, True, False, + graph_scale=3.0, graph_presence_bonus=1.5, + ), + "code": ProfileConfig( + "code", True, True, True, True, + code_scale=3.0, code_presence_bonus=1.5, + ), +} + + +def profile_config(name: str) -> ProfileConfig: + """Return a validated immutable configuration for a concrete profile.""" + normalized = str(name or "").strip().casefold() + if normalized not in _CONFIGS: + choices = ", ".join(sorted(_CONFIGS)) + raise ValueError(f"retrieval profile must resolve to one of: {choices}") + return _CONFIGS[normalized] + + +class DeterministicRetrievalPolicy: + """Offline automatic profile selector with stable, inspectable rules.""" + + identity = "engraphis.deterministic.v1" + + def profile(self, query: str) -> str: + if _CODE_RE.search(query or ""): + return "code" + if _GRAPH_RE.search(query or ""): + return "graph" + if _LEXICAL_RE.search(query or ""): + return "lexical" + return "balanced" + + def resolve(self, requested: str, query: str) -> ProfileConfig: + normalized = str(requested or "balanced").strip().casefold() + if normalized not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValueError(f"retrieval_profile must be one of: {choices}") + selected = self.profile(query) if normalized == "auto" else normalized + return profile_config(selected) diff --git a/engraphis/core/schema.py b/engraphis/core/schema.py index 140cd849..5c69b7c3 100644 --- a/engraphis/core/schema.py +++ b/engraphis/core/schema.py @@ -8,7 +8,7 @@ """ from __future__ import annotations -SCHEMA_VERSION = 4 +SCHEMA_VERSION = 6 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -73,8 +73,11 @@ last_access REAL, valid_from REAL, -- world-time validity valid_to REAL, + valid_to_recorded_at REAL, -- system-time when valid_to was learned ingested_at REAL, -- system-time validity expired_at REAL, + subject_key TEXT DEFAULT '', -- stable claim subject, optional + claim_kind TEXT DEFAULT '', -- optional claim predicate/category pinned INTEGER DEFAULT 0, sensitivity TEXT DEFAULT 'normal', provenance TEXT DEFAULT '{}', @@ -84,6 +87,31 @@ CREATE INDEX IF NOT EXISTS idx_mem_session ON memories(session_id); CREATE INDEX IF NOT EXISTS idx_mem_valid ON memories(valid_from, valid_to, expired_at); +-- Persisted memory↔entity incidence lets graph retrieval attach evidence without +-- rescanning every memory's prose. Temporal fields preserve historical walks. +CREATE TABLE IF NOT EXISTS memory_entities ( + id TEXT PRIMARY KEY, + memory_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + workspace_id TEXT, + repo_id TEXT, + source_kind TEXT NOT NULL DEFAULT 'edge_support', + confidence REAL NOT NULL DEFAULT 1.0, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL, + provenance TEXT DEFAULT '{}' +); +CREATE INDEX IF NOT EXISTS idx_memory_entity_entity + ON memory_entities(workspace_id, repo_id, entity_id, valid_to, expired_at); +CREATE INDEX IF NOT EXISTS idx_memory_entity_memory + ON memory_entities(memory_id, valid_to, expired_at); +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_live_unique + ON memory_entities(memory_id, entity_id, source_kind) + WHERE valid_to IS NULL AND expired_at IS NULL; + -- Vectors (Phase 0 reference store; Phase 1 → sqlite-vec vec0 virtual table). CREATE TABLE IF NOT EXISTS mem_vectors ( id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE, @@ -118,6 +146,7 @@ weight REAL DEFAULT 1.0, valid_from REAL, valid_to REAL, + valid_to_recorded_at REAL, ingested_at REAL, expired_at REAL, provenance TEXT DEFAULT '{}' @@ -141,6 +170,7 @@ confidence REAL NOT NULL DEFAULT 0.5, valid_from REAL, valid_to REAL, + valid_to_recorded_at REAL, ingested_at REAL, expired_at REAL, provenance TEXT DEFAULT '{}', @@ -262,7 +292,14 @@ relation TEXT, layer TEXT DEFAULT 'semantic', reason TEXT DEFAULT '', - created_at REAL + created_at REAL, + -- Direct memory relationships participate in graph recall. Give them the + -- same history as other graph bridges so later links cannot alter past reads. + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_mem_links_ab ON mem_links(a, b); -- Links are undirected: Store.get_links()/has_link()/add_link() all match "a=? OR b=?". @@ -284,7 +321,12 @@ exported INTEGER, content_hash TEXT, embedding_ref TEXT, - updated_at REAL + updated_at REAL, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_sym_repo ON symbols(repo_id, name); @@ -296,7 +338,12 @@ relation TEXT, -- calls|imports|references|implements|tests layer TEXT DEFAULT 'entity', file TEXT, - line INTEGER + line INTEGER, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_code_edge_src ON code_edges(repo_id, src); CREATE INDEX IF NOT EXISTS idx_code_edge_dst ON code_edges(repo_id, dst); @@ -314,6 +361,31 @@ ); CREATE INDEX IF NOT EXISTS idx_code_files_lang ON code_files(repo_id, lang); +-- ``code_files`` is the current indexing manifest. Historical code exports use this +-- append-only companion so a deleted or replaced file remains visible at the correct +-- world/system-time anchors alongside its retired symbols and code edges. +CREATE TABLE IF NOT EXISTS code_file_history ( + version INTEGER PRIMARY KEY, + repo_id TEXT NOT NULL, + file TEXT NOT NULL, + lang TEXT, + content_hash TEXT NOT NULL, + size_bytes INTEGER DEFAULT 0, + mtime_ns INTEGER DEFAULT 0, + backend TEXT DEFAULT '', + indexed_at REAL, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL +); +CREATE INDEX IF NOT EXISTS idx_code_file_history_temporal + ON code_file_history(repo_id, file, valid_to, expired_at); +CREATE UNIQUE INDEX IF NOT EXISTS idx_code_file_history_live + ON code_file_history(repo_id, file) + WHERE valid_to IS NULL AND expired_at IS NULL; + CREATE TABLE IF NOT EXISTS code_memory_links ( id TEXT PRIMARY KEY, repo_id TEXT NOT NULL, @@ -322,7 +394,11 @@ relation TEXT DEFAULT 'mentions', confidence REAL DEFAULT 1.0, created_at REAL, - UNIQUE(repo_id, symbol_id, memory_id, relation) + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_code_mem_symbol ON code_memory_links(repo_id, symbol_id); @@ -363,6 +439,7 @@ operation TEXT NOT NULL, workspace_id TEXT, repo_id TEXT, + sequence INTEGER NOT NULL CHECK(sequence >= 1), scope_digest TEXT NOT NULL, actor TEXT DEFAULT 'system', target_count INTEGER DEFAULT 0, diff --git a/engraphis/core/store.py b/engraphis/core/store.py index d868569d..ffd377ae 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -12,6 +12,7 @@ import hashlib import json +import math import os import re import sqlite3 @@ -189,6 +190,28 @@ def _edge_support_confidence(provenance: Any, source_kind: str) -> float: return _SUPPORT_CONFIDENCE.get(source_kind, 0.50) +_PUBLIC_RECEIPT_LABELS_BY_KEY = { + "mtype": {"working", "episodic", "semantic", "procedural"}, + "scope": {"session", "repo", "workspace", "user"}, + "resolution": {"add", "noop", "invalidate", "relate"}, + "retention": { + "ephemeral", "normal", "critical", "short", "standard", "long", "permanent", + }, + "intent": { + "recall", "recall_context", "grounded", "http_read_only", + "explain", "timeline", "code", "locate_code", + }, + "relation": { + "related", "mentions", "supports", "supersedes", "consolidates", + "promotes", "causes", "depends_on", "calls", "imports", "references", + "implements", "tests", "uses", "owned_by", "co_occurs", + }, + "layer": {"temporal", "entity", "causal", "semantic"}, + "retrieval_profile": {"balanced", "auto", "lexical", "graph", "code"}, + "response_mode": {"full", "compact"}, +} + + def _receipt_metadata(metadata: dict) -> dict: """Keep receipt metadata useful but content-free and bounded.""" allowed = { @@ -197,27 +220,210 @@ def _receipt_metadata(metadata: dict) -> dict: "files_scanned", "files_indexed", "files_removed", "symbols", "edges", "entities", "relations", "tables", "dry_run", "error_count", "entities_added", "relations_added", + "retrieval_profile", "response_mode", "historical", "token_usage", } + def content_free_label(key: str, value: str) -> str: + normalized = value.strip().casefold().replace(" ", "_") + if normalized in _PUBLIC_RECEIPT_LABELS_BY_KEY.get(key, set()): + return normalized + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + out: dict[str, Any] = {} for key in sorted(metadata, key=lambda item: str(item))[:24]: safe_key = str(key)[:64] if safe_key not in allowed: continue value = metadata[key] - if isinstance(value, bool) or value is None: + if safe_key == "token_usage": + if not isinstance(value, dict): + continue + numeric = { + name: value[name] + for name in ( + "budget_tokens", "context_tokens", "source_tokens", "saved_tokens", + "savings_ratio", "packed_count", "omitted_count", + ) + if type(value.get(name)) in (int, float) + and math.isfinite(float(value[name])) + } + counter = value.get("token_counter") + if isinstance(counter, str): + if counter in {"engraphis.regex.v1", "estimate_tokens"}: + numeric["token_counter"] = counter + else: + numeric["token_counter"] = ( + "sha256:" + hashlib.sha256(counter.encode("utf-8")).hexdigest() + ) + out[safe_key] = numeric + elif isinstance(value, bool) or value is None: out[safe_key] = value elif isinstance(value, (int, float)): - out[safe_key] = value + if math.isfinite(float(value)): + out[safe_key] = value elif isinstance(value, str): - out[safe_key] = ( - value[:80] if len(value) <= 80 - else "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() - ) + out[safe_key] = content_free_label(safe_key, value) elif isinstance(value, (list, tuple)): out[safe_key] = len(value) return out +_PUBLIC_RECEIPT_ID = re.compile(r"^rcpt_[0-9ABCDEFGHJKMNPQRSTVWXYZ]{26}$") +_PUBLIC_RECEIPT_HASH = re.compile(r"^[0-9a-f]{64}$") +_PUBLIC_RECEIPT_HASHED_LABEL = re.compile(r"^sha256:[0-9a-f]{64}$") +_PUBLIC_RECEIPT_KEYS = { + "version", "id", "ts_ms", "operation", "scope_digest", "actor_digest", + "target_count", "status", "metadata", "prev_hash", +} +_PUBLIC_RECEIPT_METADATA_KEYS = { + "mtype", "scope", "resolution", "retention", "extracted", "intent", "k", + "result_count", "grounded", "citations", "relation", "layer", "graph_layers", + "files_scanned", "files_indexed", "files_removed", "symbols", "edges", + "entities", "relations", "tables", "dry_run", "error_count", + "entities_added", "relations_added", "retrieval_profile", "response_mode", + "historical", "token_usage", +} +_PUBLIC_RECEIPT_OPERATIONS = { + "remember", "recall", "promote", "link", "index_repo", + "graph_index", "grounded_recall", "consolidate", "sync", +} +_PUBLIC_RECEIPT_STATUSES = { + "ok", "add", "noop", "invalidate", "relate", "ingested", + "postgres_schema", "grounded", "abstained", "promoted", + "indexed", "skipped", "error", "failed", "cancelled", "partial", +} + + +def _redacted_receipt_value(value: Any) -> str: + raw = value if isinstance(value, str) else str(value or "") + return "redacted_sha256:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _public_receipt_row(row: dict) -> dict: + """Return one validated content-free receipt or a hash-only corruption marker.""" + raw = row.get("payload") + raw = raw if isinstance(raw, str) else str(raw or "") + raw_id = row.get("id") + raw_prev = row.get("prev_hash") + raw_hash = row.get("receipt_hash") + + def safe_id() -> str: + value = raw_id if isinstance(raw_id, str) else str(raw_id or "") + return value if _PUBLIC_RECEIPT_ID.fullmatch(value) else _redacted_receipt_value(value) + + def safe_hash(value: Any, *, allow_empty: bool = False) -> str: + text = value if isinstance(value, str) else str(value or "") + if allow_empty and not text: + return "" + return ( + text if _PUBLIC_RECEIPT_HASH.fullmatch(text) + else _redacted_receipt_value(text) + ) + + invalid = { + "id": safe_id(), + "prev_hash": safe_hash(raw_prev, allow_empty=True), + "hash": safe_hash(raw_hash), + "invalid_payload": True, + "payload_bytes": len(raw.encode("utf-8")), + "payload_sha256": hashlib.sha256(raw.encode("utf-8")).hexdigest(), + } + try: + payload = json.loads(raw) + except (TypeError, ValueError, RecursionError): + return invalid + if ( + not isinstance(payload, dict) + or set(payload) != _PUBLIC_RECEIPT_KEYS + or payload.get("version") != 1 + or payload.get("id") != raw_id + or payload.get("prev_hash") != raw_prev + or not isinstance(raw_id, str) + or _PUBLIC_RECEIPT_ID.fullmatch(raw_id) is None + or ( + raw_prev != "" + and ( + not isinstance(raw_prev, str) + or _PUBLIC_RECEIPT_HASH.fullmatch(raw_prev) is None + ) + ) + or not isinstance(raw_hash, str) + or _PUBLIC_RECEIPT_HASH.fullmatch(raw_hash) is None + or hashlib.sha256(raw.encode("utf-8")).hexdigest() != raw_hash + ): + return invalid + if type(payload.get("ts_ms")) is not int or payload["ts_ms"] < 0: + return invalid + if type(payload.get("target_count")) is not int or payload["target_count"] < 0: + return invalid + operation = payload.get("operation") + if not ( + operation in _PUBLIC_RECEIPT_OPERATIONS + or ( + isinstance(operation, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(operation) + ) + ): + return invalid + status = payload.get("status") + if not ( + status in _PUBLIC_RECEIPT_STATUSES + or ( + isinstance(status, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(status) + ) + ): + return invalid + if not ( + isinstance(payload.get("scope_digest"), str) + and re.fullmatch(r"[0-9a-f]{24}", payload["scope_digest"]) + and isinstance(payload.get("actor_digest"), str) + and re.fullmatch(r"[0-9a-f]{16}", payload["actor_digest"]) + ): + return invalid + metadata = payload.get("metadata") + if not isinstance(metadata, dict) or not set(metadata).issubset( + _PUBLIC_RECEIPT_METADATA_KEYS + ): + return invalid + for key, value in metadata.items(): + if key == "token_usage": + if not isinstance(value, dict): + return invalid + allowed_usage = { + "budget_tokens", "context_tokens", "source_tokens", "saved_tokens", + "savings_ratio", "packed_count", "omitted_count", "token_counter", + } + if not set(value).issubset(allowed_usage): + return invalid + for usage_key, usage_value in value.items(): + if usage_key == "token_counter": + if not ( + usage_value in {"engraphis.regex.v1", "estimate_tokens"} + or ( + isinstance(usage_value, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(usage_value) + ) + ): + return invalid + elif ( + type(usage_value) not in (int, float) + or not math.isfinite(float(usage_value)) + ): + return invalid + elif isinstance(value, str): + public_labels = _PUBLIC_RECEIPT_LABELS_BY_KEY.get(key, set()) + if not ( + value in public_labels + or _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(value) + ): + return invalid + elif isinstance(value, bool) or value is None: + continue + elif type(value) not in (int, float) or not math.isfinite(float(value)): + return invalid + return {**payload, "hash": raw_hash} + + def _fts5_available(conn: sqlite3.Connection) -> bool: try: conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_probe USING fts5(x)") @@ -227,6 +433,39 @@ def _fts5_available(conn: sqlite3.Connection) -> bool: return False +def _temporal_anchors(flt: Optional[SearchFilter], *, valid_at: Optional[float] = None + ) -> tuple[float, float]: + """Return world-time and system-time anchors for one read. + + ``valid_at`` is an explicit per-operation override used by graph traversal; + otherwise the filter's normalized ``valid_at``/legacy ``as_of`` value applies. + System-time defaults to the present, which preserves ordinary current reads. + """ + world = valid_at + if world is None and flt is not None: + world = flt.valid_at + known = flt.known_at if flt is not None else None + present = now_ts() + return (present if world is None else world, + present if known is None else known) + + +def _temporal_visibility_sql(alias: str, flt: Optional[SearchFilter], *, + valid_at: Optional[float] = None) -> tuple[str, list[Any]]: + """SQL predicate shared by temporal code-history reads.""" + world, known = _temporal_anchors(flt, valid_at=valid_at) + p = f"{alias}." if alias else "" + return ( + f"({p}valid_from IS NULL OR {p}valid_from<=?) " + f"AND ({p}valid_to IS NULL OR ?<{p}valid_to " + f"OR ({p}valid_to_recorded_at IS NOT NULL " + f"AND ?<{p}valid_to_recorded_at)) " + f"AND ({p}ingested_at IS NULL OR {p}ingested_at<=?) " + f"AND ({p}expired_at IS NULL OR ?<{p}expired_at)", + [world, world, known, known, known], + ) + + def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *, at: Optional[float] = None, include_invalid: bool = False) -> bool: @@ -270,14 +509,18 @@ def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *, return False if include_invalid: return True - t = at if at is not None else ( - flt.as_of if flt and flt.as_of is not None else now_ts() - ) - if rec.expired_at is not None: + valid_at, known_at = _temporal_anchors(flt, valid_at=at) + if rec.ingested_at is not None and rec.ingested_at > known_at: + return False + if rec.expired_at is not None and known_at >= rec.expired_at: return False - if rec.valid_from is not None and rec.valid_from > t: + if rec.valid_from is not None and rec.valid_from > valid_at: return False - if rec.valid_to is not None and t >= rec.valid_to: + if (rec.valid_to is not None and valid_at >= rec.valid_to + and not ( + rec.valid_to_recorded_at is not None + and known_at < rec.valid_to_recorded_at + )): return False return True @@ -559,18 +802,26 @@ def _cleanup_v4_backup_temps(self, backup_path: str) -> None: if changed: self._fsync_backup_parent(backup_path) - def _backup_before_v4_migration(self) -> str: - """Create and verify the mandatory pre-v4 backup without mutating source data. + def _backup_before_v4_migration(self, *, previous_version: int = 0) -> str: + """Create and verify the mandatory pre-migration backup without mutating data. Source and destination both use the injected connector, so SQLCipher databases remain keyed throughout. The caller holds ``BEGIN IMMEDIATE`` on the primary connection, preventing another writer from changing the source between this snapshot and the migration commit. Only a quick-checked temporary backup may atomically replace the stable backup path; every failure aborts the migration. + + Each migration target needs its own durable recovery artifact. For example, a + v5 database can legitimately retain the immutable ``.pre-migration-v5.bak`` + created during its v4→v5 upgrade. Reusing that name for a v5→v6 upgrade would + compare the older v4 snapshot with the later v5 source and abort the upgrade. + Preserve the legacy v4/v5 names and use the target schema version for newer + backups. """ if self.path in (":memory:", "") or self.path.startswith("file::memory:"): - raise RuntimeError("schema v4 migration requires a durable pre-migration backup") - backup_path = f"{self.path}.pre-migration-v4.bak" + raise RuntimeError("schema migration requires a durable pre-migration backup") + backup_version = max(4, min(SCHEMA_VERSION, previous_version + 1)) + backup_path = f"{self.path}.pre-migration-v{backup_version}.bak" self._cleanup_v4_backup_temps(backup_path) temp_path = ( f"{backup_path}.tmp-{os.getpid()}-{threading.get_ident()}-{time.time_ns()}" @@ -657,7 +908,7 @@ def _backup_before_v4_migration(self) -> str: except OSError: pass raise RuntimeError( - "schema v4 migration aborted: could not create and verify the " + f"schema v{backup_version} migration aborted: could not create and verify the " "pre-migration backup" ) from exc @@ -692,18 +943,33 @@ def init_schema(self) -> None: ).fetchone() value = row[0] if row is not None else None previous_version = int(value) if value is not None else 0 + # Early v5 databases recorded direct memory links with only ``created_at``. + # Track that shape independently of the version row so they receive the + # missing bi-temporal fields and backfill on their next safe open. + mem_link_columns: set[str] = set() + if "mem_links" in object_names: + mem_link_columns = { + str(row["name"]) + for row in self.conn.execute("PRAGMA table_info(mem_links)").fetchall() + } + mem_links_need_temporal_backfill = not { + "valid_from", "valid_to", "valid_to_recorded_at", "ingested_at", "expired_at", + }.issubset(mem_link_columns) + self._mem_links_need_temporal_backfill = mem_links_need_temporal_backfill if previous_version > SCHEMA_VERSION: raise RuntimeError( f"database schema {previous_version} is newer than supported " f"schema {SCHEMA_VERSION}" ) - needs_backup = bool(object_names) and previous_version < SCHEMA_VERSION + needs_backup = bool(object_names) and ( + previous_version < SCHEMA_VERSION or mem_links_need_temporal_backfill + ) try: # Reserve the writer before the snapshot. This is read/locking state only; # every schema/data transform remains inside the transaction below. self.conn.execute("BEGIN IMMEDIATE") if needs_backup: - self._backup_before_v4_migration() + self._backup_before_v4_migration(previous_version=previous_version) self._apply_schema(previous_version) self.conn.commit() except BaseException: @@ -712,6 +978,15 @@ def init_schema(self) -> None: raise def _apply_schema(self, previous_version: int) -> None: + mem_links_need_temporal_backfill = bool( + getattr(self, "_mem_links_need_temporal_backfill", False) + ) + receipt_sequence_existed = any( + str(row["name"]) == "sequence" + for row in self.conn.execute( + "PRAGMA table_info(operation_receipts)" + ).fetchall() + ) self._execute_script_transactional(SCHEMA_SQL) self.has_fts5 = _fts5_available(self.conn) self.conn.execute(FTS_SQL_FTS5 if self.has_fts5 else FTS_SQL_FALLBACK) @@ -720,15 +995,38 @@ def _apply_schema(self, previous_version: int) -> None: # explicit, idempotent ALTER TABLE here (SQLite has no "ADD COLUMN IF NOT EXISTS"). for stmt in ( "ALTER TABLE memories ADD COLUMN sort_order REAL", + "ALTER TABLE memories ADD COLUMN subject_key TEXT DEFAULT ''", + "ALTER TABLE memories ADD COLUMN claim_kind TEXT DEFAULT ''", + "ALTER TABLE memories ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE edges ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE entities ADD COLUMN normalized_name TEXT NOT NULL DEFAULT ''", "ALTER TABLE entities ADD COLUMN canonical_method TEXT NOT NULL DEFAULT 'exact'", "ALTER TABLE entities ADD COLUMN canonical_confidence REAL NOT NULL DEFAULT 1.0", "ALTER TABLE mem_links ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE mem_links ADD COLUMN reason TEXT DEFAULT ''", + "ALTER TABLE mem_links ADD COLUMN valid_from REAL", + "ALTER TABLE mem_links ADD COLUMN valid_to REAL", + "ALTER TABLE mem_links ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE mem_links ADD COLUMN ingested_at REAL", + "ALTER TABLE mem_links ADD COLUMN expired_at REAL", "ALTER TABLE code_edges ADD COLUMN layer TEXT DEFAULT 'entity'", "ALTER TABLE symbols ADD COLUMN docstring TEXT DEFAULT ''", + "ALTER TABLE symbols ADD COLUMN valid_from REAL", + "ALTER TABLE symbols ADD COLUMN valid_to REAL", + "ALTER TABLE symbols ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE symbols ADD COLUMN ingested_at REAL", + "ALTER TABLE symbols ADD COLUMN expired_at REAL", + "ALTER TABLE code_edges ADD COLUMN valid_from REAL", + "ALTER TABLE code_edges ADD COLUMN valid_to REAL", + "ALTER TABLE code_edges ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE code_edges ADD COLUMN ingested_at REAL", + "ALTER TABLE code_edges ADD COLUMN expired_at REAL", + "ALTER TABLE edges ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE edge_supports ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE memory_entities ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE code_memory_links ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE receipt_chain_heads ADD COLUMN integrity_error TEXT DEFAULT ''", + "ALTER TABLE operation_receipts ADD COLUMN sequence INTEGER", "ALTER TABLE jobs ADD COLUMN runner_id TEXT", "ALTER TABLE jobs ADD COLUMN heartbeat_at REAL", ): @@ -736,6 +1034,50 @@ def _apply_schema(self, previous_version: int) -> None: self.conn.execute(stmt) except sqlite3.OperationalError: pass # column already exists + # This cannot live in SCHEMA_SQL: CREATE TABLE IF NOT EXISTS leaves an + # early-v5 ``mem_links`` table untouched, so the index would reference + # temporal columns before the additive ALTERs above install them. + self.conn.execute( + "CREATE INDEX IF NOT EXISTS idx_mem_links_temporal " + "ON mem_links(a, valid_to, expired_at)" + ) + self.conn.execute( + "UPDATE operation_receipts SET workspace_id='' WHERE workspace_id IS NULL" + ) + self.conn.execute( + "UPDATE operation_receipts SET repo_id='' WHERE repo_id IS NULL" + ) + if not receipt_sequence_existed: + self._backfill_receipt_sequences() + self._execute_script_transactional( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_receipt_sequence " + "ON operation_receipts(workspace_id, sequence) " + "WHERE sequence IS NOT NULL;" + "DROP TRIGGER IF EXISTS trg_receipt_sequence_required;" + "CREATE TRIGGER trg_receipt_sequence_required " + "BEFORE INSERT ON operation_receipts " + "WHEN NEW.sequence IS NULL OR typeof(NEW.sequence)!='integer' " + "OR NEW.sequence<1 BEGIN " + "SELECT RAISE(ABORT, 'receipt sequence is required'); END;" + "DROP TRIGGER IF EXISTS trg_receipt_sequence_immutable;" + "CREATE TRIGGER trg_receipt_sequence_immutable " + "BEFORE UPDATE OF sequence ON operation_receipts " + "WHEN NEW.sequence IS NOT OLD.sequence BEGIN " + "SELECT RAISE(ABORT, 'receipt sequence is immutable'); END;" + ) + # These are migration transforms, not startup maintenance. Re-running the + # incidence backfill on every open scans the entire evidence graph and turns + # otherwise constant-time startup into O(workspace history). The schema-version + # row is written in the same transaction below, so an interrupted migration + # remains < v5 and safely retries all three transforms. + if previous_version < 5: + self._migrate_code_history_v5() + self._backfill_claim_identity_v5() + self._backfill_memory_entities_v5() + if previous_version < 5 or mem_links_need_temporal_backfill: + self._migrate_mem_link_history_v5() + if previous_version < 6: + self._migrate_code_file_history_v6() # Classify pre-v3 edges. Existing rows defaulted to semantic during ALTER TABLE; # infer their more specific logical layer from the relationship label. if previous_version < 3: @@ -780,6 +1122,24 @@ def _apply_schema(self, previous_version: int) -> None: "WHERE workspace_id IS NOT NULL AND repo_id IS NOT NULL " "AND valid_to IS NULL AND expired_at IS NULL;" ) + self._execute_script_transactional( + "CREATE INDEX IF NOT EXISTS idx_mem_claim_live " + "ON memories(workspace_id, repo_id, scope, mtype, subject_key, claim_kind) " + "WHERE subject_key<>'' AND valid_to IS NULL AND expired_at IS NULL;" + "CREATE INDEX IF NOT EXISTS idx_sym_repo_live " + "ON symbols(repo_id, file, fqname, valid_to, expired_at);" + "CREATE INDEX IF NOT EXISTS idx_code_edge_live " + "ON code_edges(repo_id, file, valid_to, expired_at);" + "CREATE UNIQUE INDEX IF NOT EXISTS idx_code_mem_live_unique " + "ON code_memory_links(repo_id, symbol_id, memory_id, relation) " + "WHERE valid_to IS NULL AND expired_at IS NULL;" + "CREATE INDEX IF NOT EXISTS idx_code_mem_symbol " + "ON code_memory_links(repo_id, symbol_id);" + "CREATE INDEX IF NOT EXISTS idx_code_mem_memory " + "ON code_memory_links(repo_id, memory_id);" + "CREATE INDEX IF NOT EXISTS idx_code_mem_live_symbol " + "ON code_memory_links(repo_id, symbol_id, valid_to, expired_at);" + ) # Every workspace has a cheap graph generation/state row, including databases # that already contained graph data before the v4 explorer tables were added. # Triggers in SCHEMA_SQL advance the generation on subsequent graph mutations. @@ -793,27 +1153,218 @@ def _apply_schema(self, previous_version: int) -> None: # anchor table existed. From this point onward every append updates it atomically, # allowing verification to detect deletion of the newest receipt as well as an # interior chain break. + if previous_version < 5: + receipt_scopes = self.conn.execute( + "SELECT r.workspace_id, COALESCE(MAX(r.ts), 0) AS updated_at " + "FROM operation_receipts r " + "LEFT JOIN receipt_chain_heads h ON h.workspace_id=r.workspace_id " + "WHERE h.workspace_id IS NULL " + "GROUP BY r.workspace_id" + ).fetchall() + for receipt_scope in receipt_scopes: + workspace_id = str(receipt_scope["workspace_id"] or "") + chain = self._receipt_chain_state(workspace_id) + self.conn.execute( + "INSERT OR IGNORE INTO receipt_chain_heads " + "(workspace_id, receipt_count, head_hash, integrity_error, updated_at) " + "VALUES (?,?,?,?,?)", + ( + workspace_id, + len(chain["rows"]), + chain["head"], + "" if not chain["errors"] else "migration_chain_invalid", + receipt_scope["updated_at"], + ), + ) self.conn.execute( - "UPDATE operation_receipts SET workspace_id='' WHERE workspace_id IS NULL" + "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (?,?)", + (SCHEMA_VERSION, now_ts()), ) + + def _migrate_code_history_v5(self) -> None: + """Give pre-v5 code graph rows open bi-temporal intervals. + + ``code_memory_links`` formerly had a table-level uniqueness constraint, which + made it impossible to retain a closed link and later create the same live link. + SQLite cannot drop that constraint in place, so rebuild that one narrow table + transactionally before installing the partial live-uniqueness index. + """ + stamp = now_ts() self.conn.execute( - "UPDATE operation_receipts SET repo_id='' WHERE repo_id IS NULL" + "UPDATE symbols SET valid_from=COALESCE(valid_from, updated_at, ?), " + "ingested_at=COALESCE(ingested_at, updated_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), ) self.conn.execute( - "INSERT OR IGNORE INTO receipt_chain_heads " - "(workspace_id, receipt_count, head_hash, integrity_error, updated_at) " - "SELECT COALESCE(r.workspace_id, ''), COUNT(*), " - " (SELECT r2.receipt_hash FROM operation_receipts r2 " - " WHERE COALESCE(r2.workspace_id, '')=COALESCE(r.workspace_id, '') " - " ORDER BY r2.rowid DESC LIMIT 1), " - " '', " - " COALESCE(MAX(r.ts), 0) " - "FROM operation_receipts r GROUP BY COALESCE(r.workspace_id, '')" + "UPDATE code_edges SET valid_from=COALESCE(valid_from, ?), " + "ingested_at=COALESCE(ingested_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), ) + columns = { + row["name"] for row in self.conn.execute( + "PRAGMA table_info(code_memory_links)" + ).fetchall() + } + if "valid_from" not in columns: + self.conn.execute( + "CREATE TABLE code_memory_links_v5 (" + "id TEXT PRIMARY KEY, repo_id TEXT NOT NULL, symbol_id TEXT NOT NULL, " + "memory_id TEXT NOT NULL, relation TEXT DEFAULT 'mentions', " + "confidence REAL DEFAULT 1.0, created_at REAL, valid_from REAL, " + "valid_to REAL, valid_to_recorded_at REAL, " + "ingested_at REAL, expired_at REAL)" + ) + self.conn.execute( + "INSERT INTO code_memory_links_v5(" + "id, repo_id, symbol_id, memory_id, relation, confidence, created_at, " + "valid_from, ingested_at) " + "SELECT id, repo_id, symbol_id, memory_id, relation, confidence, " + "created_at, COALESCE(created_at, ?), COALESCE(created_at, ?) " + "FROM code_memory_links", + (stamp, stamp), + ) + self.conn.execute("DROP TABLE code_memory_links") + self.conn.execute("ALTER TABLE code_memory_links_v5 RENAME TO code_memory_links") + else: + self.conn.execute( + "UPDATE code_memory_links SET valid_from=COALESCE(valid_from, created_at, ?), " + "ingested_at=COALESCE(ingested_at, created_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), + ) + + def _migrate_mem_link_history_v5(self) -> None: + """Give legacy direct memory links an open bi-temporal interval. + + ``created_at`` was the only historical signal on old rows, so it is both + the best available world-time and system-time start. Rows without a clock + start at migration time rather than being projected into every past view. + """ + stamp = now_ts() self.conn.execute( - "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (?,?)", - (SCHEMA_VERSION, now_ts()), + "UPDATE mem_links SET valid_from=COALESCE(valid_from, created_at, ?), " + "ingested_at=COALESCE(ingested_at, created_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), + ) + + def _migrate_code_file_history_v6(self) -> None: + """Seed temporal file manifests from the v5 current-file snapshot.""" + stamp = now_ts() + rows = self.conn.execute("SELECT * FROM code_files").fetchall() + for row in rows: + existing = self.conn.execute( + "SELECT 1 FROM code_file_history WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (row["repo_id"], row["file"]), + ).fetchone() + if existing is None: + started = row["indexed_at"] if row["indexed_at"] is not None else stamp + self.conn.execute( + "INSERT INTO code_file_history(" + "repo_id, file, lang, content_hash, size_bytes, mtime_ns, backend, " + "indexed_at, valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + row["repo_id"], row["file"], row["lang"], row["content_hash"], + row["size_bytes"], row["mtime_ns"], row["backend"], + row["indexed_at"], started, started, + ), + ) + + def _backfill_claim_identity_v5(self) -> None: + """Lift already-present metadata hints into indexed, optional claim columns.""" + rows = self.conn.execute( + "SELECT id, metadata, subject_key, claim_kind FROM memories" + ).fetchall() + for row in rows: + metadata = _loads(row["metadata"], {}) + if not isinstance(metadata, dict): + metadata = {} + subject_key = str(row["subject_key"] or metadata.get("subject_key") or "").strip() + claim_kind = str(row["claim_kind"] or metadata.get("claim_kind") or "").strip() + if subject_key != (row["subject_key"] or "") or claim_kind != (row["claim_kind"] or ""): + self.conn.execute( + "UPDATE memories SET subject_key=?, claim_kind=? WHERE id=?", + (subject_key, claim_kind, row["id"]), + ) + + def _backfill_memory_entities_v5(self, memory_id: Optional[str] = None) -> None: + """Materialize deterministic incidence already evidenced by graph supports.""" + sql = ( + "SELECT s.memory_id, endpoint.entity_id, e.workspace_id, e.repo_id, " + "s.confidence, s.valid_from, s.valid_to, s.valid_to_recorded_at, " + "s.ingested_at, s.expired_at, " + "e.valid_from AS edge_valid_from, e.valid_to AS edge_valid_to, " + "e.valid_to_recorded_at AS edge_valid_to_recorded_at, " + "e.ingested_at AS edge_ingested_at, e.expired_at AS edge_expired_at, " + "s.provenance " + "FROM edge_supports s JOIN edges e ON e.id=s.edge_id " + "JOIN (SELECT id AS edge_id, src AS entity_id FROM edges " + "UNION ALL SELECT id, dst FROM edges) endpoint ON endpoint.edge_id=e.id " + "WHERE 1=1" ) + params: list[Any] = [] + if memory_id is not None: + sql += " AND s.memory_id=?" + params.append(memory_id) + rows = self.conn.execute(sql, params).fetchall() + for row in rows: + valid_starts = [ + value for value in (row["valid_from"], row["edge_valid_from"]) + if value is not None + ] + valid_ends = [ + value for value in (row["valid_to"], row["edge_valid_to"]) + if value is not None + ] + known_starts = [ + value for value in (row["ingested_at"], row["edge_ingested_at"]) + if value is not None + ] + known_ends = [ + value for value in (row["expired_at"], row["edge_expired_at"]) + if value is not None + ] + valid_from = max(valid_starts) if valid_starts else None + valid_to = min(valid_ends) if valid_ends else None + closure_candidates = [ + (row["valid_to"], row["valid_to_recorded_at"]), + (row["edge_valid_to"], row["edge_valid_to_recorded_at"]), + ] + controlling_closures = [ + recorded for end, recorded in closure_candidates + if end is not None and end == valid_to + ] + valid_to_recorded_at = ( + None + if not controlling_closures or any( + recorded is None for recorded in controlling_closures + ) + else min(controlling_closures) + ) + ingested_at = max(known_starts) if known_starts else None + expired_at = min(known_ends) if known_ends else None + if (valid_from is not None and valid_to is not None + and valid_from >= valid_to): + continue + if (ingested_at is not None and expired_at is not None + and ingested_at >= expired_at): + continue + self.link_memory_entity( + memory_id=row["memory_id"], entity_id=row["entity_id"], + workspace_id=row["workspace_id"], repo_id=row["repo_id"], + source_kind="edge_support", confidence=row["confidence"], + valid_from=valid_from, valid_to=valid_to, + valid_to_recorded_at=valid_to_recorded_at, + ingested_at=ingested_at, expired_at=expired_at, + provenance=_loads(row["provenance"], {}), commit=False, + ) + + def backfill_memory_entities_for_memory(self, memory_id: str) -> None: + """Materialize the evidence incidence for one freshly written memory.""" + self._backfill_memory_entities_v5(memory_id) def _backfill_entity_canonicalization(self) -> None: rows = [dict(row) for row in self.conn.execute( @@ -1007,14 +1558,16 @@ def _deduplicate_live_edges(self) -> None: provenance = {} provenance["canonical_deduplicated_into"] = survivor["id"] self.conn.execute( - "UPDATE edges SET valid_to=?, provenance=? WHERE id=?", - (closed_at, _dumps(provenance), row["id"]), + "UPDATE edges SET valid_to=?, valid_to_recorded_at=?, " + "provenance=? WHERE id=?", + (closed_at, closed_at, _dumps(provenance), row["id"]), ) retired_marks = ",".join("?" for _ in retired_ids) self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id IN (" + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id IN (" + retired_marks + ") AND valid_to IS NULL AND expired_at IS NULL", - (closed_at, *retired_ids), + (closed_at, closed_at, *retired_ids), ) # Retire duplicates before normalizing the survivor endpoints. A pre-release # v4 database may already have the partial unique index; reversing the @@ -1305,15 +1858,17 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, f"existing provenance={existing['provenance']}, " f"incoming provenance={_dumps(rec.provenance)}", commit=False) ts = now_ts() - rec.ingested_at = rec.ingested_at or ts + rec.ingested_at = rec.ingested_at if rec.ingested_at is not None else ts rec.valid_from = rec.valid_from if rec.valid_from is not None else ts - rec.last_access = rec.last_access or ts + rec.last_access = rec.last_access if rec.last_access is not None else ts self.conn.execute( """INSERT INTO memories (id, workspace_id, repo_id, session_id, scope, mtype, title, content, summary, keywords, metadata, importance, surprise, stability, access_count, last_access, - valid_from, valid_to, ingested_at, expired_at, pinned, sensitivity, provenance) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, + subject_key, claim_kind, + pinned, sensitivity, provenance) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, repo_id=excluded.repo_id, session_id=excluded.session_id, scope=excluded.scope, mtype=excluded.mtype, @@ -1322,14 +1877,19 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, importance=excluded.importance, surprise=excluded.surprise, stability=excluded.stability, access_count=excluded.access_count, last_access=excluded.last_access, valid_from=excluded.valid_from, - valid_to=excluded.valid_to, ingested_at=excluded.ingested_at, - expired_at=excluded.expired_at, pinned=excluded.pinned, + valid_to=excluded.valid_to, + valid_to_recorded_at=excluded.valid_to_recorded_at, + ingested_at=excluded.ingested_at, + expired_at=excluded.expired_at, subject_key=excluded.subject_key, + claim_kind=excluded.claim_kind, pinned=excluded.pinned, sensitivity=excluded.sensitivity, provenance=excluded.provenance""", (rec.id, rec.workspace_id, rec.repo_id, rec.session_id, _enum(rec.scope), _enum(rec.mtype), rec.title, rec.content, rec.summary, _dumps(rec.keywords), _dumps(rec.metadata), rec.importance, rec.surprise, rec.stability, rec.access_count, rec.last_access, rec.valid_from, rec.valid_to, - rec.ingested_at, rec.expired_at, int(rec.pinned), rec.sensitivity, + rec.valid_to_recorded_at, rec.ingested_at, rec.expired_at, + rec.subject_key, rec.claim_kind, + int(rec.pinned), rec.sensitivity, _dumps(rec.provenance)), ) # full-text mirror @@ -1383,6 +1943,63 @@ def list_memories(self, flt: Optional[SearchFilter] = None, rows = self.conn.execute(sql, params).fetchall() return [_row_to_record(r) for r in rows] + def list_live_claims(self, *, workspace_id: str, repo_id: Optional[str], + session_id: Optional[str], scope: Scope, mtype: MemoryType, + subject_key: str, claim_kind: str) -> list[MemoryRecord]: + """Return the current instances of one exact claim identity. + + Conflict resolution normally looks at a candidate's valid-time neighbourhood. A + backdated candidate still needs to see a later, live instance of its *own* durable + claim key so it cannot create an overlapping history merely because an unrelated + anchored hit filled the vector candidate budget. + """ + subject_key = str(subject_key or "").strip() + if not subject_key: + return [] + sql = ( + "SELECT * FROM memories WHERE workspace_id=? AND repo_id IS ? " + "AND scope=? AND mtype=? AND subject_key=? AND claim_kind=? " + "AND valid_to IS NULL AND expired_at IS NULL" + ) + params: list[Any] = [ + workspace_id, repo_id, _enum(scope), _enum(mtype), subject_key, + str(claim_kind or "").strip(), + ] + if scope == Scope.SESSION: + sql += " AND session_id=?" + params.append(session_id) + sql += " ORDER BY ingested_at DESC, id" + rows = self.conn.execute(sql, params).fetchall() + return [_row_to_record(row) for row in rows] + + def list_claim_history(self, *, workspace_id: str, repo_id: Optional[str], + session_id: Optional[str], scope: Scope, mtype: MemoryType, + subject_key: str, claim_kind: str) -> list[MemoryRecord]: + """Return every recorded interval for one exact durable claim identity. + + Resolution uses this only to bound a newly inserted, backfilled keyed claim at + the next known successor. Closed rows are deliberately included: they are the + authoritative temporal chain and must not disappear merely because they are no + longer visible to present-day recall. + """ + subject_key = str(subject_key or "").strip() + if not subject_key: + return [] + sql = ( + "SELECT * FROM memories WHERE workspace_id=? AND repo_id IS ? " + "AND scope=? AND mtype=? AND subject_key=? AND claim_kind=?" + ) + params: list[Any] = [ + workspace_id, repo_id, _enum(scope), _enum(mtype), subject_key, + str(claim_kind or "").strip(), + ] + if scope == Scope.SESSION: + sql += " AND session_id=?" + params.append(session_id) + sql += " ORDER BY valid_from, ingested_at, id" + rows = self.conn.execute(sql, params).fetchall() + return [_row_to_record(row) for row in rows] + def list_memories_page(self, flt: Optional[SearchFilter] = None, *, after_id: str = "", limit: int = 500) -> list[MemoryRecord]: """Return one deterministic keyset page without materializing the full scope.""" @@ -1401,12 +2018,21 @@ def list_memories_page(self, flt: Optional[SearchFilter] = None, *, def close_validity(self, memory_id: str, *, at: Optional[float] = None, actor: str = "system", reason: str = "contradicted") -> None: - """Bi-temporal invalidation (§8.3): close a fact's validity window without deleting.""" - at = at if at is not None else now_ts() - self.conn.execute("UPDATE memories SET valid_to=? WHERE id=? AND valid_to IS NULL", - (at, memory_id)) + """Bi-temporal invalidation (§8.3): shorten a fact's validity without deleting.""" + recorded_at = now_ts() + at = at if at is not None else recorded_at + updated = self.conn.execute( + "UPDATE memories SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND (valid_to IS NULL OR valid_to>?)", + (at, recorded_at, memory_id, at), + ).rowcount + if updated: + self.invalidate_edges_for_memory(memory_id, at=at, commit=False) + # Governance attempts are audit-worthy even when the interval was already + # closed. MCP callers deliberately expose forget as non-idempotent so a + # repeated request keeps its own audit evidence while avoiding a second edge + # invalidation or widening a closed interval. self.audit(actor, "invalidate", memory_id, reason, commit=False) - self.invalidate_edges_for_memory(memory_id, at=at, commit=False) self.conn.commit() def set_pinned(self, memory_id: str, pinned: bool) -> None: @@ -1520,32 +2146,80 @@ def upsert_entity(self, node: Node, *, commit: bool = True) -> str: (node.workspace_id, node.repo_id, normalized, node.ntype), ).fetchone() if existing: - return existing["id"] - nid = node.id or ids.new_id("entity") - canonical_id = node.canonical_id - method = "provided" if canonical_id else "identity" - if not canonical_id: - canonical = self.conn.execute( - "SELECT COALESCE(canonical_id, id) AS canonical_id FROM entities " - "WHERE workspace_id=? AND normalized_name=? AND etype IS ? " - "ORDER BY id LIMIT 1", - (node.workspace_id, normalized, node.ntype), - ).fetchone() - if canonical: - canonical_id = canonical["canonical_id"] - method = "exact_normalized" - canonical_id = canonical_id or nid - self.conn.execute( - "INSERT INTO entities(id, workspace_id, repo_id, name, etype, canonical_id, " - "normalized_name, canonical_method, canonical_confidence, created_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?)", - (nid, node.workspace_id, node.repo_id, node.name, node.ntype, - canonical_id, normalized, method, 1.0, now_ts()), + nid = existing["id"] + else: + nid = node.id or ids.new_id("entity") + canonical_id = node.canonical_id + method = "provided" if canonical_id else "identity" + if not canonical_id: + canonical = self.conn.execute( + "SELECT COALESCE(canonical_id, id) AS canonical_id FROM entities " + "WHERE workspace_id=? AND normalized_name=? AND etype IS ? " + "ORDER BY id LIMIT 1", + (node.workspace_id, normalized, node.ntype), + ).fetchone() + if canonical: + canonical_id = canonical["canonical_id"] + method = "exact_normalized" + canonical_id = canonical_id or nid + self.conn.execute( + "INSERT INTO entities(id, workspace_id, repo_id, name, etype, canonical_id, " + "normalized_name, canonical_method, canonical_confidence, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + (nid, node.workspace_id, node.repo_id, node.name, node.ntype, + canonical_id, normalized, method, 1.0, now_ts()), + ) + self._backfill_entity_text_mentions( + nid, name=node.name, workspace_id=node.workspace_id, repo_id=node.repo_id, ) if commit: self.conn.commit() return nid + def _backfill_entity_text_mentions(self, entity_id: str, *, name: str, + workspace_id: Optional[str], + repo_id: Optional[str]) -> None: + """Attach an entity added after its matching prose memories already existed. + + New writes are linked by ``MemoryEngine._link_memory_entities``. This bounded, + exact-word backfill preserves the same graph reachability for imported or legacy + memories when their entity is introduced later, without a recall-time prose scan. + """ + name = (name or "").strip() + if len(name) < 2: + return + scope_sql = "repo_id IS NULL" + scope_params: list[Any] = [] + if repo_id is not None: + # A contextual repo read includes its workspace/user ancestors. Do not + # lose a legacy workspace mention merely because the matching entity was + # introduced later in a repository; sibling repositories stay isolated. + scope_sql = "(repo_id=? OR repo_id IS NULL)" + scope_params.append(repo_id) + rows = self.conn.execute( + "SELECT id, title, content, workspace_id, repo_id, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM memories " + "WHERE workspace_id IS ? AND scope<>'session' AND " + scope_sql + " " + "AND (lower(title) LIKE ? ESCAPE '\\' OR lower(content) LIKE ? ESCAPE '\\') " + "ORDER BY id LIMIT 12000", + (workspace_id, *scope_params, + "%" + _escape_like(name.casefold()) + "%", + "%" + _escape_like(name.casefold()) + "%"), + ).fetchall() + pattern = re.compile(r"(? list[Node]: """Entities in scope, newest first — the seed set the profile-consolidation @@ -1558,7 +2232,10 @@ def list_entities(self, flt: Optional[SearchFilter] = None, where.append("workspace_id=?") params.append(flt.workspace_id) if flt and flt.repo_id: - where.append("repo_id=?") + if flt.include_ancestors: + where.append("(repo_id=? OR repo_id IS NULL)") + else: + where.append("repo_id=?") params.append(flt.repo_id) if where: sql += " WHERE " + " AND ".join(where) @@ -1570,6 +2247,171 @@ def list_entities(self, flt: Optional[SearchFilter] = None, workspace_id=r["workspace_id"], repo_id=r["repo_id"], canonical_id=r["canonical_id"]) for r in rows] + def link_memory_entity(self, *, memory_id: str, entity_id: str, + workspace_id: Optional[str], repo_id: Optional[str], + source_kind: str = "explicit", confidence: float = 1.0, + valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None, + provenance: Optional[dict] = None, + commit: bool = True) -> str: + """Create one idempotent, bi-temporal memory↔entity incidence record.""" + stamp = now_ts() + if valid_to is None and expired_at is None: + existing = self.conn.execute( + "SELECT id, confidence, valid_from, ingested_at " + "FROM memory_entities WHERE memory_id=? AND entity_id=? " + "AND source_kind=? AND valid_to IS NULL AND expired_at IS NULL", + (memory_id, entity_id, source_kind), + ).fetchone() + requested_valid = ( + valid_from if valid_from is not None + else (existing["valid_from"] if existing is not None else stamp) + ) + requested_known = ( + ingested_at if ingested_at is not None + else (existing["ingested_at"] if existing is not None else stamp) + ) + else: + requested_valid = valid_from if valid_from is not None else stamp + requested_known = ingested_at if ingested_at is not None else stamp + existing = self.conn.execute( + "SELECT id FROM memory_entities WHERE memory_id=? AND entity_id=? " + "AND source_kind=? AND valid_from IS ? AND valid_to IS ? " + "AND valid_to_recorded_at IS ? " + "AND ingested_at IS ? AND expired_at IS ?", + ( + memory_id, entity_id, source_kind, requested_valid, valid_to, + valid_to_recorded_at, requested_known, expired_at, + ), + ).fetchone() + if existing is not None: + if valid_to is None and expired_at is None: + desired_confidence = max( + float(existing["confidence"] or 0.0), + max(0.0, min(1.0, float(confidence))), + ) + if (requested_valid == existing["valid_from"] + and requested_known == existing["ingested_at"]): + if desired_confidence != float(existing["confidence"] or 0.0): + self.conn.execute( + "UPDATE memory_entities SET confidence=? WHERE id=?", + (desired_confidence, existing["id"]), + ) + if commit: + self.conn.commit() + return existing["id"] + + # A later observation can describe the same incidence with a different + # valid/known pair. Version it instead of independently minimising the + # coordinates, which would fabricate a historical interval no source ever + # asserted (for example valid_from=50 paired with ingested_at=100). + retire_at = max( + (value for value in (existing["ingested_at"], requested_known) + if value is not None), + default=stamp, + ) + self.conn.execute( + "UPDATE memory_entities SET expired_at=? WHERE id=?", + (retire_at, existing["id"]), + ) + else: + return existing["id"] + link_id = ids.new_id("edge") + self.conn.execute( + "INSERT INTO memory_entities(" + "id, memory_id, entity_id, workspace_id, repo_id, source_kind, confidence, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, " + "provenance) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (link_id, memory_id, entity_id, workspace_id, repo_id, source_kind, + max(0.0, min(1.0, float(confidence))), + requested_valid, valid_to, valid_to_recorded_at, requested_known, expired_at, + _dumps(provenance or {})), + ) + if commit: + self.conn.commit() + return link_id + + def list_memory_entities(self, flt: Optional[SearchFilter] = None, *, + entity_ids: Optional[list[str]] = None, + memory_ids: Optional[list[str]] = None, + limit: Optional[int] = None) -> list[dict]: + """Return bounded scoped/temporal incidence rows for graph retrieval.""" + # Consolidation scans up to 2,000 memories, while portable SQLite builds may + # allow only 999 bind variables. Partition ID filters before building the SQL + # predicate; each pair of chunks is disjoint, so merging preserves results. + entity_chunks = ( + [entity_ids[start:start + IN_CLAUSE_CHUNK] + for start in range(0, len(entity_ids), IN_CLAUSE_CHUNK)] + if entity_ids is not None else [None] + ) + memory_chunks = ( + [memory_ids[start:start + IN_CLAUSE_CHUNK] + for start in range(0, len(memory_ids), IN_CLAUSE_CHUNK)] + if memory_ids is not None else [None] + ) + if not entity_chunks or not memory_chunks: + return [] + if len(entity_chunks) > 1 or len(memory_chunks) > 1: + rows = [ + row + for entity_chunk in entity_chunks + for memory_chunk in memory_chunks + for row in self.list_memory_entities( + flt, entity_ids=entity_chunk, memory_ids=memory_chunk, + ) + ] + rows.sort(key=lambda row: (-float(row.get("confidence") or 0.0), row["id"])) + return rows if limit is None else rows[:max(0, int(limit))] + valid_at, known_at = _temporal_anchors(flt) + sql = ( + "SELECT me.* FROM memory_entities me " + "JOIN memories m ON m.id=me.memory_id WHERE " + "(me.valid_from IS NULL OR me.valid_from<=?) " + "AND (me.valid_to IS NULL OR ? str: eid = edge.id or ids.new_id("edge") layer = normalize_graph_layer(edge.layer, edge.relation).value @@ -1579,7 +2421,7 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: incoming_provenance = _merge_edge_provenance([edge.provenance]) existing = self.conn.execute( "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " - "valid_from, valid_to, ingested_at, expired_at, provenance " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, provenance " "FROM edges WHERE id=?", (eid,) ).fetchone() replacing = existing is not None @@ -1613,17 +2455,29 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: value for value in (existing["valid_from"], edge.valid_from) if value is not None ) + desired_ingested_at = existing["ingested_at"] + if edge.ingested_at is not None: + desired_ingested_at = min( + value for value in (existing["ingested_at"], edge.ingested_at) + if value is not None + ) serialized_provenance = _dumps(merged_provenance) if desired_weight != float(existing["weight"] or 0.0) \ or desired_valid_from != existing["valid_from"] \ + or desired_ingested_at != existing["ingested_at"] \ or serialized_provenance != (existing["provenance"] or "{}"): self.conn.execute( - "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", - (desired_weight, desired_valid_from, serialized_provenance, eid), + "UPDATE edges SET weight=?, valid_from=?, ingested_at=?, " + "provenance=? WHERE id=?", + ( + desired_weight, desired_valid_from, desired_ingested_at, + serialized_provenance, eid, + ), ) self._write_edge_supports( eid, edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1632,7 +2486,7 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: equivalent = None if edge.valid_to is None and edge.expired_at is None: equivalent = self.conn.execute( - "SELECT id, weight, valid_from, provenance FROM edges " + "SELECT id, weight, valid_from, ingested_at, provenance FROM edges " "WHERE workspace_id IS ? AND repo_id IS ? AND src=? AND dst=? " "AND relation=? AND layer=? AND valid_to IS NULL AND expired_at IS NULL " "AND id<>? ORDER BY id LIMIT 1", @@ -1645,13 +2499,15 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: if replacing: closed_at = now_ts() self.conn.execute( - "UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", - (closed_at, eid), + "UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND valid_to IS NULL", + (closed_at, closed_at, eid), ) self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (closed_at, eid), + (closed_at, closed_at, eid), ) existing_provenance = _loads(equivalent["provenance"], {}) merged_provenance = _merge_edge_provenance( @@ -1661,17 +2517,25 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: valid_values = [value for value in ( equivalent["valid_from"], edge.valid_from ) if value is not None] + known_values = [ + value for value in ( + equivalent["ingested_at"], edge.ingested_at + ) if value is not None + ] self.conn.execute( - "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", + "UPDATE edges SET weight=?, valid_from=?, ingested_at=?, provenance=? " + "WHERE id=?", ( max(float(equivalent["weight"] or 0.0), float(edge.weight or 0.0)), min(valid_values) if valid_values else now_ts(), + min(known_values) if known_values else now_ts(), _dumps(merged_provenance), equivalent["id"], ), ) self._write_edge_supports( equivalent["id"], edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1681,29 +2545,36 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: # ``upsert_edge`` replaces the supplied edge record. Close its previous # normalized evidence before writing the replacement so sources removed # from the new provenance cannot remain live invisibly. + closed_at = now_ts() self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (now_ts(), eid), + (closed_at, closed_at, eid), ) self.conn.execute( "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " - "weight, valid_from, valid_to, ingested_at, expired_at, provenance) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) " + "weight, valid_from, valid_to, valid_to_recorded_at, ingested_at, " + "expired_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) " "ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, " "repo_id=excluded.repo_id, src=excluded.src, dst=excluded.dst, " "relation=excluded.relation, layer=excluded.layer, weight=excluded.weight, " "valid_from=excluded.valid_from, valid_to=excluded.valid_to, " + "valid_to_recorded_at=excluded.valid_to_recorded_at, " "ingested_at=excluded.ingested_at, expired_at=excluded.expired_at, " "provenance=excluded.provenance", (eid, edge.workspace_id, edge.repo_id, source, target, edge.relation, layer, edge.weight, edge.valid_from if edge.valid_from is not None else now_ts(), - edge.valid_to, edge.ingested_at or now_ts(), edge.expired_at, + edge.valid_to, edge.valid_to_recorded_at, + edge.ingested_at if edge.ingested_at is not None else now_ts(), + edge.expired_at, _dumps(incoming_provenance)), ) self._write_edge_supports( eid, edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1711,18 +2582,24 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: return eid def invalidate_edge(self, edge_id: str, at: Optional[float] = None) -> None: - ts = now_ts() if at is None else at - self.conn.execute("UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", - (ts, edge_id)) + recorded_at = now_ts() + ts = recorded_at if at is None else at + self.conn.execute( + "UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND valid_to IS NULL", + (ts, recorded_at, edge_id), + ) self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " - "AND valid_to IS NULL AND expired_at IS NULL", (ts, edge_id) + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? AND valid_to IS NULL AND expired_at IS NULL", + (ts, recorded_at, edge_id), ) self.conn.commit() def _write_edge_supports(self, edge_id: str, relation: str, provenance: dict, *, valid_from: Optional[float] = None, valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, ingested_at: Optional[float] = None, expired_at: Optional[float] = None) -> None: source_kind = _edge_source_kind(provenance, relation) @@ -1772,13 +2649,17 @@ def _write_edge_supports(self, edge_id: str, relation: str, provenance: dict, self.conn.execute( "INSERT OR IGNORE INTO edge_supports " "(edge_id, memory_id, source_kind, confidence, valid_from, valid_to, " - "ingested_at, expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?)", + "valid_to_recorded_at, ingested_at, expired_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", (edge_id, memory_id, source_kind, confidence, - support_valid_from, valid_to, support_ingested_at, expired_at, + support_valid_from, valid_to, valid_to_recorded_at, + support_ingested_at, expired_at, _dumps(support_provenance)), ) def add_edge_support(self, edge_id: str, provenance: dict, *, + valid_from: Optional[float] = None, + ingested_at: Optional[float] = None, commit: bool = True) -> None: """Record another source memory supporting an existing graph edge.""" incoming = _provenance_memory_ids(provenance) @@ -1795,15 +2676,43 @@ def add_edge_support(self, edge_id: str, provenance: dict, *, self.conn.execute("UPDATE edges SET provenance=? WHERE id=?", (_dumps(merged_provenance), edge_id)) edge_row = self.conn.execute( - "SELECT relation, valid_from, valid_to, ingested_at, expired_at " + "SELECT relation, valid_from, valid_to, valid_to_recorded_at, " + "ingested_at, expired_at " "FROM edges WHERE id=?", (edge_id,) ).fetchone() if edge_row: + support_valid_from = ( + valid_from if valid_from is not None else edge_row["valid_from"] + ) + support_ingested_at = ( + ingested_at if ingested_at is not None else edge_row["ingested_at"] + ) self._write_edge_supports( edge_id, edge_row["relation"] or "", provenance, - valid_from=edge_row["valid_from"], valid_to=edge_row["valid_to"], - ingested_at=edge_row["ingested_at"], expired_at=edge_row["expired_at"], + valid_from=support_valid_from, valid_to=edge_row["valid_to"], + valid_to_recorded_at=edge_row["valid_to_recorded_at"], + ingested_at=support_ingested_at, expired_at=edge_row["expired_at"], ) + # The edge is the union of its supporting evidence intervals. A + # backdated support must make the relation visible at that earlier + # world time, and a historically imported support may likewise be + # known before the edge's previous system-time anchor. + valid_values = [ + value for value in (edge_row["valid_from"], support_valid_from) + if value is not None + ] + ingested_values = [ + value for value in (edge_row["ingested_at"], support_ingested_at) + if value is not None + ] + earlier_valid = min(valid_values) if valid_values else None + earlier_ingested = min(ingested_values) if ingested_values else None + if (earlier_valid != edge_row["valid_from"] + or earlier_ingested != edge_row["ingested_at"]): + self.conn.execute( + "UPDATE edges SET valid_from=?, ingested_at=? WHERE id=?", + (earlier_valid, earlier_ingested, edge_id), + ) if commit: self.conn.commit() @@ -1826,7 +2735,8 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N directly (service.py): those edges would carry provenance but no support rows, and would then silently never be invalidated. Normalize the edge writes first. """ - ts = at if at is not None else now_ts() + recorded_at = now_ts() + ts = at if at is not None else recorded_at owner = self.conn.fetchall( "SELECT workspace_id FROM memories WHERE id=?", (memory_id,)) workspace_id = owner[0]["workspace_id"] if owner else None @@ -1857,9 +2767,10 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N if memory_id not in supports: continue self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? AND memory_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? AND memory_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (ts, row["id"], memory_id), + (ts, recorded_at, row["id"], memory_id), ) normalized_remaining = [r["memory_id"] for r in self.conn.execute( "SELECT DISTINCT memory_id FROM edge_supports WHERE edge_id=? " @@ -1876,12 +2787,16 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N (_dumps(prov), row["id"])) if ids_to_close: marks = ",".join("?" for _ in ids_to_close) - self.conn.execute(f"UPDATE edges SET valid_to=? WHERE id IN ({marks})", - (ts, *ids_to_close)) self.conn.execute( - f"UPDATE edge_supports SET valid_to=? WHERE edge_id IN ({marks}) " + f"UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + f"WHERE id IN ({marks})", + (ts, recorded_at, *ids_to_close), + ) + self.conn.execute( + f"UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + f"WHERE edge_id IN ({marks}) " "AND valid_to IS NULL AND expired_at IS NULL", - (ts, *ids_to_close), + (ts, recorded_at, *ids_to_close), ) if commit: self.conn.commit() @@ -1889,19 +2804,44 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N # ── memory-to-memory links (A-MEM style) ──────────────────────────────────── def edge_supports_in_scope(self, edge_ids: Optional[list[str]] = None, *, at: Optional[float] = None, + flt: Optional[SearchFilter] = None, limit: Optional[int] = None) -> list[dict]: - """Return live normalized evidence rows for graph inspection/scene scoring.""" - t = at if at is not None else now_ts() + """Return evidence visible at the supplied world/system-time anchors.""" + valid_at, known_at = _temporal_anchors(flt, valid_at=at) row_cap = None if limit is None else max(0, int(limit)) if row_cap == 0: return [] sql = ( - "SELECT id, edge_id, memory_id, source_kind, confidence, valid_from, " - "valid_to, ingested_at, expired_at, provenance FROM edge_supports " - "WHERE (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? None: + *, valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None, + commit: bool = True) -> None: """Idempotent per (pair, relation): re-linking the same two memories with the same relation is a no-op in either direction, so auto-evolution and explicit ``engraphis_link`` calls can't accrete duplicate rows.""" - existing = self.conn.execute( - "SELECT rowid, layer, reason FROM mem_links " - "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? LIMIT 1", - (a, b, b, a, relation), - ).fetchone() - if existing: - updates: list[str] = [] - params: list[Any] = [] - if layer is not None: - graph_layer = normalize_graph_layer(layer, relation).value - if existing["layer"] != graph_layer: - updates.append("layer=?") - params.append(graph_layer) - if reason and existing["reason"] != reason: - updates.append("reason=?") - params.append(reason) - if updates: - params.append(existing["rowid"]) - self.conn.execute( - f"UPDATE mem_links SET {', '.join(updates)} WHERE rowid=?", - params, + requested_layer = ( + normalize_graph_layer(layer, relation).value + if layer is not None else None + ) + graph_layer = requested_layer or normalize_graph_layer(None, relation).value + started_transaction = not self.conn.in_transaction + if started_transaction: + self.conn.execute("BEGIN IMMEDIATE") + try: + # A sync bundle may carry a closed link interval. It has no live row to + # match below, so recognize an exact historical version before inserting + # it again on every replay. ``IS`` deliberately gives NULL-safe equality. + exact = self.conn.execute( + "SELECT 1 FROM mem_links " + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND layer=? AND reason=? AND valid_from IS ? AND valid_to IS ? " + "AND valid_to_recorded_at IS ? AND ingested_at IS ? AND expired_at IS ? " + "LIMIT 1", + ( + a, b, b, a, relation, graph_layer, reason, + valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, + ), + ).fetchone() + if exact is not None: + if started_transaction: + self.conn.commit() + return + existing = self.conn.execute( + "SELECT rowid, a, b, relation, layer, reason, created_at, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at " + "FROM mem_links " + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND valid_to IS NULL AND expired_at IS NULL " + "ORDER BY rowid DESC LIMIT 1", + (a, b, b, a, relation), + ).fetchone() + if existing: + graph_layer = ( + requested_layer + if requested_layer is not None else existing["layer"] ) - if commit: + replacement_reason = reason if reason else existing["reason"] + if ( + graph_layer != existing["layer"] + or replacement_reason != existing["reason"] + ): + # Metadata is part of what the system knew about this link. Updating + # it in place would rewrite a historical ``known_at`` view. Retire the + # system-time version and open a replacement over the same world-time + # interval so past reads remain immutable while current reads converge. + stamp = max( + now_ts(), + ( + float(existing["ingested_at"]) + if existing["ingested_at"] is not None + else float("-inf") + ), + ) + self.conn.execute( + "UPDATE mem_links SET expired_at=? " + "WHERE rowid=? AND expired_at IS NULL", + (stamp, existing["rowid"]), + ) + self.conn.execute( + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,NULL)", + ( + existing["a"], existing["b"], existing["relation"], + graph_layer, replacement_reason, stamp, + existing["valid_from"], existing["valid_to"], + existing["valid_to_recorded_at"], stamp, + ), + ) + if commit: + self.conn.commit() + elif started_transaction: + # The pre-read reservation has no write to batch. Release it even + # for ``commit=False``; the old no-op path never opened a transaction. self.conn.commit() - return + return + stamp = now_ts() + world_start = stamp if valid_from is None else valid_from + system_start = stamp if ingested_at is None else ingested_at + self.conn.execute( + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (a, b, relation, graph_layer, reason, stamp, world_start, valid_to, + valid_to_recorded_at, system_start, expired_at), + ) + if commit: + self.conn.commit() + except BaseException: + if started_transaction and self.conn.in_transaction: + self.conn.rollback() + raise + + def add_link_version(self, a: str, b: str, relation: str = "related", + layer: Optional[GraphLayer] = None, reason: str = "", *, + valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None, + commit: bool = True) -> bool: + """Persist one exact temporal link version without collapsing live evidence. + + Normal :meth:`add_link` intentionally de-duplicates active relationships for + interactive callers. Sync is different: two peers can independently observe the + same relation with distinct valid/known intervals, and both intervals are needed + for a convergent historical graph. This method appends that exact observation and + returns whether it was new, while replaying the same version remains a no-op. + """ graph_layer = normalize_graph_layer(layer, relation).value - self.conn.execute( - "INSERT INTO mem_links(a, b, relation, layer, reason, created_at) " - "VALUES (?,?,?,?,?,?)", - (a, b, relation, graph_layer, reason, now_ts()), - ) - if commit: - self.conn.commit() + stamp = now_ts() + world_start = stamp if valid_from is None else valid_from + system_start = stamp if ingested_at is None else ingested_at + started_transaction = not self.conn.in_transaction + if started_transaction: + self.conn.execute("BEGIN IMMEDIATE") + try: + exact = self.conn.execute( + "SELECT 1 FROM mem_links " + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND layer=? AND reason=? AND valid_from IS ? AND valid_to IS ? " + "AND valid_to_recorded_at IS ? AND ingested_at IS ? AND expired_at IS ? " + "LIMIT 1", + ( + a, b, b, a, relation, graph_layer, reason, + world_start, valid_to, valid_to_recorded_at, system_start, expired_at, + ), + ).fetchone() + if exact is not None: + if started_transaction: + self.conn.commit() + return False + self.conn.execute( + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (a, b, relation, graph_layer, reason, stamp, world_start, valid_to, + valid_to_recorded_at, system_start, expired_at), + ) + if commit: + self.conn.commit() + return True + except BaseException: + if started_transaction and self.conn.in_transaction: + self.conn.rollback() + raise def has_link(self, a: str, b: str, *, relation: Optional[str] = None) -> bool: - sql = "SELECT 1 FROM mem_links WHERE ((a=? AND b=?) OR (a=? AND b=?))" + """Return whether the pair has a current open link interval. + + Closed history must not block a later reactivation of the same relationship. + Historical visibility remains available through ``get_links``/``links_among``. + """ + sql = ( + "SELECT 1 FROM mem_links WHERE ((a=? AND b=?) OR (a=? AND b=?)) " + "AND valid_to IS NULL AND expired_at IS NULL" + ) params: list[Any] = [a, b, b, a] if relation is not None: sql += " AND relation=?" params.append(relation) return self.conn.execute(sql + " LIMIT 1", params).fetchone() is not None - def get_links(self, memory_id: str) -> list[dict]: + def get_links(self, memory_id: str, *, + flt: Optional[SearchFilter] = None) -> list[dict]: + """Return direct links visible at the filter's bi-temporal anchors.""" + visible_sql, params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT a, b, relation, layer, reason, created_at " - "FROM mem_links WHERE a=? OR b=?", - (memory_id, memory_id), + "SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM mem_links " + f"WHERE (a=? OR b=?) AND {visible_sql} ORDER BY a, b, relation", + (memory_id, memory_id, *params), ).fetchall() return [dict(r) for r in rows] def edges_in_scope(self, flt: Optional[SearchFilter] = None, *, at: Optional[float] = None, limit: Optional[int] = None) -> list[Edge]: - """Every edge valid at ``at`` within the filter's workspace/repo — the graph - the PPR retrieval arm walks (edges outside their validity window are invisible, - same bi-temporal rule as memories).""" - t = at if at is not None else now_ts() + """Edges visible at ``at``/``filter.valid_at`` and ``filter.known_at``. + + Normalized supports are authoritative for edges that have them. The edge row + aggregates its support starts for current-read efficiency, but independently + minimizing world and system time can fabricate a pair no source established. + A historical read must therefore see at least one individually visible support. + Legacy direct edges with no normalized support retain the edge-row fallback. + """ + valid_at, known_at = _temporal_anchors(flt, valid_at=at) sql = ("SELECT * FROM edges WHERE (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? list[dict]: - """mem_links rows where *both* endpoints are in ``ids`` (for graph retrieval).""" + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None, + include_invalid: bool = False, + limit: Optional[int] = None) -> list[dict]: + """Return memory links visible under both temporal anchors. + + ``include_invalid`` is for full-state replication only: a closed interval is + state that must synchronize even though normal graph reads do not expose it. + + Chunk only the indexed ``a`` side and filter ``b`` against an in-memory set. + This keeps every statement below SQLite's portable variable limit while + preserving exact pair semantics for graphs containing thousands of memories. + """ if not ids: return [] - marks = ",".join("?" for _ in ids) - sql = ( - f"SELECT a, b, relation, layer, reason FROM mem_links " - f"WHERE a IN ({marks}) AND b IN ({marks})" - ) - params: list[Any] = [*ids, *ids] - if layers: - layer_marks = ",".join("?" for _ in layers) - sql += f" AND layer IN ({layer_marks})" - params.extend(_enum(layer) for layer in layers) - rows = self.conn.execute(sql, params).fetchall() - return [dict(r) for r in rows] + if layers is not None and not layers: + return [] + row_cap = None if limit is None else max(0, int(limit)) + if row_cap == 0: + return [] + wanted = set(ids) + ordered_ids = sorted(wanted) + visibility_sql, visibility_params = _temporal_visibility_sql("", flt) + rows: list[dict] = [] + # Leave headroom for the time anchor and optional layer parameters. + chunk_size = max(1, IN_CLAUSE_CHUNK - 16) + for start in range(0, len(ordered_ids), chunk_size): + if row_cap is not None and len(rows) >= row_cap: + break + chunk = ordered_ids[start:start + chunk_size] + marks = ",".join("?" for _ in chunk) + sql = ( + "SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM mem_links " + f"WHERE a IN ({marks})" + ) + params: list[Any] = [*chunk] + if not include_invalid: + sql += f" AND {visibility_sql}" + params.extend(visibility_params) + if layers is not None: + layer_marks = ",".join("?" for _ in layers) + sql += f" AND layer IN ({layer_marks})" + params.extend(_enum(layer) for layer in layers) + sql += " ORDER BY a, b, relation, valid_from, ingested_at" + found = self.conn.execute(sql, params).fetchall() + for row in found: + if row["b"] not in wanted: + continue + rows.append(dict(row)) + if row_cap is not None and len(rows) >= row_cap: + break + return rows + + def links_touching(self, ids: list[str], *, + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None, + include_invalid: bool = False, + limit: Optional[int] = None) -> list[dict]: + """Return visible links with at least one endpoint in ``ids``. + + This bounded frontier expansion is distinct from :meth:`links_among`: graph + recall uses it to retain an unmentioned endpoint linked to an entity-attached + memory, without first materializing every memory in a large scope. + """ + if not ids: + return [] + if layers is not None and not layers: + return [] + row_cap = None if limit is None else max(0, int(limit)) + if row_cap == 0: + return [] + ordered_ids = sorted(set(ids)) + visibility_sql, visibility_params = _temporal_visibility_sql("", flt) + rows: list[dict] = [] + seen: set[tuple] = set() + # Each id appears once for each endpoint predicate; reserve parameters for + # time/layer filters so this remains under SQLite's portable bind limit. + chunk_size = max(1, (IN_CLAUSE_CHUNK - 16) // 2) + for start in range(0, len(ordered_ids), chunk_size): + if row_cap is not None and len(rows) >= row_cap: + break + chunk = ordered_ids[start:start + chunk_size] + marks = ",".join("?" for _ in chunk) + sql = ( + "SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM mem_links " + f"WHERE (a IN ({marks}) OR b IN ({marks}))" + ) + params: list[Any] = [*chunk, *chunk] + if not include_invalid: + sql += f" AND {visibility_sql}" + params.extend(visibility_params) + if layers is not None: + layer_marks = ",".join("?" for _ in layers) + sql += f" AND layer IN ({layer_marks})" + params.extend(_enum(layer) for layer in layers) + sql += " ORDER BY a, b, relation, valid_from, ingested_at" + for row in self.conn.execute(sql, params).fetchall(): + item = dict(row) + key = ( + item["a"], item["b"], item["relation"], item["layer"], + item["valid_from"], item["valid_to"], item["ingested_at"], + ) + if key in seen: + continue + seen.add(key) + rows.append(item) + if row_cap is not None and len(rows) >= row_cap: + break + return rows def neighbors(self, node_ids: list[str], *, at: Optional[float] = None, - layers: Optional[list[GraphLayer]] = None) -> list[Edge]: + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None, + limit: Optional[int] = None) -> list[Edge]: if not node_ids: return [] - t = at if at is not None else now_ts() + valid_at, known_at = _temporal_anchors(flt, valid_at=at) marks = ",".join("?" for _ in node_ids) sql = ( f"SELECT * FROM edges WHERE (src IN ({marks}) OR dst IN ({marks})) " - f"AND (valid_from IS NULL OR valid_from<=?) AND (valid_to IS NULL OR ? None: - """Re-indexing a file replaces its symbols/edges — incremental indexing is - idempotent per file, not additive.""" + """Retire a file's live code graph rows before an incremental re-index.""" + stamp = now_ts() symbol_rows = self.conn.execute( - "SELECT id FROM symbols WHERE repo_id=? AND file=?", (repo_id, file) + "SELECT id FROM symbols WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id, file) ).fetchall() symbol_ids = [row["id"] for row in symbol_rows] if symbol_ids: marks = ",".join("?" for _ in symbol_ids) self.conn.execute( - f"DELETE FROM code_memory_links WHERE repo_id=? " - f"AND symbol_id IN ({marks})", - (repo_id, *symbol_ids), + f"UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + f"WHERE repo_id=? " + f"AND symbol_id IN ({marks}) AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, *symbol_ids), ) - self.conn.execute("DELETE FROM symbols WHERE repo_id=? AND file=?", (repo_id, file)) - self.conn.execute("DELETE FROM code_edges WHERE repo_id=? AND file=?", (repo_id, file)) + self.conn.execute( + "UPDATE symbols SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, file), + ) + self.conn.execute( + "UPDATE code_edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, file), + ) if commit: self.conn.commit() @@ -2079,10 +3323,10 @@ def upsert_symbol(self, *, repo_id: str, kind: str, name: str, fqname: str, file sid = ids.new_id("symbol") self.conn.execute( "INSERT INTO symbols(id, repo_id, kind, name, fqname, file, span, signature, " - "docstring, lang, exported, content_hash, updated_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "docstring, lang, exported, content_hash, updated_at, valid_from, ingested_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (sid, repo_id, kind, name, fqname, file, span, signature, docstring, - lang, int(exported), content_hash, now_ts()), + lang, int(exported), content_hash, now_ts(), now_ts(), now_ts()), ) if commit: self.conn.commit() @@ -2096,9 +3340,10 @@ def add_code_edge(self, *, repo_id: str, src: str, dst: str, relation: str, if layer is None and graph_layer == GraphLayer.SEMANTIC: graph_layer = GraphLayer.ENTITY self.conn.execute( - "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line) " - "VALUES (?,?,?,?,?,?,?,?)", - (eid, repo_id, src, dst, relation, graph_layer.value, file, line), + "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line, " + "valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?,?)", + (eid, repo_id, src, dst, relation, graph_layer.value, file, line, + now_ts(), now_ts()), ) if commit: self.conn.commit() @@ -2112,14 +3357,22 @@ def get_code_file(self, repo_id: str, file: str) -> Optional[dict]: def list_code_files(self, repo_id: str, *, languages: Optional[set] = None, + flt: Optional[SearchFilter] = None, limit: Optional[int] = None) -> list[dict]: - sql = "SELECT * FROM code_files WHERE repo_id=?" + """Return the current manifest, or its bi-temporal history when anchored.""" + historical = bool(flt and flt.historical) + table = "code_file_history" if historical else "code_files" + sql = f"SELECT * FROM {table} WHERE repo_id=?" params: list[Any] = [repo_id] + if historical: + temporal, temporal_params = _temporal_visibility_sql("", flt) + sql += " AND " + temporal + params.extend(temporal_params) if languages: marks = ",".join("?" for _ in languages) sql += f" AND lang IN ({marks})" params.extend(sorted(languages)) - sql += " ORDER BY file" + sql += " ORDER BY file" + (", version" if historical else "") if limit is not None: sql += " LIMIT ?" params.append(max(0, int(limit))) # never -1 == SQLite "unlimited" @@ -2128,6 +3381,34 @@ def list_code_files(self, repo_id: str, *, def upsert_code_file(self, *, repo_id: str, file: str, lang: str, content_hash: str, size_bytes: int, mtime_ns: int, backend: str, commit: bool = True) -> None: + stamp = now_ts() + current_history = self.conn.execute( + "SELECT version, lang, content_hash, size_bytes, mtime_ns, backend " + "FROM code_file_history WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (repo_id, file), + ).fetchone() + unchanged = current_history is not None and ( + current_history["lang"], current_history["content_hash"], + int(current_history["size_bytes"] or 0), int(current_history["mtime_ns"] or 0), + current_history["backend"] or "", + ) == (lang, content_hash, int(size_bytes), int(mtime_ns), backend) + if not unchanged: + if current_history is not None: + self.conn.execute( + "UPDATE code_file_history SET valid_to=?, valid_to_recorded_at=? " + "WHERE version=?", + (stamp, stamp, current_history["version"]), + ) + self.conn.execute( + "INSERT INTO code_file_history(" + "repo_id, file, lang, content_hash, size_bytes, mtime_ns, backend, " + "indexed_at, valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + repo_id, file, lang, content_hash, int(size_bytes), int(mtime_ns), + backend, stamp, stamp, stamp, + ), + ) self.conn.execute( "INSERT INTO code_files(repo_id, file, lang, content_hash, size_bytes, " "mtime_ns, backend, indexed_at) VALUES (?,?,?,?,?,?,?,?) " @@ -2136,13 +3417,19 @@ def upsert_code_file(self, *, repo_id: str, file: str, lang: str, "size_bytes=excluded.size_bytes, mtime_ns=excluded.mtime_ns, " "backend=excluded.backend, indexed_at=excluded.indexed_at", (repo_id, file, lang, content_hash, int(size_bytes), int(mtime_ns), - backend, now_ts()), + backend, stamp), ) if commit: self.conn.commit() def remove_code_file(self, repo_id: str, file: str, *, commit: bool = True) -> None: self.clear_symbols_for_file(repo_id, file, commit=False) + stamp = now_ts() + self.conn.execute( + "UPDATE code_file_history SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? AND file=? AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, file), + ) self.conn.execute("DELETE FROM code_files WHERE repo_id=? AND file=?", (repo_id, file)) if commit: self.conn.commit() @@ -2159,9 +3446,48 @@ def update_repo_index(self, repo_id: str, *, root_path: str, ) self.conn.commit() - def list_symbols(self, repo_id: str, *, limit: Optional[int] = None) -> list[dict]: - sql = "SELECT * FROM symbols WHERE repo_id=? ORDER BY file, fqname" - params: list[Any] = [repo_id] + def list_symbols(self, repo_id: str, *, limit: Optional[int] = None, + identifiers: Optional[list[str]] = None, + flt: Optional[SearchFilter] = None) -> list[dict]: + """List visible symbols, optionally resolving exact identifiers first. + + ``identifiers`` matches a symbol's ID, short name, or fully-qualified + name. The predicate deliberately precedes ``LIMIT``: callers that + follow a code edge must not lose its endpoint merely because unrelated + files sort earlier in a large repository. + """ + if identifiers is not None: + identifiers = list(dict.fromkeys(value for value in identifiers if value)) + if not identifiers: + return [] + # Three IN predicates consume three bindings per identifier. Keep + # each recursive query below SQLite's conservative parameter limit, + # then apply the requested cap to the merged, ordered result. + chunk_size = max(1, IN_CLAUSE_CHUNK // 3) + if len(identifiers) > chunk_size: + rows_by_id = { + row["id"]: row + for start in range(0, len(identifiers), chunk_size) + for row in self.list_symbols( + repo_id, + identifiers=identifiers[start:start + chunk_size], + flt=flt, + ) + } + rows = sorted(rows_by_id.values(), key=lambda row: ( + row.get("file") or "", row.get("fqname") or "", row.get("id") or "", + )) + return rows if limit is None else rows[:max(0, int(limit))] + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM symbols WHERE repo_id=? AND " + temporal + params = [repo_id, *params] + if identifiers is not None: + marks = ",".join("?" for _ in identifiers) + sql += f" AND (id IN ({marks}) OR name IN ({marks}) OR fqname IN ({marks}))" + params.extend(identifiers) + params.extend(identifiers) + params.extend(identifiers) + sql += " ORDER BY file, fqname" if limit is not None: sql += " LIMIT ?" params.append(max(0, int(limit))) # never -1 == SQLite "unlimited" @@ -2169,9 +3495,11 @@ def list_symbols(self, repo_id: str, *, limit: Optional[int] = None) -> list[dic def list_symbols_page(self, repo_id: str, *, after: Optional[tuple[str, str, str]] = None, - limit: int = 500) -> list[dict]: - sql = "SELECT * FROM symbols WHERE repo_id=?" - params: list[Any] = [repo_id] + limit: int = 500, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM symbols WHERE repo_id=? AND " + temporal + params = [repo_id, *params] if after is not None: file, fqname, symbol_id = after sql += ( @@ -2184,83 +3512,113 @@ def list_symbols_page(self, repo_id: str, *, return [dict(row) for row in self.conn.execute(sql, params).fetchall()] def list_code_edges(self, repo_id: str, *, limit: Optional[int] = None, - layers: Optional[list[GraphLayer]] = None) -> list[dict]: - sql = "SELECT * FROM code_edges WHERE repo_id=?" - params: list[Any] = [repo_id] + layers: Optional[list[GraphLayer]] = None, + endpoints: Optional[list[str]] = None, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM code_edges WHERE repo_id=? AND " + temporal + params = [repo_id, *params] if layers is not None: if not layers: return [] marks = ",".join("?" for _ in layers) sql += f" AND layer IN ({marks})" params.extend(_enum(layer) for layer in layers) + if endpoints is not None: + if not endpoints: + return [] + marks = ",".join("?" for _ in endpoints) + sql += f" AND (src IN ({marks}) OR dst IN ({marks}))" + params.extend(endpoints) + params.extend(endpoints) sql += " ORDER BY file, line, id" if limit is not None: sql += " LIMIT ?" params.append(max(0, int(limit))) # never -1 == SQLite "unlimited" return [dict(r) for r in self.conn.execute(sql, params).fetchall()] - def symbols_for_files(self, repo_id: str, files: list[str]) -> list[dict]: + def symbols_for_files(self, repo_id: str, files: list[str], *, + flt: Optional[SearchFilter] = None) -> list[dict]: if not files: return [] marks = ",".join("?" for _ in files) + temporal, params = _temporal_visibility_sql("", flt) rows = self.conn.execute( f"SELECT * FROM symbols WHERE repo_id=? AND file IN ({marks}) " - "ORDER BY file, fqname", - (repo_id, *files), + f"AND {temporal} ORDER BY file, fqname", + (repo_id, *files, *params), ).fetchall() return [dict(r) for r in rows] def count_code_edges(self, repo_id: str) -> int: row = self.conn.execute( - "SELECT COUNT(*) AS n FROM code_edges WHERE repo_id=?", (repo_id,) + "SELECT COUNT(*) AS n FROM code_edges WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id,) ).fetchone() return int(row["n"]) if row else 0 - def search_symbols(self, repo_id: str, query: str, *, limit: int = 20) -> list[dict]: + def search_symbols(self, repo_id: str, query: str, *, limit: int = 20, + flt: Optional[SearchFilter] = None) -> list[dict]: """Substring match on name/fqname (no embedding yet — v1 is lexical).""" like = f"%{_escape_like(query)}%" + temporal, temporal_params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT * FROM symbols WHERE repo_id=? AND (name LIKE ? ESCAPE '\\' OR fqname LIKE ? ESCAPE '\\') " + f"SELECT * FROM symbols WHERE repo_id=? AND {temporal} " + "AND (name LIKE ? ESCAPE '\\' OR fqname LIKE ? ESCAPE '\\') " "ORDER BY name LIMIT ?", - (repo_id, like, like, limit), + (repo_id, *temporal_params, like, like, limit), ).fetchall() return [dict(r) for r in rows] - def get_symbol_callers(self, repo_id: str, name: str, *, limit: int = 50) -> list[dict]: + def get_symbol_callers(self, repo_id: str, name: str, *, limit: int = 50, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, temporal_params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT * FROM code_edges WHERE repo_id=? AND dst=? AND relation='calls' LIMIT ?", - (repo_id, name, limit), + "SELECT * FROM code_edges WHERE repo_id=? AND dst=? AND relation='calls' " + f"AND {temporal} LIMIT ?", + (repo_id, name, *temporal_params, limit), ).fetchall() return [dict(r) for r in rows] def count_symbols(self, repo_id: str) -> int: row = self.conn.execute( - "SELECT COUNT(*) AS n FROM symbols WHERE repo_id=?", (repo_id,) + "SELECT COUNT(*) AS n FROM symbols WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id,) ).fetchone() return int(row["n"]) if row else 0 def link_memory_symbol(self, *, repo_id: str, symbol_id: str, memory_id: str, relation: str = "mentions", confidence: float = 1.0, commit: bool = True) -> str: + existing = self.conn.execute( + "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " + "AND memory_id=? AND relation=? AND valid_to IS NULL AND expired_at IS NULL", + (repo_id, symbol_id, memory_id, relation), + ).fetchone() + if existing is not None: + return existing["id"] link_id = ids.new_id("edge") + stamp = now_ts() self.conn.execute( "INSERT OR IGNORE INTO code_memory_links(" - "id, repo_id, symbol_id, memory_id, relation, confidence, created_at" - ") VALUES (?,?,?,?,?,?,?)", + "id, repo_id, symbol_id, memory_id, relation, confidence, created_at, " + "valid_from, ingested_at" + ") VALUES (?,?,?,?,?,?,?,?,?)", (link_id, repo_id, symbol_id, memory_id, relation, - max(0.0, min(1.0, float(confidence))), now_ts()), + max(0.0, min(1.0, float(confidence))), stamp, stamp, stamp), ) - row = self.conn.execute( - "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " - "AND memory_id=? AND relation=?", - (repo_id, symbol_id, memory_id, relation), - ).fetchone() if commit: self.conn.commit() - return row["id"] if row else link_id + return link_id def clear_code_memory_links(self, repo_id: str, *, commit: bool = True) -> None: - self.conn.execute("DELETE FROM code_memory_links WHERE repo_id=?", (repo_id,)) + stamp = now_ts() + self.conn.execute( + "UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id), + ) if commit: self.conn.commit() @@ -2269,9 +3627,12 @@ def clear_code_memory_links_for_memories(self, repo_id: str, memory_ids: list[st if not memory_ids: return marks = ",".join("?" for _ in memory_ids) + stamp = now_ts() self.conn.execute( - f"DELETE FROM code_memory_links WHERE repo_id=? AND memory_id IN ({marks})", - (repo_id, *memory_ids), + f"UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + f"WHERE repo_id=? " + f"AND memory_id IN ({marks}) AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, *memory_ids), ) if commit: self.conn.commit() @@ -2280,12 +3641,14 @@ def prune_code_memory_links(self, repo_id: str, *, commit: bool = True) -> None: """Remove bridges whose repo-associated memory is no longer live.""" t = now_ts() self.conn.execute( - "DELETE FROM code_memory_links WHERE repo_id=? AND NOT EXISTS (" + "UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL AND NOT EXISTS (" "SELECT 1 FROM memories AS m WHERE m.id=code_memory_links.memory_id AND m.repo_id=? " "AND (m.valid_from IS NULL OR m.valid_from<=?) " "AND (m.valid_to IS NULL OR ? list[dict]: sql = ( "SELECT l.*, s.name, s.fqname, s.file, s.kind AS symbol_kind, " - "m.title, m.mtype, m.valid_to, m.expired_at " + "m.title, m.mtype, m.valid_to AS memory_valid_to, " + "m.expired_at AS memory_expired_at " "FROM code_memory_links l " - "LEFT JOIN symbols s ON s.id=l.symbol_id " - "LEFT JOIN memories m ON m.id=l.memory_id " + "JOIN symbols s ON s.id=l.symbol_id " + "JOIN memories m ON m.id=l.memory_id " "WHERE l.repo_id=?" ) params: list[Any] = [repo_id] - if flt is not None: - where, visibility_params = self._where(flt, include_invalid=False, alias="m") - if where: - sql += " AND " + " AND ".join(where) - params.extend(visibility_params) + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) + symbol_visibility, symbol_params = _temporal_visibility_sql("s", flt) + sql += " AND " + symbol_visibility + params.extend(symbol_params) + where, visibility_params = self._where(flt, include_invalid=False, alias="m") + if where: + sql += " AND " + " AND ".join(where) + params.extend(visibility_params) sql += " ORDER BY l.created_at, l.id" if limit is not None: sql += " LIMIT ?" @@ -2325,6 +3694,9 @@ def memories_for_symbol(self, repo_id: str, symbol_id: str, *, "WHERE l.repo_id=? AND l.symbol_id=?" ) params: list[Any] = [repo_id, symbol_id] + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) where, visibility_params = self._where(flt, include_invalid=False, alias="m") if where: sql += " AND " + " AND ".join(where) @@ -2341,12 +3713,60 @@ def memories_for_symbol(self, repo_id: str, symbol_id: str, *, out.append(item) return out - def symbols_for_memory(self, repo_id: str, memory_id: str) -> list[dict]: + def memories_for_symbols(self, repo_id: str, symbol_ids: list[str], *, + flt: Optional[SearchFilter] = None, + limit: int = 20) -> dict[str, list[dict]]: + """Return a bounded memory ranking for many symbols in one SQL query.""" + unique_ids = list(dict.fromkeys( + str(symbol_id) for symbol_id in symbol_ids if str(symbol_id) + ))[:500] + if not unique_ids: + return {} + per_symbol_limit = max(1, min(100, int(limit))) + placeholders = ",".join("?" for _ in unique_ids) + sql = ( + "WITH ranked AS (" + "SELECT l.symbol_id, m.id, m.title, m.content, m.mtype, m.scope, " + "m.importance, m.provenance, l.relation, l.confidence, " + "ROW_NUMBER() OVER (PARTITION BY l.symbol_id " + "ORDER BY l.confidence DESC, m.importance DESC, " + "m.ingested_at DESC, l.id, m.id) AS row_rank " + "FROM code_memory_links l JOIN memories m ON m.id=l.memory_id " + f"WHERE l.repo_id=? AND l.symbol_id IN ({placeholders})" + ) + params: list[Any] = [repo_id, *unique_ids] + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) + where, visibility_params = self._where(flt, include_invalid=False, alias="m") + if where: + sql += " AND " + " AND ".join(where) + params.extend(visibility_params) + sql += ( + ") SELECT symbol_id, id, title, content, mtype, scope, importance, " + "provenance, relation, confidence FROM ranked WHERE row_rank<=? " + "ORDER BY symbol_id, row_rank" + ) + params.append(per_symbol_limit) + grouped: dict[str, list[dict]] = {} + for row in self.conn.execute(sql, params).fetchall(): + item = dict(row) + symbol_id = str(item.pop("symbol_id")) + item["provenance"] = _loads(item.get("provenance"), {}) + grouped.setdefault(symbol_id, []).append(item) + return grouped + + def symbols_for_memory(self, repo_id: str, memory_id: str, *, + flt: Optional[SearchFilter] = None) -> list[dict]: + link_visibility, link_params = _temporal_visibility_sql("l", flt) + symbol_visibility, symbol_params = _temporal_visibility_sql("s", flt) rows = self.conn.execute( "SELECT s.*, l.relation, l.confidence FROM code_memory_links l " "JOIN symbols s ON s.id=l.symbol_id " - "WHERE l.repo_id=? AND l.memory_id=? ORDER BY l.confidence DESC, s.fqname", - (repo_id, memory_id), + f"WHERE l.repo_id=? AND l.memory_id=? AND {link_visibility} " + f"AND {symbol_visibility} " + "ORDER BY l.confidence DESC, s.fqname", + (repo_id, memory_id, *link_params, *symbol_params), ).fetchall() return [dict(row) for row in rows] @@ -2403,6 +3823,149 @@ def audit(self, actor: str, action: str, target: str, detail: str = "", if commit: self.conn.commit() + def _backfill_receipt_sequences(self) -> None: + """Assign durable logical ordinals once when the sequence column is introduced.""" + scopes = self.conn.execute( + "SELECT DISTINCT workspace_id FROM operation_receipts" + ).fetchall() + for scope in scopes: + workspace_id = str(scope["workspace_id"] or "") + chain = self._receipt_chain_state(workspace_id) + for sequence, row in enumerate(chain["rows"], 1): + self.conn.execute( + "UPDATE operation_receipts SET sequence=? WHERE id=?", + (sequence, row["id"]), + ) + + def _receipt_chain_state(self, workspace_id: str) -> dict: + """Reconstruct one receipt chain from immutable predecessor hashes. + + SQLite ``rowid`` is physical placement, not durable ordering: VACUUM and table + rewrites may renumber it. The receipt payload already carries the true linked-list + order, while ``receipt_chain_heads`` anchors the expected tail. This helper keeps + traversal independent of storage layout and returns a deterministic fallback order + when corruption makes a single chain impossible. + """ + rows = [dict(row) for row in self.conn.execute( + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts " + "WHERE workspace_id=?", + (workspace_id,), + ).fetchall()] + + def text(value: Any) -> str: + return value if isinstance(value, str) else str(value or "") + + def stable_key(row: dict) -> tuple[str, str]: + material = "\0".join(( + text(row.get("receipt_hash")), + text(row.get("prev_hash")), + text(row.get("id")), + hashlib.sha256(text(row.get("payload")).encode("utf-8")).hexdigest(), + )) + return hashlib.sha256(material.encode("utf-8")).hexdigest(), material + + children: dict[str, list[dict]] = {} + for row in rows: + children.setdefault(text(row.get("prev_hash")), []).append(row) + for candidates in children.values(): + candidates.sort(key=stable_key) + + structure_errors: list[dict] = [] + ordered: list[dict] = [] + roots = children.get("", []) + if rows and len(roots) != 1: + structure_errors.append({ + "index": 0, + "id": "", + "error": "chain_root_count", + }) + if len(roots) == 1: + current = roots[0] + visited_hashes: set[str] = set() + while current is not None: + receipt_hash = text(current.get("receipt_hash")) + if receipt_hash in visited_hashes: + structure_errors.append({ + "index": len(ordered), + "id": text(current.get("id")), + "error": "chain_cycle", + }) + break + visited_hashes.add(receipt_hash) + ordered.append(current) + successors = children.get(receipt_hash, []) + if len(successors) > 1: + structure_errors.append({ + "index": len(ordered) - 1, + "id": text(current.get("id")), + "error": "chain_fork", + }) + break + current = successors[0] if successors else None + + ordered_identity = {id(row) for row in ordered} + if len(ordered) != len(rows): + structure_errors.append({ + "index": len(ordered), + "id": "", + "error": "chain_disconnected", + }) + ordered.extend(sorted( + (row for row in rows if id(row) not in ordered_identity), + key=stable_key, + )) + + row_errors: list[dict] = [] + for index, row in enumerate(ordered): + if type(row.get("sequence")) is not int or row["sequence"] != index + 1: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "sequence_mismatch", + }) + raw = text(row.get("payload")) + stored_hash = text(row.get("receipt_hash")) + if hashlib.sha256(raw.encode("utf-8")).hexdigest() != stored_hash: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "hash_mismatch", + }) + try: + payload = json.loads(raw) + except (TypeError, ValueError, RecursionError): + payload = None + if ( + not isinstance(payload, dict) + or payload.get("id") != row.get("id") + or payload.get("prev_hash") != row.get("prev_hash") + ): + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "payload_mismatch", + }) + if _public_receipt_row(row).get("invalid_payload") is True: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "payload_schema_invalid", + }) + + structurally_valid = not structure_errors and len(ordered) == len(rows) + head = ( + text(ordered[-1].get("receipt_hash")) + if ordered and structurally_valid else "" + ) + return { + "rows": ordered, + "head": head, + "structure_errors": structure_errors, + "row_errors": row_errors, + "errors": [*row_errors, *structure_errors], + } + def record_receipt(self, operation: str, *, workspace_id: str = "", repo_id: str = "", actor: str = "system", target_count: int = 0, status: str = "ok", @@ -2415,7 +3978,24 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", is anchored independently, so modification, reordering, interior deletion, and tail truncation are detectable during verification. """ - operation = str(operation or "unknown")[:80] + operation = str(operation or "unknown") + operation_normalized = operation.strip().casefold() + operation = ( + operation_normalized + if operation_normalized in _PUBLIC_RECEIPT_OPERATIONS + else "sha256:" + hashlib.sha256(operation.encode("utf-8")).hexdigest() + ) + raw_status = str(status or "ok") + status_normalized = raw_status.strip().casefold() + safe_status = ( + status_normalized + if status_normalized in _PUBLIC_RECEIPT_STATUSES + else "sha256:" + hashlib.sha256(raw_status.encode("utf-8")).hexdigest() + ) + try: + safe_target_count = max(0, int(target_count)) + except (TypeError, ValueError, OverflowError): + safe_target_count = 0 actor = str(actor or "system")[:200] workspace_id = str(workspace_id or "") repo_id = str(repo_id or "") @@ -2433,15 +4013,6 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", f"{workspace_id}\0{repo_id}".encode("utf-8") ).hexdigest()[:24] actor_digest = hashlib.sha256(actor.encode("utf-8")).hexdigest()[:16] - chain = self.conn.execute( - "SELECT COUNT(*) AS n, " - "COALESCE((SELECT receipt_hash FROM operation_receipts " - "WHERE workspace_id=? ORDER BY rowid DESC LIMIT 1), '') AS head " - "FROM operation_receipts WHERE workspace_id=?", - (workspace_id, workspace_id), - ).fetchone() - current_count = int(chain["n"] or 0) - prev_hash = str(chain["head"] or "") anchor = self.conn.execute( "SELECT receipt_count, head_hash, integrity_error " "FROM receipt_chain_heads " @@ -2449,15 +4020,72 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", (workspace_id,), ).fetchone() anchor_error = str(anchor["integrity_error"] or "") if anchor else "" - if anchor is not None and ( - int(anchor["receipt_count"]) != current_count - or str(anchor["head_hash"]) != prev_hash - ): - # Preserve evidence of the mismatch without bricking the operation that - # requested this receipt. The new receipt continues from the rows that - # actually remain, while verification stays invalid until an explicit - # repair/export decision clears the persistent integrity marker. - anchor_error = anchor_error or "pre_append_anchor_mismatch" + latest = self.conn.execute( + "SELECT sequence FROM operation_receipts " + "WHERE workspace_id=? ORDER BY sequence DESC LIMIT 1", + (workspace_id,), + ).fetchone() + current_count: Optional[int] = None + prev_hash = "" + if anchor is None and latest is None: + # First receipt for a workspace: no scan and no anchor are expected. + current_count = 0 + elif anchor is not None: + anchor_count = anchor["receipt_count"] + anchor_head = anchor["head_hash"] + if ( + type(anchor_count) is int + and anchor_count == 0 + and anchor_head == "" + and latest is None + ): + current_count = 0 + elif ( + type(anchor_count) is int + and anchor_count > 0 + and latest is not None + and latest["sequence"] == anchor_count + ): + head_row = self.conn.execute( + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts " + "WHERE workspace_id=? AND sequence=?", + (workspace_id, anchor_count), + ).fetchone() + if ( + head_row is not None + and head_row["receipt_hash"] == anchor_head + and not _public_receipt_row(dict(head_row)).get( + "invalid_payload", False + ) + ): + current_count = anchor_count + prev_hash = str(anchor_head) + + if current_count is None: + # The independently stored anchor/ordinal did not describe a healthy + # head. Reconstruct only on this exceptional path so a safe unique + # predecessor can still be extended without retrying the memory action. + chain = self._receipt_chain_state(workspace_id) + if chain["structure_errors"]: + raise sqlite3.IntegrityError( + "receipt chain has no unique structural head; append refused" + ) + current_count = len(chain["rows"]) + prev_hash = str(chain["head"] or "") + if chain["row_errors"]: + anchor_error = anchor_error or "pre_append_chain_corruption" + if anchor is None and current_count: + anchor_error = anchor_error or "pre_append_anchor_missing" + elif anchor is not None and ( + type(anchor["receipt_count"]) is not int + or anchor["receipt_count"] != current_count + or str(anchor["head_hash"]) != prev_hash + ): + # Keep evidence of deletion or anchor damage while extending the + # unique chain that actually remains. + anchor_error = anchor_error or "pre_append_anchor_mismatch" + next_sequence = current_count + 1 safe_meta = _receipt_metadata(metadata or {}) payload_obj = { "version": 1, @@ -2466,8 +4094,8 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", "operation": operation, "scope_digest": scope_digest, "actor_digest": actor_digest, - "target_count": max(0, int(target_count)), - "status": str(status or "ok")[:40], + "target_count": safe_target_count, + "status": safe_status, "metadata": safe_meta, "prev_hash": prev_hash, } @@ -2477,10 +4105,11 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", receipt_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() self.conn.execute( "INSERT INTO operation_receipts(id, ts, operation, workspace_id, repo_id, " - "scope_digest, actor, target_count, status, payload, prev_hash, " - "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + "sequence, scope_digest, actor, target_count, status, payload, prev_hash, " + "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( - receipt_id, ts, operation, workspace_id, repo_id, scope_digest, + receipt_id, ts, operation, workspace_id, repo_id, next_sequence, + scope_digest, actor_digest, payload_obj["target_count"], payload_obj["status"], payload, prev_hash, receipt_hash, ), @@ -2507,40 +4136,21 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", raise def list_receipts(self, *, workspace_id: str, limit: int = 100) -> list[dict]: + safe_limit = max(1, min(10_000, int(limit))) rows = self.conn.execute( - "SELECT payload, receipt_hash FROM operation_receipts WHERE workspace_id=? " - "ORDER BY rowid DESC LIMIT ?", - (workspace_id, max(1, min(10_000, int(limit)))), + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts WHERE workspace_id=? " + "ORDER BY sequence DESC LIMIT ?", + (workspace_id, safe_limit), ).fetchall() - out = [] - for row in rows: - payload = _loads(row["payload"], {}) - if isinstance(payload, dict): - payload["hash"] = row["receipt_hash"] - out.append(payload) - return out + return [_public_receipt_row(dict(row)) for row in rows] def verify_receipts(self, *, workspace_id: str, expected_head: str = "", expected_count: Optional[int] = None) -> dict: - rows = self.conn.execute( - "SELECT id, payload, prev_hash, receipt_hash FROM operation_receipts " - "WHERE workspace_id=? ORDER BY rowid ASC", - (workspace_id,), - ).fetchall() - previous = "" - errors: list[dict] = [] - for index, row in enumerate(rows): - actual = hashlib.sha256(row["payload"].encode("utf-8")).hexdigest() - if actual != row["receipt_hash"]: - errors.append({"index": index, "id": row["id"], "error": "hash_mismatch"}) - payload = _loads(row["payload"], {}) - if not isinstance(payload, dict) or payload.get("id") != row["id"] \ - or payload.get("prev_hash") != row["prev_hash"]: - errors.append({"index": index, "id": row["id"], - "error": "payload_mismatch"}) - if row["prev_hash"] != previous: - errors.append({"index": index, "id": row["id"], "error": "chain_break"}) - previous = row["receipt_hash"] + chain = self._receipt_chain_state(workspace_id) + rows = chain["rows"] + errors: list[dict] = list(chain["errors"]) + head = str(chain["head"] or "") anchor = self.conn.execute( "SELECT receipt_count, head_hash, integrity_error " "FROM receipt_chain_heads WHERE workspace_id=?", @@ -2549,11 +4159,12 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", if rows and anchor is None: errors.append({"index": len(rows), "id": "", "error": "missing_anchor"}) elif anchor is not None: - if int(anchor["receipt_count"]) != len(rows): + anchor_count = anchor["receipt_count"] + if type(anchor_count) is not int or anchor_count < 0 or anchor_count != len(rows): errors.append({ "index": len(rows), "id": "", "error": "anchor_count_mismatch", }) - if str(anchor["head_hash"]) != previous: + if str(anchor["head_hash"]) != head: errors.append({ "index": len(rows), "id": "", "error": "anchor_head_mismatch", }) @@ -2562,7 +4173,7 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", "index": len(rows), "id": "", "error": "anchor_integrity_error", }) expected_head = str(expected_head or "").strip() - if expected_head and previous != expected_head: + if expected_head and head != expected_head: errors.append({ "index": len(rows), "id": "", "error": "expected_head_mismatch", }) @@ -2578,7 +4189,7 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", return { "valid": not errors, "count": len(rows), - "head": previous, + "head": head, "anchored": anchor is not None, "errors": errors, } @@ -2656,12 +4267,19 @@ def _where(self, flt: Optional[SearchFilter], include_invalid: bool, where.append(f"{p}mtype IN ({marks})") params.extend(_enum(m) for m in flt.mtypes) if not include_invalid: - t = (flt.as_of if flt and flt.as_of is not None else now_ts()) + valid_at, known_at = _temporal_anchors(flt) where.append(f"({p}valid_from IS NULL OR {p}valid_from<=?)") - params.append(t) - where.append(f"({p}valid_to IS NULL OR ?<{p}valid_to)") - params.append(t) - where.append(f"{p}expired_at IS NULL") + params.append(valid_at) + where.append( + f"({p}valid_to IS NULL OR ?<{p}valid_to OR " + f"({p}valid_to_recorded_at IS NOT NULL " + f"AND ?<{p}valid_to_recorded_at))" + ) + params.extend((valid_at, known_at)) + where.append(f"({p}ingested_at IS NULL OR {p}ingested_at<=?)") + params.append(known_at) + where.append(f"({p}expired_at IS NULL OR ?<{p}expired_at)") + params.append(known_at) return where, params @@ -2681,7 +4299,13 @@ def _row_to_record(row: sqlite3.Row) -> MemoryRecord: importance=row["importance"], surprise=row["surprise"], stability=row["stability"], access_count=row["access_count"], last_access=row["last_access"], valid_from=row["valid_from"], valid_to=row["valid_to"], + valid_to_recorded_at=( + row["valid_to_recorded_at"] + if "valid_to_recorded_at" in row.keys() else None + ), ingested_at=row["ingested_at"], expired_at=row["expired_at"], + subject_key=row["subject_key"] if "subject_key" in row.keys() else "", + claim_kind=row["claim_kind"] if "claim_kind" in row.keys() else "", pinned=bool(row["pinned"]), sensitivity=row["sensitivity"], provenance=_loads(row["provenance"], {}), ) @@ -2696,6 +4320,10 @@ def _row_to_edge(row: sqlite3.Row) -> Edge: weight=row["weight"], workspace_id=row["workspace_id"] if "workspace_id" in row.keys() else None, repo_id=row["repo_id"] if "repo_id" in row.keys() else None, valid_from=row["valid_from"], valid_to=row["valid_to"], + valid_to_recorded_at=( + row["valid_to_recorded_at"] + if "valid_to_recorded_at" in row.keys() else None + ), ingested_at=row["ingested_at"], expired_at=row["expired_at"], provenance=_loads(row["provenance"], {}), ) diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 9b3b9014..7a08bd6b 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -56,7 +56,8 @@ # ── bundle format ───────────────────────────────────────────────────────────── SYNC_FORMAT = "engraphis-sync" -SYNC_VERSION = 1 +SYNC_VERSION = 2 +SYNC_ACCEPTED_VERSIONS = frozenset({1, 2}) # ── validation caps (untrusted bundle → clamp, don't trust) ─────────────────── MAX_MEMORIES = 200_000 @@ -90,7 +91,7 @@ _LWW_FIELDS = ( "title", "content", "summary", "keywords", "metadata", "mtype", "scope", "importance", "surprise", "sensitivity", "valid_from", "ingested_at", - "session_id", "provenance", + "session_id", "provenance", "subject_key", "claim_kind", ) @@ -135,6 +136,7 @@ def _label_tuple(rec: MemoryRecord) -> list: rec.title, rec.content, rec.summary, sorted(rec.keywords or []), _enum(rec.mtype), _enum(rec.scope), rec.importance, rec.surprise, rec.sensitivity, rec.valid_from, rec.session_id, + rec.subject_key, rec.claim_kind, json.dumps(rec.metadata or {}, sort_keys=True, default=str), json.dumps(rec.provenance or {}, sort_keys=True, default=str), ] @@ -156,6 +158,7 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: are taken from ``local`` here and never LWW-merged, so re-homing is never undone. """ winner = local if _version_key(local) >= _version_key(incoming) else incoming + valid_to, valid_to_recorded_at = _merge_closure(local, incoming) return MemoryRecord( id=local.id, # scope pointers are always local — never merged from the remote @@ -167,6 +170,7 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: mtype=winner.mtype, scope=winner.scope, importance=winner.importance, surprise=winner.surprise, sensitivity=winner.sensitivity, session_id=winner.session_id, provenance=dict(winner.provenance or {}), + subject_key=winner.subject_key, claim_kind=winner.claim_kind, valid_from=winner.valid_from, # ``ingested_at`` is a LWW field (_LWW_FIELDS), NOT a lattice field, and it is the # SECOND component of _version_key. Merging it as a min-lattice made @@ -180,12 +184,40 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: # entirely from the winner. ingested_at=winner.ingested_at, # lattice fields: commutative joins (independent of the LWW winner) - valid_to=_min_nonnull(local.valid_to, incoming.valid_to), + valid_to=valid_to, expired_at=_min_nonnull(local.expired_at, incoming.expired_at), stability=max(local.stability, incoming.stability), access_count=max(local.access_count, incoming.access_count), last_access=_max_nonnull(local.last_access, incoming.last_access), pinned=bool(local.pinned or incoming.pinned), + valid_to_recorded_at=valid_to_recorded_at, + ) + + +def _merge_closure( + local: MemoryRecord, incoming: MemoryRecord, +) -> tuple[Optional[float], Optional[float]]: + """Join a world-time closure with the system-time at which it was learned. + + The earliest world-time closure wins. Its knowledge timestamp must travel + with it; independently learned equal closures use the earliest timestamp. + A missing timestamp is legacy v1 state whose closure was always visible, so + it remains ``None`` rather than being silently assigned a later time. + """ + if local.valid_to is None: + return incoming.valid_to, ( + incoming.valid_to_recorded_at if incoming.valid_to is not None else None + ) + if incoming.valid_to is None: + return local.valid_to, local.valid_to_recorded_at + if local.valid_to < incoming.valid_to: + return local.valid_to, local.valid_to_recorded_at + if incoming.valid_to < local.valid_to: + return incoming.valid_to, incoming.valid_to_recorded_at + if local.valid_to_recorded_at is None or incoming.valid_to_recorded_at is None: + return local.valid_to, None + return local.valid_to, min( + local.valid_to_recorded_at, incoming.valid_to_recorded_at ) @@ -228,7 +260,8 @@ def inherit_store_defaults(existing: MemoryRecord, incoming: MemoryRecord) -> Me def _signature(rec: MemoryRecord) -> str: """Fingerprint of everything sync persists — to tell 'changed' from 'no-op'.""" return _stable_hash(_label_tuple(rec) + [ - rec.valid_to, rec.expired_at, rec.ingested_at, rec.stability, + rec.valid_to, rec.valid_to_recorded_at, rec.expired_at, + rec.ingested_at, rec.stability, rec.access_count, rec.last_access, bool(rec.pinned), ]) @@ -244,8 +277,10 @@ def record_to_dict(rec: MemoryRecord) -> dict: "importance": rec.importance, "surprise": rec.surprise, "stability": rec.stability, "access_count": rec.access_count, "last_access": rec.last_access, "valid_from": rec.valid_from, "valid_to": rec.valid_to, + "valid_to_recorded_at": rec.valid_to_recorded_at, "ingested_at": rec.ingested_at, "expired_at": rec.expired_at, "pinned": bool(rec.pinned), "sensitivity": rec.sensitivity, + "subject_key": rec.subject_key, "claim_kind": rec.claim_kind, "provenance": rec.provenance or {}, } @@ -424,9 +459,12 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]: # (they are the version key's primary ordering / anti-poison defense). valid_from=_clamp_world_ts(d.get("valid_from")), valid_to=_clamp_world_ts(d.get("valid_to")), + valid_to_recorded_at=_clamp_ts(d.get("valid_to_recorded_at"), now), ingested_at=_clamp_ts(d.get("ingested_at"), now), expired_at=_clamp_ts(d.get("expired_at"), now), pinned=bool(d.get("pinned")), sensitivity=sens, + subject_key=_clamp_str(d.get("subject_key"), 512), + claim_kind=_clamp_str(d.get("claim_kind"), 256), provenance=_safe_json_obj(d.get("provenance")), ) @@ -483,7 +521,7 @@ def export_bundle(self, workspace_id: str, *, repo_id: Optional[str] = None) -> repo_rows = self.store.conn.execute( "SELECT id, name FROM repos WHERE workspace_id=?", (workspace_id,)).fetchall() ids_in = [m.id for m in mems] - links = self.store.links_among(ids_in) if ids_in else [] + links = self.store.links_among(ids_in, include_invalid=True) if ids_in else [] return { "format": SYNC_FORMAT, "version": SYNC_VERSION, "device_id": self.device_id, "created_at": now_ts(), @@ -495,6 +533,11 @@ def export_bundle(self, workspace_id: str, *, repo_id: Optional[str] = None) -> "a": ln["a"], "b": ln["b"], "relation": ln["relation"], "layer": ln.get("layer") or "semantic", "reason": ln.get("reason") or "", + "valid_from": ln.get("valid_from"), + "valid_to": ln.get("valid_to"), + "valid_to_recorded_at": ln.get("valid_to_recorded_at"), + "ingested_at": ln.get("ingested_at"), + "expired_at": ln.get("expired_at"), } for ln in links ], @@ -514,7 +557,7 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, raise SyncError("bundle is not an object") if bundle.get("format") != SYNC_FORMAT: raise SyncError("not an %s bundle" % SYNC_FORMAT) - if _as_int(bundle.get("version"), 0) != SYNC_VERSION: + if _as_int(bundle.get("version"), 0) not in SYNC_ACCEPTED_VERSIONS: raise SyncError("unsupported bundle version %r" % bundle.get("version")) src_device = bundle.get("device_id") @@ -664,6 +707,15 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, # overwrite the local private row with a non-session scope either. report["rejected"] += 1 return + if existing is not None: + # Sync v1 bundles predate durable claim identity. Omission means + # "unknown to this peer", not an instruction to erase local keys. + if "subject_key" not in d: + rec.subject_key = existing.subject_key + if "claim_kind" not in d: + rec.claim_kind = existing.claim_kind + if "valid_to_recorded_at" not in d and rec.valid_to == existing.valid_to: + rec.valid_to_recorded_at = existing.valid_to_recorded_at if (existing is not None and only_repo_id is not None and existing.repo_id != only_repo_id): # The incoming row's claimed repo cannot re-home an existing memory from @@ -733,9 +785,51 @@ def _apply_links(self, link_dicts: list, report: dict, accepted: dict, if not dry_run: self.store.conn.commit() pending = 0 + # v2 bundles carry a complete bi-temporal link version. Preserve it + # verbatim (after the normal untrusted-input clamps), including closed + # intervals. v1 omitted these fields, so it retains the established + # grow-only/current-link merge below. + if ("valid_from" in ln and "ingested_at" in ln + and _clamp_world_ts(ln.get("valid_from")) is not None + and _clamp_ts(ln.get("ingested_at"), now_ts()) is not None): + valid_from = _clamp_world_ts(ln.get("valid_from")) + valid_to = _clamp_world_ts(ln.get("valid_to")) + valid_to_recorded_at = _clamp_ts(ln.get("valid_to_recorded_at"), now_ts()) + ingested_at = _clamp_ts(ln.get("ingested_at"), now_ts()) + expired_at = _clamp_ts(ln.get("expired_at"), now_ts()) + existing_version = self.store.conn.execute( + "SELECT 1 FROM mem_links " + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND layer=? AND reason=? AND valid_from IS ? AND valid_to IS ? " + "AND valid_to_recorded_at IS ? AND ingested_at IS ? AND expired_at IS ? " + "LIMIT 1", + ( + a, b, b, a, rel, layer, reason, + valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, + ), + ).fetchone() + if existing_version: + continue + if not dry_run: + inserted = self.store.add_link_version( + a, b, rel, layer=layer, reason=reason, + valid_from=valid_from, valid_to=valid_to, + valid_to_recorded_at=valid_to_recorded_at, + ingested_at=ingested_at, expired_at=expired_at, + commit=False, + ) + if inserted: + self.store.audit( + "sync:%s" % _clamp_str(src_device or "peer", 128), + "sync_link", a, + f"linked to {b} with relation {rel}", commit=False) + report["links_added"] += 1 + continue existing_link = self.store.conn.execute( "SELECT layer, reason FROM mem_links " - "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? LIMIT 1", + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND valid_to IS NULL AND expired_at IS NULL " + "ORDER BY rowid DESC LIMIT 1", (a, b, b, a, rel), ).fetchone() if existing_link: diff --git a/engraphis/dashboard_assets/engraphis-graph.js b/engraphis/dashboard_assets/engraphis-graph.js index 3284b3c4..aa56d250 100644 --- a/engraphis/dashboard_assets/engraphis-graph.js +++ b/engraphis/dashboard_assets/engraphis-graph.js @@ -92,6 +92,45 @@ function idOf(value) { return value && typeof value === 'object' ? value.id : value; } function nodeName(node) { return String(node.name || node.label || node.id || ''); } + /* Replace force-graph's round flow particles with a small directional glyph. The vendor + callback supplies the particle's current position and its link; the context already has + the resolved particle colour, so this only changes the silhouette and orientation. */ + function paintFlowArrow(x, y, link, ctx, globalScale) { + const source = link && link.source; + const target = link && link.target; + if (!source || !target || !Number.isFinite(source.x) || !Number.isFinite(target.x)) return; + const dx = target.x - source.x; + const dy = target.y - source.y; + if (!dx && !dy) return; + const size = 1 / Math.sqrt(Math.max(0.01, Number(globalScale) || 1)); + const angle = Math.atan2(dy, dx); + ctx.save(); + ctx.translate(x, y); + ctx.rotate(angle); + ctx.beginPath(); + ctx.moveTo(size * 0.55, 0); + ctx.lineTo(-size * 0.45, size * 0.32); + ctx.lineTo(-size * 0.45, -size * 0.32); + ctx.closePath(); + ctx.fill(); + ctx.restore(); + } + /* Keep node geometry in the same compact world-space range as the Classic/Ledger renderer. + The previous overview formula used the full size-slider value plus a normalized degree + bonus, which made a seven-node workspace occupy only a small simulation area while each + node still had a dense-graph radius. `zoomToFit()` then magnified those radii into large + discs. Material style must not change geometry; it only changes the painted surface. */ + function graphNodeRadius(node, base, metric) { + const size = Number.isFinite(+base) && +base > 0 ? +base : 3; + if (node && node.cluster) { + const members = Math.max(1, Number(node.members) || 1); + const radius = size * 0.45 * (1.4 + Math.min(3, Math.sqrt(members) * 0.7)); + return Math.max(2, Math.min(size * 2.7, radius)); + } + const normalized = Math.max(0, Math.min(1, Number(metric) || 0)); + const radius = size * 0.45 * (0.55 + Math.min(1.6, normalized * 1.9)); + return Math.max(0.8, Math.min(size * 1.1, radius)); + } function linkEndpoint(link, side) { return idOf(link[side] !== undefined ? link[side] : link[side === 'source' ? 'from' : 'to']); } @@ -1196,7 +1235,6 @@ paintMaterialSurface(ctx, node.x, node.y, r, scale, nodeMaterial, materialLow); } else if (state.styleName === 'solar') { const sun = node.rank === 0; - if (sun) r *= 1.7; nodeMaterial = materialRecipe( 'solar', state.themeColors, state.palette, sun ? mixColours(col, '#d38b43', 0.46) : col @@ -1340,9 +1378,7 @@ const sizeMetric = n => state.sizeBy === 'betweenness' ? (n.betweenness || 0) : ((n.degree || 0) / Math.max(1, maxDeg)); data.nodes.forEach(n => { const base = (state.settings.size || 3); - n.radius = n.cluster - ? Math.max(3, base * (1.4 + Math.min(3, Math.sqrt(n.members || 1) * 0.7))) - : Math.max(0.8, base * (0.55 + Math.min(1.6, sizeMetric(n) * 1.9))); + n.radius = graphNodeRadius(n, base, sizeMetric(n)); n.color = nodeColor(n); n.stroke = contrastOn(n.color); }); @@ -1379,7 +1415,7 @@ if (fg.linkCurvature) { fg.linkCurvature(dense ? 0 : ((PRESETS[state.settings.mode] || PRESETS.compact).curve || 0)); } - fg.linkDirectionalArrowLength(dense ? 0 : 2.5).linkDirectionalArrowRelPos(1); + fg.linkDirectionalArrowLength(dense ? 0 : 0.625).linkDirectionalArrowRelPos(1); applyLinkLabels(); if (fg.linkDirectionalParticles) { const flowing = !fullGraph @@ -1390,7 +1426,8 @@ ? 0 : (state.styleName === 'cyber' ? 3 : ((PRESETS[state.settings.mode] || {}).particles || 2)); fg.linkDirectionalParticles(l => l.suggested || l.ghost ? 0 : particles) - .linkDirectionalParticleWidth(2) + .linkDirectionalParticleWidth(1) + .linkDirectionalParticleCanvasObject(paintFlowArrow) .linkDirectionalParticleColor(l => alpha(layerColor(l.layer), 0.95)) .linkDirectionalParticleSpeed(l => 0.002 + ((state.settings.flowSpeed || 45) / 100) * 0.008); } @@ -1804,6 +1841,7 @@ Nothing in the dashboard uses these; treat them as the engine's unit-test seam. */ _internals: { esc, hexRgb, alpha, contrastOn, communities, betweenness, findBridges, maxOf, + graphNodeRadius, paintFlowArrow, nodeName, linkEndpoint, asOfValue, materialRecipe, materialTier, paintMaterialDirect, renderMaterialSample, sampleMaterialColour, materialCacheStats, clearMaterialCache, setMaterialCanvasFactory diff --git a/engraphis/dashboard_assets/index.html b/engraphis/dashboard_assets/index.html index dd6270fb..0d19f388 100644 --- a/engraphis/dashboard_assets/index.html +++ b/engraphis/dashboard_assets/index.html @@ -14,7 +14,7 @@