diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 523f2311..e2eebe60 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "engraphis-memory", "source": "./", "description": "Discipline for giving agents durable, scoped, explainable memory across sessions and repos with the Engraphis MCP tools.", - "version": "1.4.0" + "version": "1.4.5" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5a4093a4..88751d85 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "engraphis-memory", - "version": "1.4.0", + "version": "1.4.5", "description": "Give agents durable, scoped, explainable memory across sessions and repos via the Engraphis MCP tools. Use when you learn something worth keeping, need prior context before acting, or ask why/how a fact changed. Covers remember/recall, why/timeline, forget/pin/correct, sessions, and code search.", "author": { "name": "The Engraphis Authors", diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256 index daf8b9a6..9220bbd0 100644 --- a/.claude-plugin/skill-assets.sha256 +++ b/.claude-plugin/skill-assets.sha256 @@ -1,6 +1,6 @@ -b3122186525b688060558721dadf8ca4a20e192097556adb1daecca0649a4e28 .claude-plugin/marketplace.json -5a870fabc9814e177a570a8878371d1c4c50a5b245076c2cfbb7ca659e41ebf6 .claude-plugin/plugin.json -911c70ead2c5aa3de24a6c645a9e921382a149aba52b0a9582ecd5b560e5b8a8 skills/engraphis-memory/SKILL.md +e7e4ecd111d9b04c290ddd60e0fadb90e3afd8c67e39dcb8fbce0b50b5e3ce42 .claude-plugin/marketplace.json +65bff1596f3db2bc75b74c6970d87e806d46ef1cb3e612cd2002f19c3a8f6acb .claude-plugin/plugin.json +56be8d078a2a8fc6e6cd1c2be5716605d8621dab953caa8cfcd20e2dce474305 skills/engraphis-memory/SKILL.md 45dd73ca6afdd9e12ecd38c48e4a612b7646c25a07a75a80ca0e68d0e0b85f0e skills/engraphis-memory/references/CONVENTIONS.md 529fff3bdbe73f83209087fd10055fad77c5e5224ad8a9e6b0254052aa50e109 skills/engraphis-memory/references/SCOPING.md eecd861f0f8cc2a9def07a53387ca66d8cb68d8b62d9b048dcd1b0b250fa3fee skills/engraphis-memory/references/TOOLS.md diff --git a/.dockerignore b/.dockerignore index ac120352..7848fc73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,8 @@ __pycache__ *.db *.db-wal *.db-shm +*.db-journal +*.bak .env .venv venv @@ -14,3 +16,50 @@ docs/_build *.egg-info build dist +# Test and local tooling output. +.coverage +.coverage.* +htmlcov +.tox +.nox +.hypothesis +.audit-venv +models_cache +.secrets +internal +.commandcode + +# Machine-local runtime state and credentials. +.engraphis_processed_webhooks +undelivered_license_keys.tsv +automation.json +autosync.json +.engraphis_update_check.json +cookies.txt + +# Private research and local demo material. +/COMMERCIAL_AUDIT.md +/COMPETITIVE_ANALYSIS.md +/docs/COMMERCIAL_AUDIT.md +/docs/COMPETITIVE_ANALYSIS.md +/demo/generated/ +/demo/output/ +/demo/assets/ +/demo/*_killer.html +/demo/*_social_demo.html + +# Local QA/eval/automation state — never part of the build context. +node_modules +integrations/pi/node_modules +test-results +playwright-report +.playwright +.private-eval +.hosted-eval-results +/.tmp-*/ +/.tmp_*/ +/.release-full-tmp/ +/_to_delete/ +*.log +*.whl +*.tar.gz diff --git a/.env.example b/.env.example index 2bb0c15b..4acc32d3 100644 --- a/.env.example +++ b/.env.example @@ -21,9 +21,10 @@ ENGRAPHIS_SERVICE_MODE=customer # Behind Traefik, use its LAN hostname instead: # ENGRAPHIS_DASHBOARD_URL=http://engraphis.local -# Update reminder. When on (default), the server checks for a newer Engraphis release -# once a day and surfaces it in the dashboard banner, the startup log, and over MCP. -# The check is fail-silent and cached; set to 0 to disable all update network activity. +# Update reminder. It is OFF by default, so a local installation makes no update-related +# network request. Set this to 1 to check for a newer Engraphis release once a day and +# surface it in the dashboard banner, startup log, and over MCP. The check is cached +# and fail-silent. # ENGRAPHIS_UPDATE_CHECK=1 # Override the release source. Default: the GitHub releases/latest API for the project # repo. Accepts any HTTPS endpoint returning a GitHub-release, PyPI, or @@ -32,6 +33,11 @@ ENGRAPHIS_SERVICE_MODE=customer # Point the default GitHub source at a different owner/repo (ignored when # ENGRAPHIS_UPDATE_URL is set). Default: Coding-Dev-Tools/engraphis. # ENGRAPHIS_UPDATE_REPO=Coding-Dev-Tools/engraphis +# Which extras the self-updater installs on top of the base package. The installer +# cannot see which extras the current install selected, so it defaults to the safe +# superset `engraphis[all]`; set a comma-separated list (e.g. `server,mcp`) to pin +# a smaller surface, or `none` for the base package only. +# ENGRAPHIS_UPDATE_EXTRAS= # Optional local API bearer. If set, supported protected routes accept # Authorization: Bearer . Use a strong, independently revocable value and do not @@ -70,10 +76,16 @@ ENGRAPHIS_API_TOKEN= ENGRAPHIS_EMBED_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Embedding dimension is auto-detected from the model. Override only if needed. # ENGRAPHIS_EMBED_DIM=384 +# Vector index backend for the v2 engine: "numpy" (default, deterministic reference +# index), "sqlite-vec" (require the accelerated ANN backend; needs the sqlite-vec +# package), or "auto" (use sqlite-vec when available, fall back to NumPy). +# ENGRAPHIS_VECTOR_BACKEND=numpy # ── LLM (external, you choose the provider) ───────────────────────────────── # Provider: openai | anthropic | google | openrouter | custom # Copy-ready provider setups and endpoint requirements: docs/LLM_PROVIDERS.md +# Codex subscription users: connect Codex to this installation over MCP; the subscription +# path does not use ENGRAPHIS_LLM_PROVIDER or an Engraphis LLM API key. # ── v2 write-path fact extraction (optional) ───────────────────────────────── # "none" (default): store text as given. "chunk": deterministic offline chunks. # "llm": free-form fact extraction. "llm_structured": schema-validated typed facts, @@ -97,6 +109,9 @@ ENGRAPHIS_GRAPH_EXTRACTOR=regex # "llm" sends a bounded excerpt to the configured provider for an advisory # ephemeral/normal/critical signal. Writes are never discarded. ENGRAPHIS_RETENTION_SUPERVISOR=none +# A remote retention supervisor is advisory: without this opt-in, its "critical" +# recommendations keep normal retention strength. Set 1 to honor critical signals. +# ENGRAPHIS_ALLOW_AUTOMATIC_CRITICAL_RETENTION=0 # Optional local resource adapters: # ENGRAPHIS_WHISPER_MODEL=/absolute/path/to/local-whisper-model @@ -112,6 +127,19 @@ ENGRAPHIS_RETENTION_SUPERVISOR=none # ENGRAPHIS_GRAPH_HOST=127.0.0.1 # ENGRAPHIS_GRAPH_PORT=8720 +# Standalone MCP-over-HTTP server (`engraphis-mcp-http`). Loopback-only by default; +# any non-loopback bind (via these or ENGRAPHIS_HOST) requires ENGRAPHIS_API_TOKEN. +# ENGRAPHIS_HTTP_HOST=127.0.0.1 +# ENGRAPHIS_HTTP_PORT=8080 +# ENGRAPHIS_HTTP_TRANSPORT=streamable-http + +# When running under Docker (auto-detected via /.dockerenv), the self-updater skips +# in-place pip upgrades and prints a manual reinstall hint instead. +# ENGRAPHIS_DOCKER=0 + +# Docker Compose host port mapping override for the dashboard (default 8700). +# ENGRAPHIS_COMPOSE_PORT=8700 + # ── Reverse proxy (TLS termination) ───────────────────────────────────────── # When behind a proxy that terminates TLS (Railway/Fly/nginx), trust its # X-Forwarded-Proto/-For headers so request.url.scheme is https and the session @@ -135,6 +163,9 @@ ENGRAPHIS_RETENTION_SUPERVISOR=none # ENGRAPHIS_CSP="" # send no CSP at all # ENGRAPHIS_HSTS="max-age=31536000; includeSubDomains" +# Codex subscription users can use the local MCP connection documented in README.md and +# docs/AGENT_CONNECT.md; leave the optional external LLM settings below disabled unless +# Engraphis itself must call a separate provider. ENGRAPHIS_LLM_PROVIDER=openai # Model name (provider-specific): # openai: gpt-4o-mini, gpt-4o, gpt-4.1-mini, o4-mini ... @@ -143,17 +174,12 @@ ENGRAPHIS_LLM_PROVIDER=openai # openrouter: anthropic/claude-3.5-sonnet, openai/gpt-4o-mini ... # custom: any model name your OpenAI-compatible endpoint accepts ENGRAPHIS_LLM_MODEL=gpt-4o-mini -# API key for chat/synthesis, llm/llm_structured extraction, and structured consolidation: -ENGRAPHIS_LLM_API_KEY=sk-your-key-here +# API key for chat/synthesis, llm/llm_structured extraction, and structured consolidation. +# Leave unset until you explicitly choose a provider and enable an LLM-backed feature. +# ENGRAPHIS_LLM_API_KEY= # For openrouter / custom: the base URL of the OpenAI-compatible endpoint. # openrouter: https://openrouter.ai/api/v1 # custom: https://your-endpoint/v1 -# ollama: http://localhost:11434/v1 -# Ollama example (replace the model with one you have pulled): -# ENGRAPHIS_LLM_PROVIDER=custom -# ENGRAPHIS_LLM_MODEL=qwen2.5-coder:latest -# ENGRAPHIS_LLM_API_KEY=ollama # must be non-empty; default local Ollama ignores it -# ENGRAPHIS_LLM_BASE_URL=http://localhost:11434/v1 # ENGRAPHIS_LLM_BASE_URL=https://openrouter.ai/api/v1 # Optional: extra headers (JSON string) for custom providers. # ENGRAPHIS_LLM_EXTRA_HEADERS={"HTTP-Referer":"https://myapp.com","X-Title":"engraphis"} @@ -277,6 +303,12 @@ ENGRAPHIS_LLM_API_KEY=sk-your-key-here # ENGRAPHIS_CHUNK_TOKENIZER_REVISION= # ENGRAPHIS_LOOP_INTERVAL=60 # ENGRAPHIS_LOOP_TOP_K=20 +# Automatic local consolidation inside the background loop. OFF by default (0): the +# sweep is a workspace-wide cluster scan, so it should be an explicit operator choice. +# N > 0 runs it at most once every N loop ticks (e.g. 30 with a 60s interval ≈ every +# 30 minutes). The sweep is deterministic/offline (never passes an LLM), archives +# decayed transients, and distills recurring episodic memories into semantic digests. +# ENGRAPHIS_LOOP_CONSOLIDATE=0 # ENGRAPHIS_RERANK_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 # Workspace allow-list: comma-separated names. Empty = all allowed. @@ -286,6 +318,11 @@ ENGRAPHIS_LLM_API_KEY=sk-your-key-here # ENGRAPHIS_RELAY_URL=https://relay.example.com # ENGRAPHIS_SYNC_TOKEN= # ENGRAPHIS_SYNC_READ_ONLY=0 +# End-to-end encryption key for Cloud Sync bundles (relay transport). A single +# immutable 32-byte URL-safe base64 value (43 chars, or 44 with one '=' pad) that +# every authorized device shares; changing it makes previously stored ciphertext +# unreadable. The folder transport does not encrypt at rest — see docs/SYNC.md. +# ENGRAPHIS_SYNC_E2EE_KEY= # Hosted plan upgrade URLs: override the default upgrade landing pages. # ENGRAPHIS_UPGRADE_URL= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02676df3..d7425b9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,8 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml - name: Install (core + server/mcp/code extras; no torch or SQLCipher) run: | python -m pip install --upgrade pip @@ -54,6 +56,8 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml - name: Install encryption integration gate run: | python -m pip install --upgrade pip @@ -72,6 +76,8 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.9" + cache: pip + cache-dependency-path: pyproject.toml - name: Install (numpy-only core — the minimum supported runtime) run: | python -m pip install --upgrade pip @@ -83,6 +89,37 @@ jobs: - name: Ablation run: python -m eval.ablation + coverage: + name: coverage gate (Python 3.11) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" pytest-cov + - name: Coverage run (all extras-gated tests, tracked modules) + run: python -m pytest -o addopts="" tests/ -q -rs --cov=engraphis --cov-report=term-missing --cov-fail-under=60 + + hygiene: + name: repo hygiene gate (no stray DBs/logs) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Reject stray runtime artifacts at repo root + run: | + stray=$(find . -maxdepth 1 -type f \( -name '*.db' -o -name '*.db-wal' -o -name '*.db-shm' \ + -o -name '*.bak' -o -name '*.log' \) -print) + if [ -n "$stray" ]; then + echo "Refusing to commit stray runtime artifacts:"; echo "$stray"; exit 1 + fi + echo "repo root clean" + pi-extension: name: Pi extension (${{ matrix.os }}, Python ${{ matrix.python-version }}, Node ${{ matrix.node-version }}) runs-on: ${{ matrix.os }} diff --git a/AGENTS.md b/AGENTS.md index 1b9e88ba..56e9bbf1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ most common mistake here. | Status | Primary scoped, bi-temporal, interface-driven implementation. | Compatibility/reference implementation with flat namespaces. | | Model | Scoped + bi-temporal + typed; interface-driven. | Single flat `namespace` string per memory. | | Code | `engraphis/core/`, `engraphis/backends/`, `eval/`, `tests/`, `scripts/migrate_to_v2.py` | `engraphis/app.py`, `config.py`, `models.py`, `routes/`, `stores/`, `engines/`, `llm/`, `static/` | -| Data | new v2 schema (`SCHEMA_VERSION = 7`) | `engraphis_v1.db` | +| Data | new v2 schema (`SCHEMA_VERSION = 9`) | `engraphis_v1.db` | | Entry | `MemoryEngine.create()` → `core/engine.py` | Internal reference only; never a public launcher | **Rule:** build new capability on **v2** (`core/` + `backends/`) behind the interfaces. @@ -35,7 +35,7 @@ task is ambiguous, decide which side it belongs to *before* editing. # ── Install ────────────────────────────────────────────────────────────────── pip install numpy pytest # v2 core + tests, fully OFFLINE (this is what CI does) pip install -e ".[all,dev]" # full stack: FastAPI server, ST embeddings, ruff -cp .env.example .env # only needed for the v1 server / LLM features +cp .env.example .env # optional; configure server, LLM, encryption, or hosted client settings # ── Quality gate (offline, no API key — KEEP THIS GREEN; mirrors .github/workflows/ci.yml) ── python -m pytest tests/ -q # unit tests (offline) @@ -182,7 +182,7 @@ These are pure, unit-tested functions — change them only with a corresponding --- -## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 7`) +## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 9`) - **Scope hierarchy:** `workspace → repo → session → memory`. Scopes: `session|repo|workspace|user`. - **Bi-temporal validity on every record:** world-time `valid_from/valid_to` + @@ -194,8 +194,7 @@ These are pure, unit-tested functions — change them only with a corresponding - **Tables:** `workspaces`, `repos`, `sessions`, `memories`, `mem_vectors`, `embedding_state`, `mem_fts` (FTS5 + plain-table fallback), `entities`, `edges` (bi-temporal), `mem_links`, `memory_entities`, `symbols`, `code_edges`, `code_files`, `code_memory_links`, - `operation_receipts`, - `events`, `audit`, `schema_migrations`. + `operation_receipts`, `events`, `audit`, `memory_tombstones`, `schema_migrations`. - **Vectors are stored L2-normalized** so cosine similarity == dot product. --- @@ -230,8 +229,6 @@ These are pure, unit-tested functions — change them only with a corresponding - **`docs/HOSTED_PLANS.md`** — concise pricing, plan contents, trial, and hosted-service boundary. - **`docs/MCP_TOOLS.md`** — standalone inventory of the public MCP surface; keep it synchronized with `engraphis/mcp_server.py`. -- **`docs/OLLAMA.md`** — local Ollama configuration. Keep setup details here instead of - duplicating them in the README. - **`docs/SYNC.md`** — cloud sync (Pro): architecture, the convergent merge, CLI usage, and the untrusted-bundle security model. - **`AGENTS.md`** (this file) + **`CLAUDE.md`** — how to work in the repo. diff --git a/CHANGELOG.md b/CHANGELOG.md index 549b1bd9..d99f17ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,28 +5,43 @@ All notable changes to Engraphis are documented here. Format loosely follows ## [Unreleased] +## [1.4.5] - 2026-08-04 + +Patch release aligning the package, runtime, commercial manifest, and plugin metadata at 1.4.5 +for the governed recall/write hardening, schema 8 migration, Smart MCP gateway fixes, and +credential-safe evaluation capture included in PR #111. +Schema 9 adds repository-scoped tombstone support and performs a one-time entity-canonicalization +repair; `confidence` and `pinned_at`/`unpinned_at` were introduced by the preceding v7-to-v8 +migration. Known-repository tombstones are terminal only within that repository, while legacy +repo-less tombstones remain global. + ## [1.4.0] - 2026-08-02 Engraphis 1.4 makes the compact Smart MCP gateway the default agent interface while preserving -the complete Classic surface for existing integrations. It also strengthens review-gated writes, -bounded context delivery, secure erasure, and release/runtime hardening without changing the v2 -database schema. +the complete Classic surface for existing integrations. It also strengthens external-write +governance, +bounded context delivery, secure erasure, and release/runtime hardening, and moves the v2 SQLite +schema to version 9 (schema-level additions include repository-scoped `memory_tombstones`; the +upgrade also performs a one-time entity-canonicalization repair), which migrates automatically on +first open. Known-repository tombstones are terminal only within that repository; legacy repo-less +tombstones remain global. ### Upgrade notes -- `engraphis-mcp` now exposes six Smart tools instead of 33 direct tools. Clients that depend on +- `engraphis-mcp` now exposes nine Smart tools instead of 33 direct tools. Clients that depend on the former names should switch their server command to `engraphis-mcp-classic`; HTTP clients can use `engraphis-mcp-http --classic`. -- Existing v2 databases remain on schema 7 and require no migration for this release. +- Existing v2 databases migrate automatically to schema 9 on first open; the change is additive + and requires no manual step. - The NumPy-only core supports Python 3.9+. Dashboard, MCP, documents, Cloud Sync, and `all` installations require Python 3.10+ because their supported dependency versions require it. ### Added -- Smart MCP is now the zero-configuration `engraphis-mcp` default. It exposes six compact tools - for sessions, prompt-ready recall, durable memory, discovery, and validated read/action - execution. `engraphis-mcp-classic` preserves the former 33 direct tool names and legacy alias - response shapes for pinned integrations. +- Smart MCP is now the zero-configuration `engraphis-mcp` default. It exposes nine compact tools: + sessions, prompt-ready recall, durable memory, discovery, validated read/action execution, and + governed record read/update plus conflict review. `engraphis-mcp-classic` preserves the former 33 + direct tool names and legacy alias response shapes for pinned integrations. - The first-party `@engraphis/pi` package under `integrations/pi` exposes that Smart MCP surface as native Pi tools, verifies the Engraphis 1.4.x handshake, and ships with independent npm packaging and release gates. @@ -36,7 +51,7 @@ database schema. - Opt-in planned recall adds a bounded deterministic planner, an injectable planner protocol and optional LLM backend, priority-weighted multi-query RRF, post-rerank memory-type maxima, stable context revisions, and diagnostics-only planner traces across Python, service, REST, and MCP - recall surfaces. The default remains the existing single-query path on schema 7. + recall surfaces. The default remains the existing single-query path (now on schema 9). - A 40-task context-routing stress fixture, four-way five-budget ablation harness, pinned LongMemEval-V2 planner configurations, and evaluation-only imported-resource hierarchy prototype encode local regression gates and matrix tooling. Official benchmark, safety, and hosted-cache diff --git a/README.md b/README.md index 8171620b..ebeb74db 100644 --- a/README.md +++ b/README.md @@ -124,10 +124,21 @@ continues to support Python 3.9+. For MCP clients other than Codex, configure a stdio server whose command is `engraphis-mcp`; see the [agent connection guide](docs/AGENT_CONNECT.md). -> **Upgrading to 1.4:** `engraphis-mcp` now exposes the six-tool Smart gateway. Integrations that +### Updating + +Use `engraphis-update` to upgrade the installation using its detected install method. Package +metadata does not record which extras were selected, so the updater defaults to the safe +superset `engraphis[all]` rather than silently dropping an optional surface. For a deliberate +selection, set `ENGRAPHIS_UPDATE_EXTRAS` to a comma-separated list (for example +`server,mcp`), or set it to `none` for the base package only. + +> **Upgrading to 1.4:** `engraphis-mcp` now exposes the nine-tool Smart gateway. Integrations that > require the former 33 direct tool names should run `engraphis-mcp-classic`. The SQLite schema -> remains version 7, so this MCP surface change does not require a data migration. See the -> [1.4.0 release notes](CHANGELOG.md#140---2026-08-02). +> moves to version 9. Existing v7-to-v8 databases already contain `confidence` and +> `pinned_at`/`unpinned_at`; v9 adds the `memory_tombstones` repository-scope column/table support +> and performs a one-time entity-canonicalization repair, then migrates automatically on first +> open. A tombstone with a known `repo_id` is terminal only in that repository; legacy repo-less +> tombstones remain global. See the [1.4.0 release notes](CHANGELOG.md#140---2026-08-02). --- @@ -203,8 +214,8 @@ outcomes, never keys, prompts, or raw provider responses. See the > `chunk` extractor when ingestion must remain entirely local. Choose and configure an external LLM with the [LLM provider guide](docs/LLM_PROVIDERS.md), -including OpenAI, Anthropic, Google, OpenRouter, Ollama, Cohere Command, Command Code, and -compatible endpoints. +including OpenAI, Anthropic, Google, OpenRouter, Ollama, Cohere Command, Command Code Provider, +and other compatible endpoints. The guide also covers Codex subscription MCP connections. --- @@ -242,7 +253,9 @@ for the reproducible commands and reporting limits. `engraphis[encryption]` installs the driver. The cross-platform `all` extra deliberately omits it so `all` remains resolvable on macOS, Windows, Linux ARM, and musl; on those targets, provision a compatible SQLCipher driver separately before enabling a database -key. Plaintext SQLite remains the explicit default on every platform. +key. The programmatic core remains plaintext unless a database key is configured. For a +fresh database, `engraphis-init` enables SQLCipher automatically when a compatible driver is +available, creates a private key sidecar, and can be overridden with `--no-encryption`. > **Linux / macOS:** if `pip install` fails with `error: externally-managed-environment`, > your system Python is marked read-only (PEP 668). Install into a virtual environment @@ -255,6 +268,10 @@ key. Plaintext SQLite remains the explicit default on every platform. > `semantic_support=false`, and disable vector retrieval plus semantic-cosine evidence. Install > a declared embedding model for semantic retrieval. +> To require a model that is already local, set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path` +> or `local:`. This path never downloads a model. If it is unavailable, Engraphis +> explicitly enters lexical degraded mode instead of presenting hash-vector scores as semantic. + --- ## Quickstart: dashboard @@ -291,14 +308,15 @@ install premium server implementations into this image. See `docker-compose.yml` pip install "engraphis[mcp]" engraphis-init # writes .env + prints config snippets claude mcp add engraphis -- engraphis-mcp -cmd mcp add engraphis -- engraphis-mcp # Command Code CLI -``` +codex mcp add engraphis -- engraphis-mcp # Codex subscription -For Command Code scopes, verification, and its optional Provider API setup, see the -[Command Code section of the LLM provider guide](docs/LLM_PROVIDERS.md#command-code). +``` +For Codex subscription setup and verification, see the [agent connection guide](docs/AGENT_CONNECT.md) +and the [LLM provider guide](docs/LLM_PROVIDERS.md). -`engraphis-mcp` is zero-configuration Smart MCP: agents begin with six compact tools for sessions, -prompt-ready recall, durable memory, action discovery, and safe execution. For code graphs, +`engraphis-mcp` is zero-configuration Smart MCP: agents begin with nine compact tools for sessions, +prompt-ready recall, durable memory, governed record read/update, conflict review, action discovery, +and safe execution. For code graphs, governance, audit, or other advanced work, the agent calls `engraphis_discover_actions` and then the indicated read or action executor; no profile selection is required. The gateway validates the discovered capability again before it runs it, and clients remain responsible for their @@ -313,6 +331,14 @@ including `engraphis_check_update`, is in the [MCP tool reference](docs/MCP_TOOL For installation, configuration, lifecycle commands, and the local trust boundary, see the [Pi extension guide](integrations/pi/README.md). +### Hermes provider + +Engraphis also ships a native Hermes memory-provider plugin with local prefetch, bounded turn +capture, scoped recall, and explicit secure erase. Install Engraphis in the Hermes Python +environment, copy the provider, then select it with `hermes memory setup`. See the +[Hermes integration guide](integrations/hermes/README.md). The provider never installs itself or +downloads an embedding model. + ## Quickstart: repository graph ```bash diff --git a/docs/AGENT_CONNECT.md b/docs/AGENT_CONNECT.md index ad0c8452..560a888f 100644 --- a/docs/AGENT_CONNECT.md +++ b/docs/AGENT_CONNECT.md @@ -15,7 +15,7 @@ claude mcp add engraphis -- engraphis-mcp ``` The local server exposes the same memory semantics while keeping the database on your machine. -It is Smart MCP by default: agents use the six compact routine tools and discover/execute advanced +It is Smart MCP by default: agents use the nine compact routine tools and discover/execute advanced capabilities automatically when needed. There is no profile choice or manual escalation. If a legacy client pins direct tool names, configure `engraphis-mcp-classic` instead. Use `ENGRAPHIS_API_TOKEN` only when protecting a local HTTP surface; it is not a Team identity or diff --git a/docs/ARCHITECTURE_V3.md b/docs/ARCHITECTURE_V3.md index f95c74d7..01652bd3 100644 --- a/docs/ARCHITECTURE_V3.md +++ b/docs/ARCHITECTURE_V3.md @@ -7,7 +7,7 @@ retention-supervision, and privacy-receipt additions introduced with schema vers flowchart LR Agent["Agent / host LLM"] --> Intent["remember · link · recall_context (compact) · recall"] CLI["engraphis-graph CLI"] --> Service["MemoryService"] - MCP["Smart MCP (6 tools) / Classic MCP (33 tools)"] --> Service + MCP["Smart MCP (9 tools) / Classic MCP (33 tools)"] --> Service HTTP["Dashboard + read-only graph HTTP"] --> Service Import["Local resources / PostgreSQL catalog"] --> Extractors["Optional local extractors"] Extractors --> Service diff --git a/docs/KILO_CODE_INTEGRATION.md b/docs/KILO_CODE_INTEGRATION.md index 44929937..176ffe2d 100644 --- a/docs/KILO_CODE_INTEGRATION.md +++ b/docs/KILO_CODE_INTEGRATION.md @@ -41,7 +41,7 @@ Everything runs on your machine. The whole store is a single SQLite file. Local You interact with Engraphis through three surfaces, all backed by the *same* engine (`MemoryService`), so they can never drift apart: - **The dashboard WebUI** (`engraphis-dashboard`, `http://127.0.0.1:8700`): a visual product to see, search, and curate memory. -- **The MCP server** (`engraphis-mcp`): a six-tool Smart gateway for routine memory work plus +- **The MCP server** (`engraphis-mcp`): a nine-tool Smart gateway for routine memory work plus automatic discovery and validated execution of advanced capabilities. **This is the surface Kilo Code uses.** - **The Python library** (`from engraphis.service import MemoryService`): for direct programmatic use. @@ -92,7 +92,7 @@ engraphis-init This gives you a console command, `engraphis-mcp`, which is the zero-configuration Smart MCP server (it speaks stdio, exactly the transport Kilo Code's "Local (STDIO)" type expects). It starts -with six compact tools; the agent discovers and executes code, governance, audit, and other +with nine compact tools; the agent discovers and executes code, governance, audit, and other advanced actions as needed. You can sanity-check that it's on your PATH: ```bash @@ -191,7 +191,7 @@ You can also click **Approve Always** on any tool at runtime to write the same r ## 4. Smart tools and the Classic compatibility surface -Normal `engraphis-mcp` setup exposes exactly these six Smart tools. Routine memory work stays +Normal `engraphis-mcp` setup exposes exactly these nine Smart tools. Routine memory work stays compact; for everything else, discovery returns the exact schema, capability ID, and side-effect class, and the appropriate executor revalidates all of it before running. @@ -203,6 +203,9 @@ class, and the appropriate executor revalidates all of it before running. | `engraphis_discover_actions` | Find the best advanced capability and its exact schema. | | `engraphis_execute_read` | Run a discovered read-only/idempotent capability. | | `engraphis_execute_action` | Run a discovered stateful, administrative, or destructive-capable action. | +| `engraphis_get_memory` | Read one memory's governed record (content, provenance, scope, temporal fields). | +| `engraphis_update_memory` | Edit a memory's metadata fields (title, type, importance). | +| `engraphis_conflict_review` | List pending/quarantined/conflicted records for review (read-only inbox). | `engraphis-mcp-classic` is only for an existing configuration that pins direct tool names. It preserves the former 33-tool surface below; new Kilo Code installations should keep the zero-config diff --git a/docs/LLM_PROVIDERS.md b/docs/LLM_PROVIDERS.md index f3b327f8..202373ee 100644 --- a/docs/LLM_PROVIDERS.md +++ b/docs/LLM_PROVIDERS.md @@ -1,12 +1,11 @@ -# LLM providers and Command Code +# LLM providers and coding-agent connections Engraphis runs fully locally by default. An LLM is optional and is used only when you opt into LLM extraction, cited synthesis, structured consolidation, or retention supervision. Memory storage, local embeddings, conflict resolution, and recall do not require a provider. -This is the complete provider reference. It also covers Command Code both as an MCP coding agent -and as an optional OpenAI-compatible model provider. Command Code and Cohere Command are distinct -products and use different setup paths. +This is the complete provider reference. It also explains how to connect a Codex subscription +through MCP. Codex and Cohere Command are distinct products and use different setup paths. ## Contents @@ -19,6 +18,7 @@ products and use different setup paths. - [Ollama](#ollama) - [Cohere Command](#cohere-command) - [Other OpenAI-compatible endpoints](#other-openai-compatible-endpoints) +- [Codex subscription](#codex-subscription) - [Command Code](#command-code) ## Choose a provider @@ -65,6 +65,10 @@ Provider errors do not expose API keys, configured endpoint URLs, or raw provide dashboard. Features that support a local fallback degrade safely when a provider is unavailable; confirm a successful connection before depending on LLM extraction in a workflow. +For an OpenAI Codex subscription, use the local MCP connection documented in the Codex section +below. The subscription path does not use ``ENGRAPHIS_LLM_PROVIDER`` or an Engraphis LLM API key; +those settings remain for optional LLM calls made by Engraphis itself. + ## OpenAI OpenAI uses the native `openai` mode. Leave `ENGRAPHIS_LLM_BASE_URL` unset unless you deliberately @@ -124,9 +128,9 @@ through a compatible proxy. If that proxy needs extra headers, set ## Ollama -Ollama is a local, OpenAI-compatible endpoint. It uses `custom`, not a separate `ollama` runtime -mode. Start Ollama and pull a chat model, then replace `` below with an installed -model name. +Ollama is a local OpenAI-compatible endpoint. Configure it as `custom`, not as a separate +provider value. Start Ollama and pull a chat model, then replace `` with an installed +model name: ```dotenv ENGRAPHIS_LLM_PROVIDER=custom @@ -135,10 +139,10 @@ ENGRAPHIS_LLM_API_KEY=ollama ENGRAPHIS_LLM_BASE_URL=http://localhost:11434/v1 ``` -The key must be non-empty because the custom client requires a bearer token. Default local Ollama -does not authenticate it; use a real proxy token if you place Ollama behind an authenticated proxy. -The base URL ends in `/v1` because Engraphis adds `/chat/completions`. Loopback `http` is allowed -for local services; a non-loopback endpoint must use HTTPS. +The custom client requires a non-empty key, although the default local Ollama server does not +authenticate it; use a real proxy token if you put Ollama behind an authenticated proxy. The base +URL ends in `/v1` because Engraphis appends `/chat/completions`. Loopback `http` is allowed for a +local service; a non-loopback endpoint must use HTTPS. ## Cohere Command @@ -179,6 +183,21 @@ limits. Endpoints that implement another protocol, such as Anthropic Messages, n native mode or adapter. If a test fails, confirm the base URL, model, credential, required headers, and request and response shapes. +## Codex subscription + +Codex subscription users connect the coding agent to Engraphis over MCP. This keeps the +conversation and subscription inside Codex; Engraphis runs locally and receives no Codex API key. + +```bash +pip install "engraphis[mcp]" +engraphis-init +codex mcp add engraphis -- engraphis-mcp +``` + +Verify the connection with ``codex mcp list`` and then call an Engraphis MCP tool from a +tool-enabled Codex session. The MCP connection is the supported Codex-subscription path; do not +configure a local model endpoint or invent an ``ENGRAPHIS_LLM_PROVIDER=codex`` value. + ## Command Code Command Code and Cohere Command are separate products. There are two ways to combine Command Code diff --git a/docs/MCP_TOOLS.md b/docs/MCP_TOOLS.md index f63e22f7..5ec78878 100644 --- a/docs/MCP_TOOLS.md +++ b/docs/MCP_TOOLS.md @@ -1,12 +1,30 @@ # MCP tool reference -`engraphis-mcp` is the zero-configuration Smart MCP gateway. It initially exposes six concise +`engraphis-mcp` is the zero-configuration Smart MCP gateway. It initially exposes nine concise tools: `engraphis_session`, `engraphis_recall_context`, `engraphis_remember`, -`engraphis_discover_actions`, `engraphis_execute_read`, and `engraphis_execute_action`. Agents use +`engraphis_discover_actions`, `engraphis_execute_read`, `engraphis_execute_action`, +`engraphis_get_memory`, `engraphis_update_memory`, and `engraphis_conflict_review`. Agents use the routine tools directly; for any advanced capability, they discover the best action and execute the returned, version-bound capability ID. Discovery returns the precise schema and side-effect class, and execution revalidates availability, scope, authorization, and arguments. +### Smart tool inventory + +| Tool | What it does | +|---|---| +| `engraphis_session` | Starts or resumes a session, or ends it with a next-session handoff. | +| `engraphis_recall_context` | Returns one compact, bounded context packet for routine agent work. | +| `engraphis_remember` | Stores a routine durable memory with safe default provenance and deduplication. | +| `engraphis_discover_actions` | Returns exact schemas for a small set of matching advanced actions. | +| `engraphis_execute_read` | Executes only a discovered action that is read-only and idempotent. | +| `engraphis_execute_action` | Executes a discovered write, admin, or destructive-capable action. | +| `engraphis_get_memory` | Returns one governed memory record, excluding non-prompt-eligible content. | +| `engraphis_update_memory` | Edits memory metadata; content changes use the governed correction path. | +| `engraphis_conflict_review` | Lists pending, quarantined, or conflicting memories for review. | + +The Smart gateway exposes these nine tools directly; advanced capabilities remain available through +discovery and the validated executors. + No user profile choice or tool switching is required. The dashboard `/mcp` endpoint and `engraphis-mcp-http` use this Smart surface by default. `engraphis-mcp-classic` (or `engraphis-mcp-http --classic`) preserves the 33 direct tools below for integrations that pin @@ -30,24 +48,26 @@ is feature hashing with lexical overlap). In that mode vector retrieval and sema evidence are disabled; recall remains lexical/graph/code based and grounded answers use lexical support only. -Trust boundary: every MCP write is `pending` review, regardless of a caller-supplied `source` or -`trusted` label. The same rule applies to REST/dashboard-intent, import, sync, and extractor -ingress; detector matches are `quarantined` immediately. Pending and quarantined records are -available only to explicit inspection workflows and never appear in prompt-ready MCP recall or -context, `engraphis_why`, or `engraphis_timeline`, nor can they feed resolution, links, -graph/code backfill, or derived prompt context. `include_untrusted=True` is inspection-only and -must never be copied into a model prompt. +Trust boundary: normal local-agent memory creation is prompt-visible immediately after validation; +it does not require owner approval. The default `agent` source covers `engraphis_remember`, +`engraphis_ingest`, and dashboard intent writes. External sources remain `pending` regardless of +a caller-supplied `trusted` label, and detector matches are `quarantined` immediately. Pending +and quarantined records are available only to explicit inspection workflows and never appear in +prompt-ready MCP recall or context, `engraphis_why`, or `engraphis_timeline`, nor can they feed +resolution, links, graph/code backfill, or derived prompt context. `include_untrusted=True` is +inspection-only and must never be copied into a model prompt. -MCP deliberately has no approval tool. Approval creates a fresh, audited `approved` successor -while retaining the reviewed source and its provenance. In the local product it is available only -through the CSRF-bound dashboard review action (with `ENGRAPHIS_API_TOKEN`) or the interactive -TTY command `python -m scripts.approve_memory MEM_ID --reason "..."`; the command rejects -redirected input and requires a typed confirmation. Hosted approval is an owner/admin action of -the private hosted service. Direct in-process `MemoryEngine` use is a trusted-code boundary for -code that already has local database authority, not a transport permission. +MCP deliberately has no approval tool. Approval is only for external or quarantined evidence: it +creates a fresh, audited `approved` successor while retaining the reviewed source and its +provenance. In the local product it is available only through the CSRF-bound dashboard review +action (with `ENGRAPHIS_API_TOKEN`) or the interactive TTY command +`python -m scripts.approve_memory MEM_ID --reason "..."`; the command rejects redirected input +and requires a typed confirmation. Hosted approval is an owner/admin action of the private hosted +service. Direct in-process `MemoryEngine` use is a trusted-code boundary for code that already has +local database authority, not a transport permission. -For the full public-write review and existing-store migration procedure, see the -[public write review gate](WRITE_REVIEW.md). +For the full memory trust model and existing-store migration procedure, see the +[memory write trust model](WRITE_REVIEW.md). | Category | Tool | What it does | |---|---|---| diff --git a/docs/WRITE_REVIEW.md b/docs/WRITE_REVIEW.md index 50756bb3..b8bec9a4 100644 --- a/docs/WRITE_REVIEW.md +++ b/docs/WRITE_REVIEW.md @@ -1,19 +1,24 @@ -# Public write review gate +# Memory write trust model ## MCP, REST, imports, and sync -Every public write enters review as `pending`, regardless of a caller-supplied `source` or -`trusted` label. That includes MCP, dashboard/REST intent writes, imports, sync, and extractor -output. Detector matches are instead `quarantined` immediately. Pending and quarantined records -remain inspectable and auditable, but cannot enter model-ready recall/context, resolution, +Normal local-agent memory creation is immediate. The `agent` and `intent_api` service sources are +stamped `trusted` + `approved` after normal validation, so an agent can create and recall a memory +without waiting for an owner. The write is still scoped, audited, deduplicated, and subject to the +deterministic poisoning guard. + +External/imported sources (`web`, `import`, `sync`, `tool`, `api`, `mcp`, and extractor/introspector +feeds) remain `pending`; detector matches are `quarantined` immediately. Pending and quarantined +records remain inspectable and auditable, but cannot enter model-ready recall/context, resolution, links, graph/code backfill, derived prompt context, or public `why`/`timeline` history. -Corrections, promotions, and merges fail closed unless every input is explicitly approved. +Corrections, promotions, and merges fail closed when their inputs are untrusted or quarantined. -Approval creates a fresh `approved` successor and preserves the reviewed source plus an audit -link; it never relabels the source in place. There is deliberately no MCP tool or general REST -approval endpoint. A local owner can approve through the dashboard's **Approve for prompt** -action after configuring `ENGRAPHIS_API_TOKEN` (short-lived browser session plus CSRF confirmation), -or from an interactive terminal: +Approval is only for releasing an external or quarantined record. It creates a fresh `approved` +successor and preserves the reviewed source plus an audit link; it never relabels the source in +place. There is deliberately no MCP tool or general REST approval endpoint. A local owner can +approve through the dashboard's **Approve for prompt** action after configuring +`ENGRAPHIS_API_TOKEN` (short-lived browser session plus CSRF confirmation), or from an interactive +terminal: ```bash python -m scripts.approve_memory mem_... --reason "verified against the owner runbook" diff --git a/engraphis/__init__.py b/engraphis/__init__.py index 13a49146..76e2bef7 100644 --- a/engraphis/__init__.py +++ b/engraphis/__init__.py @@ -2,7 +2,7 @@ from importlib.metadata import PackageNotFoundError, version as _dist_version -_SOURCE_VERSION = "1.4.0" +_SOURCE_VERSION = "1.4.5" try: __version__ = _dist_version("engraphis") @@ -14,4 +14,4 @@ except PackageNotFoundError: # source tree without an installed distribution # Keep in step with [project] version in pyproject.toml — tests/test_packaging.py # pins the two together so a release cannot ship them out of sync. - __version__ = "1.4.0" + __version__ = "1.4.5" diff --git a/engraphis/app.py b/engraphis/app.py index 3edf6b8c..6254924c 100644 --- a/engraphis/app.py +++ b/engraphis/app.py @@ -26,6 +26,7 @@ from engraphis.routes.memory import router as memory_router from engraphis.routes.vault import VAULT_UPLOAD_REQUEST_BYTES, router as vault_router from engraphis.stores import get_conn, init_db +from engraphis.core.interfaces import SearchFilter logger = logging.getLogger("engraphis") @@ -205,13 +206,20 @@ async def _lifespan(app: FastAPI): first), then start the background consolidation loop unless it's disabled. Shutdown: cancel and await the loop.""" global _background_task + _background_task = None + background_task: Optional[asyncio.Task] = None init_db() # Warm the embedding model eagerly so the first recall call isn't paid # under request pressure (a cold load + concurrent call used to wedge # the forked PM2 worker and time out every recall). await asyncio.get_running_loop().run_in_executor(None, _warmup_embedder) if settings.loop_interval > 0: - _background_task = asyncio.create_task(_consciousness_loop()) + background_task = asyncio.create_task( + _consciousness_loop( + enable_consolidation=not bool(getattr(app.state, "legacy_reference", False)) + ) + ) + _background_task = background_task logger.info("Background consciousness loop started (interval=%ds)", settings.loop_interval) else: logger.info("Background loop disabled (ENGRAPHIS_LOOP_INTERVAL=0)") @@ -223,12 +231,14 @@ async def _lifespan(app: FastAPI): try: yield finally: - if _background_task: - _background_task.cancel() + if background_task is not None: + background_task.cancel() try: - await _background_task + await background_task except asyncio.CancelledError: pass + if _background_task is background_task: + _background_task = None def _build_legacy_reference_app() -> FastAPI: @@ -250,6 +260,10 @@ def _build_legacy_reference_app() -> FastAPI: docs_url=None, redoc_url=None, ) + # This app is the retired v1 compatibility surface. Its lifespan still owns the + # legacy decay/thought loop, but it must never ask the v2 service factory to open + # this v1 database: MemoryService.create() would auto-migrate the file in place. + app.state.legacy_reference = True # Local-first CORS: loopback by default, override with ENGRAPHIS_CORS_ORIGINS. # Credentials are only allowed when the allow-list is explicit (never with "*"). @@ -406,12 +420,23 @@ async def dashboard(): return app -async def _consciousness_loop() -> None: - """Phase 2 + Phase 4 background cycle: decay → thought synthesis → reweight.""" +async def _consciousness_loop(*, enable_consolidation: bool = True) -> None: + """Phase 2 + Phase 4 background cycle: decay → thought synthesis → reweight. + + Phase 3 (consolidation) is an opt-in extra: when ``ENGRAPHIS_LOOP_CONSOLIDATE`` is + set to N > 0 the loop runs one local consolidation sweep at most once every N ticks. + The sweep is gated behind a cheap candidate pre-check so an idle database never pays + for the workspace-wide cluster scan, and the expensive work itself runs in a worker + thread (``asyncio.to_thread``) so the event loop — and therefore every request — is + never blocked by it. Any consolidation failure is logged and swallowed: it must never + kill the loop or poison the decay/thought cadence. + """ _consecutive_errors = 0 + _ticks = 0 while True: try: await asyncio.sleep(settings.loop_interval) + _ticks += 1 touched = reweight.decay_pass(namespace=None) if touched: logger.info("Decay pass: %d memories reweighted", touched) @@ -426,6 +451,9 @@ async def _consciousness_loop() -> None: "Thought synthesized and persisted (sources=%d)", int(result.get("source_count") or 0), ) + if (enable_consolidation and settings.loop_consolidate > 0 + and _ticks % settings.loop_consolidate == 0): + await _maybe_consolidate() _consecutive_errors = 0 except asyncio.CancelledError: raise @@ -437,6 +465,107 @@ async def _consciousness_loop() -> None: await asyncio.sleep(backoff) +def _loop_consolidation_candidates(engine) -> int: + """Cheap pre-check: count live, prompt-eligible memories a sweep could act on. + + The consolidation sweep itself scans up to ``DISTILL_SCAN_LIMIT`` episodic records + and runs a Jaccard cluster pass — expensive work that should never run when there is + nothing to do. This mirrors the sweep's two inputs with two bounded COUNT(*) queries + (episodic records for pass 1 distillation, transient records for pass 2 archival; + both in the same maintenance scopes the sweep uses, prompt-only like the sweep's + reads). Pinned memories are ignored here on purpose: they are archival-exempt, and + counting them could only turn an empty sweep into a non-empty pre-check. + """ + from engraphis.core.consolidate import MAINTENANCE_SCOPES, TRANSIENT_TYPES + from engraphis.core.interfaces import MemoryType + + count = 0 + for mtype in (MemoryType.EPISODIC, *TRANSIENT_TYPES): + count += store_count_prompt_eligible( + engine.store, + SearchFilter(scopes=MAINTENANCE_SCOPES, mtypes=[mtype]), + ) + return count + + +def store_count_prompt_eligible(store, flt: SearchFilter) -> int: + """Count live memories matching ``flt`` that are also prompt-eligible. + + ``store.count_memories`` does not know about the provenance/review-state gate, so a + pending-only workspace would otherwise pass the pre-check while the sweep (whose + reads are ``prompt_only=True``) sees nothing. Kept as a module-level helper so tests + can exercise the SQL against a real store. + """ + from engraphis.core.store import _row_is_prompt_eligible + + where, params = store._where(flt, include_invalid=False) + sql = "SELECT provenance, metadata FROM memories" + if where: + sql += " WHERE " + " AND ".join(where) + rows = store.conn.execute(sql, params).fetchall() + return sum(1 for row in rows if _row_is_prompt_eligible(row["provenance"], row["metadata"])) + + +def _consolidation_candidates_exist(engine) -> bool: + """True when at least one workspace holds consolidation-eligible memories.""" + return _loop_consolidation_candidates(engine) > 0 + + +def _run_loop_consolidation(engine) -> None: + """One deterministic (LLM-free) consolidation sweep over every workspace. + + Runs inside ``asyncio.to_thread`` from the loop. The sweep is a blocking, CPU-bound + scan plus SQLite writes, so it must never run on the event loop. Every workspace in + the database is swept with no LLM (``structured``/``profiles``/``infer`` stay off) — + the same conservative defaults the explicit ``scripts/consolidate.py`` uses. + """ + rows = engine.store.conn.execute("SELECT id, name FROM workspaces").fetchall() + workspaces = [(row["id"], row["name"]) for row in rows] + for wid, name in workspaces: + try: + report = engine.consolidate(workspace_id=wid, dry_run=False) + created = len(report.get("digests_created") or []) + archived = len(report.get("archived") or []) + if created or archived: + logger.info( + "Auto-consolidation: workspace '%s' distilled=%d archived=%d", + name, created, archived, + ) + except Exception as exc: # noqa: BLE001 — one workspace must not block the rest + logger.error("Auto-consolidation failed for workspace '%s' (%s)", + name, type(exc).__name__) + + +async def _maybe_consolidate() -> None: + """Opt-in Phase 3: run the local consolidation sweep when candidates exist. + + Fail-safe by construction: every failure path is caught and logged so an error can + never propagate into (and kill) ``_consciousness_loop``. The pre-check and the sweep + both run in worker threads — the check is cheap but still a SQL scan, and the sweep + is deliberately heavy. + """ + try: + from engraphis.routes import v2_api + + svc = v2_api.service() + engine = svc.engine + except Exception as exc: # noqa: BLE001 + logger.warning("Auto-consolidation skipped (service unavailable: %s)", + type(exc).__name__) + return + try: + has_candidates = await asyncio.to_thread(_consolidation_candidates_exist, engine) + if not has_candidates: + return + except Exception as exc: # noqa: BLE001 + logger.error("Auto-consolidation pre-check failed (%s)", type(exc).__name__) + return + try: + await asyncio.to_thread(_run_loop_consolidation, engine) + except Exception as exc: # noqa: BLE001 + logger.error("Auto-consolidation sweep failed (%s)", type(exc).__name__) + + def _create_retired_direct_app() -> FastAPI: """Retire the old ``uvicorn engraphis.app:app`` deployment target safely.""" retired = FastAPI( diff --git a/engraphis/backends/embedder_deterministic.py b/engraphis/backends/embedder_deterministic.py index d1bc6d63..f93f3f92 100644 --- a/engraphis/backends/embedder_deterministic.py +++ b/engraphis/backends/embedder_deterministic.py @@ -14,7 +14,7 @@ import hashlib from numbers import Integral import re -from typing import Literal +from typing import Literal, Optional import numpy as np @@ -34,12 +34,12 @@ class DeterministicEmbedder: supports_semantic_search = False embedding_mode = "lexical_hashing" - semantic_support_reason = ( + _DEFAULT_SEMANTIC_SUPPORT_REASON = ( "deterministic feature hashing captures lexical overlap only; semantic vector " "retrieval and semantic grounding are disabled" ) - def __init__(self, dim: int = 384) -> None: + def __init__(self, dim: int = 384, *, semantic_support_reason: Optional[str] = None) -> None: if isinstance(dim, bool) or not isinstance(dim, Integral): raise ValueError("embedding dimension must be a positive integer") dimension = int(dim) @@ -48,6 +48,14 @@ def __init__(self, dim: int = 384) -> None: f"embedding dimension must be between 1 and {MAX_EMBEDDING_DIM}" ) self._dim = dimension + # A factory may supply a safe, public explanation when a requested semantic + # backend could not load. Keep the ordinary dependency-free constructor's + # capability contract unchanged. + self.semantic_support_reason = ( + str(semantic_support_reason).strip() + if semantic_support_reason + else self._DEFAULT_SEMANTIC_SUPPORT_REASON + ) @property def dim(self) -> int: diff --git a/engraphis/backends/embedder_st.py b/engraphis/backends/embedder_st.py index aa272650..2b99d8c0 100644 --- a/engraphis/backends/embedder_st.py +++ b/engraphis/backends/embedder_st.py @@ -4,6 +4,11 @@ behind the ``Embedder`` interface. ``get_embedder`` returns a real model when one is configured and importable, and otherwise falls back to the dependency-free ``DeterministicEmbedder`` so the system always runs (offline, CI). + +``local:`` is an explicit local-only selector. It asks sentence-transformers +to load only files already present at ```` or in its local cache. Engraphis +does not ship a model in this package, so a missing local model degrades to lexical +hashing and reports that fact through the normal embedder capability response. """ from __future__ import annotations @@ -14,18 +19,33 @@ from engraphis.backends.embedder_deterministic import DeterministicEmbedder +LOCAL_MODEL_PREFIX = "local:" + + class SentenceTransformerEmbedder: supports_semantic_search = True embedding_mode = "semantic" - def __init__(self, model_name: str, *, revision: Optional[str] = None) -> None: + def __init__( + self, + model_name: str, + *, + revision: Optional[str] = None, + local_files_only: bool = False, + ) -> None: from sentence_transformers import SentenceTransformer # lazy: optional dependency kwargs = {"revision": revision} if revision else {} + if local_files_only: + # This avoids a Hub request when an operator explicitly selected the + # local mode. It still supports both a local model directory and an + # already-populated sentence-transformers cache. + kwargs["local_files_only"] = True # Keep declared model provenance beside the loaded object. Benchmark # artifacts must be able to distinguish a pinned model from a mutable # fallback without inspecting implementation-specific internals. self.model_name = model_name self.revision = revision + self.local_files_only = local_files_only self.model = SentenceTransformer(model_name, **kwargs) self._dim = int(self.model.get_embedding_dimension()) @@ -49,11 +69,29 @@ def get_embedder( *, revision: Optional[str] = None, ): - """A real model if available, else the deterministic offline embedder.""" + """Return a semantic model when available, else explicit lexical degradation. + + Prefix a configured model with ``local:`` to require a local path or cached + model. That mode never asks sentence-transformers to download the model. It + is deliberately opt-in because a regular model identifier retains the existing + behavior for operators who want sentence-transformers to resolve it normally. + """ global LAST_EMBEDDER_ERROR if model_name: + raw_model_name = str(model_name).strip() + local_files_only = raw_model_name.startswith(LOCAL_MODEL_PREFIX) + resolved_model_name = ( + raw_model_name[len(LOCAL_MODEL_PREFIX):].strip() + if local_files_only + else raw_model_name + ) try: - emb = SentenceTransformerEmbedder(model_name, revision=revision) + if not resolved_model_name: + raise ValueError("local embedder selector requires a path or cached model name") + factory_kwargs = {"revision": revision} + if local_files_only: + factory_kwargs["local_files_only"] = True + emb = SentenceTransformerEmbedder(resolved_model_name, **factory_kwargs) LAST_EMBEDDER_ERROR = "" return emb except Exception as exc: # noqa: BLE001 - optional dep; record why we fall back @@ -64,5 +102,14 @@ def get_embedder( emit( "embedder '%s' unavailable (%s) - using the %d-dim deterministic " "embedder; semantic recall/why/timeline will not match stored vectors.", - model_name, LAST_EMBEDDER_ERROR, dim) + raw_model_name, LAST_EMBEDDER_ERROR, dim) + source = "requested local semantic model" if local_files_only else "requested semantic model" + return DeterministicEmbedder( + dim, + semantic_support_reason=( + f"{source} is unavailable; deterministic feature hashing captures " + "lexical overlap only, so semantic vector retrieval and semantic " + "grounding are disabled" + ), + ) return DeterministicEmbedder(dim) diff --git a/engraphis/backends/sync_relay.py b/engraphis/backends/sync_relay.py index f357ce7c..45940b42 100644 --- a/engraphis/backends/sync_relay.py +++ b/engraphis/backends/sync_relay.py @@ -80,14 +80,19 @@ def decode_sync_e2ee_key(value: object) -> bytes: each authorized device through their own trusted channel. """ raw = str(value or "").strip() - if re.fullmatch(r"[A-Za-z0-9_-]{43}", raw) is None: + # Accept both the documented unpadded form (43 chars) and a conventionally + # padded 44-char base64 (one trailing '='), so operators who paste a standard + # padded base64 from a key generator are not surprised by a 409. + if re.fullmatch(r"[A-Za-z0-9_-]{43}={0,1}", raw) is None: raise RelayError( "Cloud Sync needs a 32-byte end-to-end encryption key in " + SYNC_E2EE_KEY_ENV, status=409, ) + # ``b64decode`` needs the padding explicit: 43 chars = 32 bytes + one pad. + padded = raw if raw.endswith("=") else raw + "=" try: - key = base64.b64decode(raw + "=", altchars=b"-_", validate=True) + key = base64.b64decode(padded, altchars=b"-_", validate=True) except (ValueError, binascii.Error): raise RelayError("Cloud Sync end-to-end encryption key is malformed", status=409) from None if len(key) != SYNC_E2EE_KEY_BYTES: @@ -551,6 +556,11 @@ def _request(self, url: str, *, method: str, data: Optional[bytes] = None, status=exc.code) from None except urllib.error.URLError: raise RelayUnreachable("could not reach the relay") from None + except (TimeoutError, OSError): + # urllib can surface socket timeouts and low-level TLS/socket failures + # directly rather than wrapping them in URLError. Normalize them to the + # sanitized transport class so callers never expose provider text. + raise RelayUnreachable("could not reach the relay") from None # ── SyncTransport protocol ─────────────────────────────────────────────────────── def push(self, name: str, data: bytes) -> None: @@ -589,7 +599,17 @@ def pull(self) -> Iterable[Tuple[str, bytes]]: which exposed only the base64 bulk endpoint. """ failures: List[str] = [] - for index, name in enumerate(self.list_names()): + try: + names = self.list_names() + except RelayError as exc: + # The first relay generation exposed only the bulk endpoint and returned + # 404 for the newer names route. Fall back before iterating so that a + # missing capability is not mistaken for an empty workspace. + if exc.status == 404: + yield from self._pull_legacy() + return + raise + for index, name in enumerate(names): try: data = self._request( self._url("bundles/%s" % quote(name, safe="")), @@ -631,8 +651,11 @@ def _pull_legacy(self) -> Iterable[Tuple[str, bytes]]: raise ValueError("bundles is not a bounded list") except ( UnicodeDecodeError, json.JSONDecodeError, RecursionError, ValueError, - ) as exc: - raise RelayError("relay returned an invalid legacy bundle response") from exc + ): + # The relay controls both bytes and JSON structure. Suppress exception + # chaining so an invalid remote response cannot retain payload fragments + # in a traceback or error object's ``__cause__``. + raise RelayError("relay returned an invalid legacy bundle response") from None out: List[Tuple[str, bytes]] = [] seen = set() @@ -684,5 +707,5 @@ def list_names(self) -> List[str]: return safe_names except ( UnicodeDecodeError, json.JSONDecodeError, RecursionError, ValueError - ) as exc: - raise RelayError("relay returned an invalid name response") from exc + ): + raise RelayError("relay returned an invalid name response") from None diff --git a/engraphis/backends/vector_numpy.py b/engraphis/backends/vector_numpy.py index d28edfd2..cce0f0f6 100644 --- a/engraphis/backends/vector_numpy.py +++ b/engraphis/backends/vector_numpy.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +from numbers import Integral from typing import Optional import numpy as np @@ -16,31 +17,110 @@ from engraphis.core.store import Store +def _validated_dimension(dim: int) -> int: + if isinstance(dim, bool) or not isinstance(dim, Integral) or int(dim) < 1: + raise ValueError("embedding dimension must be a positive integer") + return int(dim) + + +def _validated_k(k: int) -> int: + if isinstance(k, bool) or not isinstance(k, Integral) or int(k) < 0: + raise ValueError("k must be a non-negative integer") + return int(k) + + +def _vector_batch(vecs: np.ndarray) -> np.ndarray: + try: + values = np.asarray(vecs, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError("vectors must be a finite float32 array") from exc + if values.ndim != 2 or values.shape[1] < 1: + raise ValueError("vector batch must be a 2-D array of shape (n, dim>0)") + if not np.isfinite(values).all(): + raise ValueError("vectors must contain only finite values") + return values + + +def _vector_query(vec: np.ndarray) -> np.ndarray: + try: + values = np.asarray(vec, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError("query vector must be a finite 1-D float32 array") from exc + if values.ndim != 1 or values.shape[0] < 1: + raise ValueError("query vector must be a 1-D array of shape (dim>0,)") + if not np.isfinite(values).all(): + raise ValueError("query vector must contain only finite values") + return values + + class NumpyVectorIndex: """Store-backed brute-force cosine index. Vectors are stored normalized.""" - def __init__(self, store: Store) -> None: + def __init__(self, store: Store, *, dim: Optional[int] = None) -> None: self.store = store + self.dim = _validated_dimension(dim) if dim is not None else None - def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None) -> None: - vecs = np.asarray(vecs, dtype=np.float32) + def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None, + *, commit: bool = True) -> None: + values = _vector_batch(vecs) + if self.dim is not None and values.shape[1] != self.dim: + raise ValueError( + f"vector dimension {values.shape[1]} does not match the index dimension {self.dim}" + ) + try: + count = len(ids) + except TypeError as exc: + raise ValueError("ids must be a sequence matching the vector batch") from exc + if count != values.shape[0]: + raise ValueError( + f"ids length {count} does not match vector batch size {values.shape[0]}" + ) + if any(not isinstance(mid, str) or not mid for mid in ids): + raise ValueError("ids must contain non-empty strings") + if not count: + return for i, mid in enumerate(ids): - self.store.put_vector(mid, vecs[i]) - self.store.conn.commit() + self.store.put_vector(mid, values[i]) + if commit: + self.store.conn.commit() + def delete(self, ids: list[str], *, commit: bool = True) -> None: + marks = ",".join("?" for _ in ids) + if not ids: + return + self.store.conn.execute(f"DELETE FROM mem_vectors WHERE id IN ({marks})", ids) + if commit: + self.store.conn.commit() def search(self, vec: np.ndarray, k: int, *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]: - if k <= 0: + k = _validated_k(k) + if k == 0: return [] - q = np.asarray(vec, dtype=np.float32) - n = float(np.linalg.norm(q)) + q = _vector_query(vec) + if self.dim is not None and q.shape[0] != self.dim: + raise ValueError( + f"query dimension {q.shape[0]} does not match the index dimension {self.dim}" + ) + with np.errstate(over="ignore", invalid="ignore"): + n = float(np.linalg.norm(q)) + if not np.isfinite(n): + raise ValueError("query vector norm must be finite") if n > 0: q = q / n - rows = list(self.store.iter_vectors(filter, dim=int(q.shape[0]))) + rows = list(self.store.iter_vectors( + filter, dim=self.dim if self.dim is not None else int(q.shape[0]) + )) if not rows: return [] - ids = [r[0] for r in rows] - mat = np.vstack([r[1] for r in rows]) # already normalized on write + # Guard against heterogeneous stored dimensions (an embedder model or + # ENGRAPHIS_EMBED_DIM change can leave legacy rows at a different width). + # Skipping mismatched rows keeps the semantic arm alive instead of + # raising on np.vstack and turning recall into a 500. + matched = [(r[0], r[1]) for r in rows if r[1].shape[0] == q.shape[0]] + if not matched: + return [] + ids = [r[0] for r in matched] + mat = np.vstack([r[1] for r in matched]) # already normalized on write scores = mat @ q # cosine == dot for unit vectors k = min(k, len(ids)) # ``argpartition`` does not define which equal-scored rows survive at @@ -50,10 +130,3 @@ def search(self, vec: np.ndarray, k: int, range(len(ids)), key=lambda index: (-float(scores[index]), ids[index]) )[:k] return [(ids[index], float(scores[index])) for index in top] - - def delete(self, ids: list[str]) -> None: - marks = ",".join("?" for _ in ids) - if not ids: - return - self.store.conn.execute(f"DELETE FROM mem_vectors WHERE id IN ({marks})", ids) - self.store.conn.commit() diff --git a/engraphis/backends/vector_sqlitevec.py b/engraphis/backends/vector_sqlitevec.py index 941719d3..6a6cf5e5 100644 --- a/engraphis/backends/vector_sqlitevec.py +++ b/engraphis/backends/vector_sqlitevec.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +import re import sys from numbers import Integral from typing import Optional @@ -44,6 +45,42 @@ def _validated_dimension(dim: int) -> int: return dimension +def _validated_k(k: int) -> int: + if isinstance(k, bool) or not isinstance(k, Integral) or int(k) < 0: + raise ValueError("k must be a non-negative integer") + return int(k) + + +def _vector_batch(vecs: np.ndarray, dim: int) -> np.ndarray: + try: + values = np.asarray(vecs, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError("vectors must be a finite float32 array") from exc + if values.ndim != 2 or values.shape[1] != dim: + actual = values.shape[1] if values.ndim == 2 else "?" + raise ValueError( + f"vector dimension {actual} does not match the ANN index dimension {dim}" + ) + if not np.isfinite(values).all(): + raise ValueError("vectors must contain only finite values") + return values + + +def _vector_query(vec: np.ndarray, dim: int) -> np.ndarray: + try: + values = np.asarray(vec, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError("query vector must be a finite 1-D float32 array") from exc + if values.ndim != 1 or values.shape[0] != dim: + actual = values.shape[0] if values.ndim == 1 else "?" + raise ValueError( + f"query dimension {actual} does not match the ANN index dimension {dim}" + ) + if not np.isfinite(values).all(): + raise ValueError("query vector must contain only finite values") + return values + + class SqliteVecVectorIndex: """ANN over embeddings using the sqlite-vec extension.""" @@ -65,33 +102,76 @@ def __init__(self, store: Store, dim: int) -> None: self.dim = dimension conn = store.conn conn.enable_load_extension(True) - sqlite_vec.load(conn) - conn.enable_load_extension(False) + try: + sqlite_vec.load(conn) + finally: + # Never leave extension loading enabled on a shared connection, including + # when the optional native load fails. + conn.enable_load_extension(False) + existing = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='mem_vec_ann'" + ).fetchone() + if existing and existing["sql"]: + match = re.search(r"FLOAT\s*\[\s*(\d+)\s*\]", existing["sql"], re.IGNORECASE) + if match and int(match.group(1)) != dimension: + raise ValueError( + f"existing ANN index dimension {match.group(1)} does not match " + f"requested dimension {dimension}" + ) conn.execute( f"CREATE VIRTUAL TABLE IF NOT EXISTS mem_vec_ann USING vec0(" f"id TEXT PRIMARY KEY, embedding FLOAT[{dimension}])" ) conn.commit() - def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None) -> None: - vecs = np.asarray(vecs, dtype=np.float32) + def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None, + *, commit: bool = True) -> None: + values = _vector_batch(vecs, self.dim) + try: + count = len(ids) + except TypeError as exc: + raise ValueError("ids must be a sequence matching the vector batch") from exc + if count != values.shape[0]: + raise ValueError( + f"ids length {count} does not match vector batch size {values.shape[0]}" + ) + if any(not isinstance(mid, str) or not mid for mid in ids): + raise ValueError("ids must contain non-empty strings") + if not count: + return for i, mid in enumerate(ids): - v = vecs[i] - n = float(np.linalg.norm(v)) + v = values[i] + with np.errstate(over="ignore", invalid="ignore"): + n = float(np.linalg.norm(v)) + if not np.isfinite(n): + raise ValueError("vector norm must be finite") if n > 0: v = v / n self.store.conn.execute( "INSERT OR REPLACE INTO mem_vec_ann(id, embedding) VALUES (?, ?)", (mid, v.tobytes()), ) - self.store.conn.commit() + if commit: + self.store.conn.commit() + + def delete(self, ids: list[str], *, commit: bool = True) -> None: + if not ids: + return + marks = ",".join("?" for _ in ids) + self.store.conn.execute(f"DELETE FROM mem_vec_ann WHERE id IN ({marks})", ids) + if commit: + self.store.conn.commit() def search(self, vec: np.ndarray, k: int, *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]: - if k <= 0: + k = _validated_k(k) + if k == 0: return [] - v = np.asarray(vec, dtype=np.float32) - n = float(np.linalg.norm(v)) + v = _vector_query(vec, self.dim) + with np.errstate(over="ignore", invalid="ignore"): + n = float(np.linalg.norm(v)) + if not np.isfinite(n): + raise ValueError("query vector norm must be finite") if n > 0: v = v / n total = k @@ -102,13 +182,7 @@ def search(self, vec: np.ndarray, k: int, return [] limit = min(k, total) while True: - # The KNN cap uses vec0's explicit ``k = ?`` constraint, NOT ``LIMIT ?``: - # SQLite < 3.41 never passes a LIMIT down to a virtual table's xBestIndex, - # so vec0 raises "A LIMIT or 'k = ?' constraint is required on vec0 knn - # queries" — which the resolve path swallows, silently degrading every - # near-duplicate write to ADD on those systems. ``k = ?`` is the syntax - # sqlite-vec documents for exactly this reason and works on every - # supported SQLite/sqlite-vec combination. + # The KNN cap uses vec0's explicit `k = ?` constraint, NOT `LIMIT ?`. rows = self.store.conn.execute( "SELECT id, distance FROM mem_vec_ann WHERE embedding MATCH ? " "AND k = ? ORDER BY distance", @@ -120,24 +194,18 @@ def search(self, vec: np.ndarray, k: int, rec = self.store.get_memory(row["id"]) if rec is None or not _visible(rec, filter): continue - out.append((row["id"], _cosine_from_l2(row["distance"]))) + # A zero query has no direction; retain the NumPy backend's + # deterministic zero similarity rather than converting its + # distance to the mathematically unrelated 0.5. + score = 0.0 if n == 0 else _cosine_from_l2(row["distance"]) + out.append((row["id"], score)) if len(out) >= k: return out if filter is None or len(rows) < limit or limit >= total: return out - # Filtered search widens geometrically until k visible hits are found. Once - # the next doubling would already cover a quarter of the index, jump straight - # to a single full scan: on a workspace dense with invisible rows (expired/ - # out-of-scope), the geometric tail otherwise re-runs several near-full ANN - # scans back to back for one query. + # Filtered search widens geometrically until k visible hits are found. limit = total if limit * 2 >= total // 4 else limit * 2 - def delete(self, ids: list[str]) -> None: - if not ids: - return - marks = ",".join("?" for _ in ids) - self.store.conn.execute(f"DELETE FROM mem_vec_ann WHERE id IN ({marks})", ids) - self.store.conn.commit() def get_vector_index(store: Store, *, dim: int = 384, prefer: str = "auto"): @@ -147,11 +215,13 @@ def get_vector_index(store: Store, *, dim: int = 384, prefer: str = "auto"): or "numpy" (force the reference index). """ dimension = _validated_dimension(dim) + if prefer not in {"auto", "sqlite-vec", "numpy"}: + raise ValueError("prefer must be one of: auto, sqlite-vec, numpy") if prefer == "numpy": - return NumpyVectorIndex(store) + return NumpyVectorIndex(store, dim=dimension) try: return SqliteVecVectorIndex(store, dimension) except Exception: if prefer == "sqlite-vec": raise - return NumpyVectorIndex(store) + return NumpyVectorIndex(store, dim=dimension) diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js index 34f33713..1fa732fb 100644 --- a/engraphis/classic_assets/dashboard.js +++ b/engraphis/classic_assets/dashboard.js @@ -21,7 +21,7 @@ function esc(s){if(s===undefined||s===null)return '';return (''+s).replace(/&/g, removes tab/newline/C0 before resolving a scheme, so a split scheme would re-form as javascript: after a naive scheme match had already failed. A scheme-less string that still contains ':' therefore fails closed rather than passing through. */ -function safeUrl(u){if(!u||typeof u!=='string')return '#';const s=u.replace(/[\u0000-\u001F\u007F]/g,'').trim();if(/^#/.test(s))return s;const m=s.match(/^([a-z][a-z0-9+.-]*):/i);if(!m)return /:/.test(s)?'#':s;if(/^(https?|mailto|ftps?)$/i.test(m[1]))return s;return '#'} +function safeUrl(u){if(!u||typeof u!=='string')return '#';const s=u.replace(/[\u0000-\u001F\u007F]/g,'').trim();if(/^#/.test(s))return s;if((s[0]==='/'||s[0]==='\\')&&(s[1]==='/'||s[1]==='\\'))return '#';const m=s.match(/^([a-z][a-z0-9+.-]*):/i);if(!m)return /:/.test(s)?'#':s;if(/^(https?|mailto|ftps?)$/i.test(m[1]))return s;return '#'} function showAs(el,visible,mode){if(!el)return;el.classList.toggle('is-hidden',!visible);for(const name of ['is-flex','is-block','is-inline-flex'])el.classList.remove(name);if(visible&&mode)el.classList.add('is-'+mode)} function setTone(el,tone){if(!el)return;for(const name of ['tone-red','tone-green','tone-muted'])el.classList.remove(name);if(tone)el.classList.add('tone-'+tone)} function renderMd(md){try{return DOMPurify.sanitize(marked.parse(md||''))}catch(e){return esc(md)}} @@ -462,7 +462,7 @@ async function wsCreate(){ await loadWorkspaceList();setWS(name);toast('Folder "'+name+'" created — now the active folder','ok');refreshFolders(); }catch(e){toast('Create failed: '+e.message,'err')} } -function wsSwitch(name){setWS(name);toast('Switched to '+name,'ok');loadOverview();navTo('overview')} +function wsSwitch(name){setWS(name);toast('Switched to '+name,'ok');navTo('overview')} /* import (files/folders from this PC — see MemoryService.import_folder/import_files) */ async function importUpload(items){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}if(!items||!items.length)return;const fd=new FormData();fd.append('workspace',WS);fd.append('memory_type','semantic');fd.append('derive_facts',document.getElementById('import-derive').checked?'true':'false');for(const it of items)fd.append('files',it.file,it.name);const el=document.getElementById('import-status');el.textContent='Extracting and importing '+items.length+' file(s)…';try{const r=await api('/workspaces/import-files',{method:'POST',body:fd});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s)'+(wc?', '+wc+' warning(s)':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"','ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}} @@ -561,12 +561,14 @@ const setLlmExtractorBase=setLlmExtractor; setLlmExtractor=async function(on){if(on&&!await confirmAction('Turn on LLM extraction',EXTERNAL_LLM_PRIVACY_COPY,'Turn on'))return;return setLlmExtractorBase(on)}; /* Route by cause, exactly like loadAnalytics/loadAutomation. Rendering the purchase panel for every failure told a paying customer to buy the plan they already own whenever the network - blipped or the cloud answered 5xx. Only 401/402/501 are billing answers. */ + blipped or the cloud answered 5xx. Only 402 is an entitlement answer; 401/403 require reconnecting. */ async function loadSyncStatus(){try{const d=await api('/sync/status');renderSync(d)}catch(e){const el=document.getElementById('sync-body');if(!el)return;if(managedConsentRequired(e))el.innerHTML=managedConsentHtml('Cloud Sync');else if(hostedFeatureUnavailable(e))el.innerHTML=unlockHtml('Cloud Sync','pro');else el.innerHTML='
'+esc(e.message)+'
'}} -function syncTotalAuthorizationDenial(last){if(!last||!(Number(last.attempted)>0)||Number(last.succeeded)!==0)return false;const errors=Array.isArray(last.errors)?last.errors:[];return errors.length===Number(last.attempted)&&errors.every(error=>[401,402,403].includes(Number(error.status)))} +function syncTotalAuthorizationDenial(last){if(!last||!(Number(last.attempted)>0)||Number(last.succeeded)!==0)return false;const errors=Array.isArray(last.errors)?last.errors:[];return errors.length===Number(last.attempted)&&errors.every(error=>[401,402,403].includes(Number(error&&error.status)))} function syncRecoveryHtml(){return unlockHtml('Cloud Sync','pro')+`
`} -function renderSync(d){const el=document.getElementById('sync-body');if(!el)return;d=d||{};const last=d.last;if(!d.available){el.innerHTML=unlockHtml('Cloud Sync','pro');return}if(syncTotalAuthorizationDenial(last)){el.innerHTML=syncRecoveryHtml();return}let status='Cloud session connected; no sync recorded on this installation.';if(last){const when=new Date((last.at||0)*1000).toLocaleString();status='Last synced '+when+' — pushed '+(last.exported||0)+', +'+(last.added||0)+' received'+((last.errors&&last.errors.length)?' · '+last.errors.length+' issue(s)':'')+'.'}el.innerHTML=`
Hosted relayCONNECTED
Relay storage and authorization run in Engraphis Cloud. This package contains only the customer client; it does not run a local relay or background scheduler.
${esc(status)}
`} -async function syncNow(){const b=document.getElementById('sync-btn')||document.getElementById('sync-retry-btn');const original=b&&b.textContent;const s=document.getElementById('sync-status');if(b){b.disabled=true;b.textContent='Syncing…'}if(s)s.textContent='Contacting the cloud…';try{const d=await api('/sync/run',{method:'POST',headers:{'Content-Type':'application/json'},body:'{}'});const su=d.summary||{};toast('Synced — pushed '+(su.exported||0)+', '+(su.added||0)+' new from other devices','ok');await loadSyncStatus()}catch(e){if(e.status===401||e.status===402||e.status===403){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncRecoveryHtml();toast(e.status===402?'Cloud Sync requires an active Pro or Team entitlement — open Engraphis Cloud to upgrade or renew.':'Cloud Sync authorization is no longer active — reconnect in Engraphis Cloud.','err');return}toast('Sync failed: '+e.message,'err');if(b){b.disabled=false;b.textContent=original||'Sync now'}if(s)s.textContent='Sync failed — try again.'}} +function syncReconnectHtml(){return `
Cloud Sync is not authorized for this installation. Reconnect in Engraphis Cloud or contact your administrator.
`} +function syncDenialHtml(last){const errors=Array.isArray(last&&last.errors)?last.errors:[];return errors.some(error=>[401,403].includes(Number(error&&error.status)))?syncReconnectHtml():syncRecoveryHtml()} +function renderSync(d){const el=document.getElementById('sync-body');if(!el)return;d=d||{};const last=d.last;if(!d.available){el.innerHTML=unlockHtml('Cloud Sync','pro');return}if (syncTotalAuthorizationDenial(last)) { el.innerHTML = syncDenialHtml(last); return; }let status='Cloud session connected; no sync recorded on this installation.';if(last){const when=new Date((last.at||0)*1000).toLocaleString();status='Last synced '+when+' — pushed '+(last.exported||0)+', +'+(last.added||0)+' received'+((last.errors&&last.errors.length)?' · '+last.errors.length+' issue(s)':'')+'.'}el.innerHTML=`
Hosted relayCONNECTED
Relay storage and authorization run in Engraphis Cloud. This package contains only the customer client; it does not run a local relay or background scheduler.
${esc(status)}
`} +async function syncNow(){const b=document.getElementById('sync-btn')||document.getElementById('sync-retry-btn');const original=b&&b.textContent;const s=document.getElementById('sync-status');if(b){b.disabled=true;b.textContent='Syncing…'}if(s)s.textContent='Contacting the cloud…';try{const d=await api('/sync/run',{method:'POST',headers:{'Content-Type':'application/json'},body:'{}'});const su=d.summary||{};toast('Synced — pushed '+(su.exported||0)+', '+(su.added||0)+' new from other devices','ok');await loadSyncStatus()}catch(e){const status=Number(e&&e.status);if(status===402){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncRecoveryHtml();toast('Cloud Sync requires an active Pro or Team entitlement — open Engraphis Cloud to upgrade or renew.','err')}else if(status===401||status===403){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncReconnectHtml();toast('Cloud Sync authorization is no longer active — reconnect in Engraphis Cloud.','err')}else{toast('Sync failed: '+e.message,'err');if(s)s.textContent='Sync failed — try again.'}}finally{if(b){b.disabled=false;b.textContent=original||'Sync now'}}} const syncNowBase=syncNow; syncNow=async function(){if(!await confirmCloudTransfer('Sync shared workspaces','Cloud Sync sends eligible changes from your shared workspaces to Engraphis Cloud and receives authorized changes from your other installations; secret and session-scoped rows stay local.','Sync now',CLOUD_SYNC_PRIVACY_COPY))return;return syncNowBase()} @@ -1104,18 +1106,30 @@ function graphSetColorBy(mode){ function graphApplyForces(){ if(!FG)return; const settings=window.GSET,mode=settings.mode||'compact'; - FG.d3Force('charge').strength(-settings.repel); + FG.d3Force('charge').strength(-(mode==='communities'?Math.max(10,settings.repel*.68):settings.repel)); FG.d3Force('link').distance(settings.link); if(typeof d3==='undefined')return; FG.d3Force('radial',null); - /* Communities remain a colour/relationship grouping, not separate gravity wells. Giving - every cluster its own off-centre target was what made the default view form a hollow ring. - Pull every standard layout toward one shared origin; charge and link forces preserve the - readable local clusters inside that coherent overall shape. */ - const centering=mode==='radial'?Math.max(.04,settings.gravity/300):settings.gravity/100; - FG.d3Force('x',d3.forceX(0).strength(centering)); - FG.d3Force('y',d3.forceY(0).strength(centering)); - if(mode==='radial'&&d3.forceRadial)FG.d3Force('radial',d3.forceRadial(node=>Math.max(0,5-Math.min(5,node.degree||0))*Math.max(8,settings.link*.72)).strength(.32)); + const layoutNodes=GACTIVE_DATA&&GACTIVE_DATA.nodes||[]; + /* Each named mode owns a different target geometry. Slider values still control local + spacing, but switching buttons must visibly change the arrangement even for one component. */ + if(mode==='communities'){ + const keys=[],seen=new Set();layoutNodes.forEach(node=>{const key=Number.isFinite(node.community)?node.community:0;if(!seen.has(key)){seen.add(key);keys.push(key)}});keys.sort((a,b)=>a-b); + const cols=Math.max(1,Math.ceil(Math.sqrt(keys.length))),rows=Math.max(1,Math.ceil(keys.length/cols)),gap=Math.max(180,(Number(settings.link)||16)*10),targets=new Map(); + keys.forEach((key,index)=>{const col=index%cols,row=Math.floor(index/cols);targets.set(key,{x:(col-(cols-1)/2)*gap,y:(row-(rows-1)/2)*gap*.72})}); + const centering=Math.max(.04,(Number(settings.gravity)||0)/100);FG.d3Force('x',d3.forceX(0).strength(centering));FG.d3Force('y',d3.forceY(0).strength(centering)); + }else if(mode==='radial'&&d3.forceRadial){ + const outer=Math.max(180,Math.min(360,Math.sqrt(Math.max(1,layoutNodes.length))*18+(Number(settings.link)||16)*4)),maxDegree=Math.max(1,layoutNodes.reduce((max,node)=>Math.max(max,node.degree||0),1)); + FG.d3Force('x',d3.forceX(0).strength(Math.max(.05,(Number(settings.gravity)||0)/500)));FG.d3Force('y',d3.forceY(0).strength(Math.max(.05,(Number(settings.gravity)||0)/500))); + FG.d3Force('radial',d3.forceRadial(node=>{const hubness=Math.max(0,Math.min(1,(node.degree||0)/maxDegree));return 34+(outer-34)*(1-hubness)}).strength(.72)); + }else if(mode==='constellation'){ + const positions=new Map(),total=Math.max(1,layoutNodes.length-1),reach=Math.max(160,Math.min(330,80+Math.sqrt(Math.max(1,layoutNodes.length))*10)); + layoutNodes.forEach((node,index)=>{const rank=Number.isFinite(node.rank)?node.rank:index,fraction=Math.max(0,Math.min(1,rank/total)),angle=index*2.399963229728653,radius=48+fraction*reach;positions.set(node.id,{x:Math.cos(angle)*radius*1.18,y:Math.sin(angle)*radius*.76})}); + const target=node=>positions.get(node.id)||{x:0,y:0};FG.d3Force('x',d3.forceX(node=>target(node).x).strength(.18));FG.d3Force('y',d3.forceY(node=>target(node).y).strength(.18)); + }else{ + const centering=mode==='compact'?Math.max(.24,(Number(settings.gravity)||0)/100):Math.max(.06,(Number(settings.gravity)||0)/100); + FG.d3Force('x',d3.forceX(0).strength(centering));FG.d3Force('y',d3.forceY(0).strength(centering)); + } FG.d3Force('collide',d3.forceCollide(node=>node.radius+1.5).iterations(GPERF.large?1:2)); } function graphSetHighlight(id){ @@ -1367,7 +1381,7 @@ function graphUpdateEditedBadge(){ function graphResetPreset(){graphApplyPreset(window.GSET.mode==='custom'?'compact':window.GSET.mode);toast('Preset restored','ok')} function graphToggleFlow(control){window.GSET.flow=control.checked;if(GRAPH_ENGINE)GRAPH_ENGINE.setSettings({flow:control.checked});else if(FG)graphRender(false,false)} function graphToggleFreeze(control){ - window.GSET.frozen=control.checked;if(GRAPH_ENGINE){GRAPH_ENGINE.freeze(control.checked);return}if(!FG)return; + window.GSET.frozen=control.checked;if(GRAPH_ENGINE){GRAPH_ENGINE.freeze(control.checked);graphSetSimulationStatus(control.checked?'Layout frozen':'Adaptive layout',false);return}if(!FG)return; const ns=(FG.graphData().nodes)||[]; if(control.checked){ns.forEach(n=>{n.fx=n.x;n.fy=n.y});graphSetSimulationStatus('Layout frozen')} else{ns.forEach(n=>{n.fx=null;n.fy=null});FG.d3ReheatSimulation()} diff --git a/engraphis/commercial_manifest.json b/engraphis/commercial_manifest.json index bc38fb55..a60f4dac 100644 --- a/engraphis/commercial_manifest.json +++ b/engraphis/commercial_manifest.json @@ -1,6 +1,6 @@ { "schema": "engraphis-commercial/v2", - "version": "1.4.0", + "version": "1.4.5", "control_plane": "https://api.engraphis.com", "account_portal": "https://api.engraphis.com/account", "billing": { diff --git a/engraphis/config.py b/engraphis/config.py index fb6c4ca4..8f8ae682 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -4,6 +4,7 @@ import errno import json import hashlib +import math import os import re import sqlite3 @@ -478,6 +479,17 @@ def _configured_db_path(root: Path = _PROJECT_ROOT) -> str: def _env(key: str, default: str = "") -> str: return os.environ.get(key, default).strip() +def _parse_vector_backend(value: str) -> str: + """Return a supported vector backend, failing closed to the portable default.""" + normalized = (value or "").strip().lower() + return normalized if normalized in {"numpy", "sqlite-vec", "auto"} else "numpy" + + +def _parse_llm_provider(value: str) -> str: + """Use the documented provider default when an env entry is blank.""" + return (value or "").strip().lower() or "openai" + + def _validate_service_mode(value: str) -> str: """Validate service mode against allowed values. @@ -501,19 +513,26 @@ def _env_int(key: str, default: int) -> int: def _env_float(key: str, default: float) -> float: try: - return float(_env(key, str(default))) - except ValueError: + value = float(_env(key, str(default))) + except (TypeError, ValueError): return default + return value if math.isfinite(value) else default _FALSY_ENV = {"0", "false", "no", "off", "disable", "disabled"} +_TRUTHY_ENV = {"1", "true", "yes", "on", "enable", "enabled"} def _env_bool(key: str, default: bool) -> bool: raw = os.environ.get(key) if raw is None or not raw.strip(): return default - return raw.strip().lower() not in _FALSY_ENV + normalized = raw.strip().lower() + if normalized in _TRUTHY_ENV: + return True + if normalized in _FALSY_ENV: + return False + return default def persist_project_env(values: dict[str, str], path: Optional[Path] = None) -> Path: @@ -613,15 +632,28 @@ class Settings: ) ) + # Vector index backend for the v2 engine: "numpy" (default — deterministic, + # offline reference index), "sqlite-vec" (require the accelerated ANN backend), + # or "auto" (use sqlite-vec when available, fall back to NumPy). The server + # entrypoints honor this so a self-host can opt into the accelerated path + # without touching code; the constructor default stays "numpy" for determinism. + vector_backend: str = field( + default_factory=lambda: _parse_vector_backend( + _env("ENGRAPHIS_VECTOR_BACKEND", "numpy") + ) + ) + # Fact extraction on the v2 write path: "none" (default — store text as given), # "chunk" (deterministic, offline structure-aware chunking — knobs # ENGRAPHIS_CHUNK_TOKENS/_OVERLAP/_MAX and optional pinned - # ENGRAPHIS_CHUNK_TOKENIZER_MODEL/_REVISION), or "llm" (distill raw text into - # discrete facts via the configured LLM before storing). + # ENGRAPHIS_CHUNK_TOKENIZER_MODEL/_REVISION), "llm" (free-form fact extraction), or + # "llm_structured" (schema-validated facts, entities, relations, and keywords via LLM). extractor: str = field(default_factory=lambda: _env("ENGRAPHIS_EXTRACTOR", "none").lower()) llm_provider: str = field( - default_factory=lambda: _env("ENGRAPHIS_LLM_PROVIDER", "openai").lower() + default_factory=lambda: _parse_llm_provider( + _env("ENGRAPHIS_LLM_PROVIDER", "openai") + ) ) llm_model: str = field(default_factory=lambda: _env("ENGRAPHIS_LLM_MODEL", "gpt-4o-mini")) llm_api_key: str = field(default_factory=lambda: _env("ENGRAPHIS_LLM_API_KEY", "")) @@ -634,8 +666,7 @@ class Settings: # Settings On/Off control, or ENGRAPHIS_LLM_AUTO_EXTRACT=1) — so a mere connection # test never silently starts provider egress of ingested content. llm_auto_extract: bool = field( - default_factory=lambda: _env("ENGRAPHIS_LLM_AUTO_EXTRACT", "0").lower() - not in ("0", "false", "no", "off") + default_factory=lambda: _env_bool("ENGRAPHIS_LLM_AUTO_EXTRACT", False) ) # Optional cross-encoder reranker model. Empty (default) -> IdentityReranker (offline). @@ -664,6 +695,13 @@ class Settings: loop_interval: int = field(default_factory=lambda: _env_int("ENGRAPHIS_LOOP_INTERVAL", 60)) loop_top_k: int = field(default_factory=lambda: _env_int("ENGRAPHIS_LOOP_TOP_K", 20)) + # OFF by default (opt-in): the background consciousness loop only runs a local + # consolidation sweep when this is enabled. Consolidation is the only loop stage that + # ever calls the LLM (``structured``/``profiles`` are never used here, so the default + # sweep is fully deterministic), and it is expensive: a workspace-wide cluster scan. + # It therefore needs an explicit operator decision. 0 = disabled; N > 0 = run at most + # once every N loop ticks (every 60s tick is usually far too often). + loop_consolidate: int = field(default_factory=lambda: _env_int("ENGRAPHIS_LOOP_CONSOLIDATE", 0)) decay_halflife_days: float = field( default_factory=lambda: _env_float("ENGRAPHIS_DECAY_HALFLIFE_DAYS", 7.0) ) @@ -674,11 +712,11 @@ class Settings: rate_window: int = field(default_factory=lambda: _env_int("ENGRAPHIS_RATE_WINDOW", 60)) # Update reminder: check the newest published release and surface it in the dashboard, - # server startup log, and MCP. On by default; ``ENGRAPHIS_UPDATE_CHECK=0`` opts out and - # stops all network activity. ``ENGRAPHIS_UPDATE_URL`` overrides the default GitHub - # releases source (see engraphis.update_check, the runtime authority for both knobs). + # server startup log, and MCP. Off by default; ``ENGRAPHIS_UPDATE_CHECK`` must contain + # a recognized affirmative value before any network activity is allowed. The runtime + # authority is :mod:`engraphis.update_check`, which reads the same knob directly. update_check: bool = field( - default_factory=lambda: _env_bool("ENGRAPHIS_UPDATE_CHECK", True)) + default_factory=lambda: _env_bool("ENGRAPHIS_UPDATE_CHECK", False)) update_check_url: str = field( default_factory=lambda: _env("ENGRAPHIS_UPDATE_URL", "")) @@ -698,10 +736,15 @@ def _parse_headers(raw: str) -> dict: if not raw: return {} try: - return json.loads(raw) + parsed = json.loads(raw) except Exception: return {} - + if not isinstance(parsed, dict): + return {} + if not all(isinstance(key, str) and isinstance(value, str) + for key, value in parsed.items()): + return {} + return parsed def _parse_origins(raw: str, port: int = 8700) -> list: """CORS allow-list. Empty -> loopback on the CONFIGURED port (safe local-first default). diff --git a/engraphis/core/consolidate.py b/engraphis/core/consolidate.py index 5f313cd0..33ddac74 100644 --- a/engraphis/core/consolidate.py +++ b/engraphis/core/consolidate.py @@ -31,7 +31,7 @@ from engraphis.core import scoring from engraphis.core.interfaces import MemoryRecord, MemoryType, Scope, SearchFilter -from engraphis.core.poisoning import prompt_eligible +from engraphis.core.poisoning import REVIEW_PENDING, prompt_eligible from engraphis.core.textutil import estimate_tokens, jaccard, tokenize logger = logging.getLogger(__name__) @@ -57,7 +57,17 @@ # Python afterwards silently returns *zero* candidates as soon as the newest ``n`` rows # happen to be of the wrong type, which reads as "nothing to consolidate" in the report. DISTILL_SCAN_LIMIT = 2000 +# Bound the population that reaches the quadratic fallback clustering pass while +# allowing the storage scan to page in smaller batches and skip pending rows. +DISTILL_CLUSTER_LIMIT = 2000 +# Cursor name for the bounded episodic sweep; the value is scoped by workspace/repo. +DISTILL_CURSOR_NAME = "episodic-consolidation" + PROFILE_SCAN_LIMIT = 5000 +PROFILE_MEMORY_LIMIT = 5000 +# Cursor name for the bounded profile-memory sweep; scoped by workspace/repo. +PROFILE_CURSOR_NAME = "profile-consolidation" +PROFILE_ENTITY_LIMIT = 2000 # Transient types eligible for archival (pass 2). TRANSIENT_TYPES = [MemoryType.WORKING, MemoryType.EPISODIC] # Types the optional local profile pass rolls up. @@ -102,6 +112,530 @@ def _compaction(tokens_before: int, tokens_after: int, units: int) -> dict: "tokens_saved": saved, "reduction_pct": pct, "units": units} +def _linked_memory_ids(store, memory_ids: list[str], *, relation: str) -> set[str]: + """Return source memories attached by a completed derived-memory relation. + + A bounded scan must skip sources that no longer need work, but it must keep every + source of a partially-written digest/profile visible so the retry path can repair + the exact row instead of creating a second derived record. + """ + unique_ids = list(dict.fromkeys(str(memory_id) for memory_id in memory_ids if memory_id)) + if not unique_ids: + return set() + linked_rows: list[Any] = [] + for start in range(0, len(unique_ids), 499): + chunk = unique_ids[start:start + 499] + marks = ",".join("?" for _ in chunk) + linked_rows.extend(store.conn.execute( + f"SELECT a, b FROM mem_links WHERE relation=? " + f"AND (a IN ({marks}) OR b IN ({marks}))", + (relation, *chunk, *chunk), + ).fetchall()) + endpoint_ids = { + str(value) for row in linked_rows for value in (row["a"], row["b"]) if value + } + rows_by_id: dict[str, Any] = {} + for start in range(0, len(endpoint_ids), 500): + chunk = sorted(endpoint_ids)[start:start + 500] + if not chunk: + continue + marks = ",".join("?" for _ in chunk) + for row in store.conn.execute( + f"SELECT id, metadata, provenance FROM memories WHERE id IN ({marks})", + chunk, + ).fetchall(): + rows_by_id[str(row["id"])] = row + + def cited_sources(row: Any) -> set[str]: + if row is None: + # A legacy/manual link whose other endpoint was deleted still means the + # source has already been handled; retain the historical skip behavior. + return set() + metadata = _loads_lenient(row["metadata"]) + metadata = metadata if isinstance(metadata, dict) else {} + provenance = _loads_lenient(row["provenance"]) + provenance = provenance if isinstance(provenance, dict) else {} + nested = metadata.get("provenance") + if isinstance(nested, dict): + provenance = {**provenance, **nested} + key = "profiles" if relation == PROFILE_RELATION else "consolidates" + return { + str(source_id) for source_id in ( + provenance.get(key) or provenance.get("source_ids") or [] + ) if source_id + } + + derived_ids: set[str] = set() + for row in linked_rows: + for endpoint in (str(row["a"]), str(row["b"])): + if endpoint not in unique_ids: + derived_ids.add(endpoint) + + complete_derived: set[str] = set() + for derived_id in derived_ids: + row = rows_by_id.get(derived_id) + cited = cited_sources(row) + if not cited: + complete_derived.add(derived_id) + continue + links = store.conn.execute( + "SELECT a, b FROM mem_links WHERE relation=? AND (a=? OR b=?)", + (relation, derived_id, derived_id), + ).fetchall() + attached = { + str(link["b"] if str(link["a"]) == derived_id else link["a"]) + for link in links + } + if cited <= attached: + complete_derived.add(derived_id) + + linked: set[str] = set() + for row in linked_rows: + a, b = str(row["a"]), str(row["b"]) + if a in unique_ids and b in complete_derived: + linked.add(a) + if b in unique_ids and a in complete_derived: + linked.add(b) + return linked + + +def _scan_memory_window(store, flt: SearchFilter, *, mtypes: list[MemoryType], + batch_size: int, prompt_only: bool = False, + max_records: Optional[int] = None, + exclude_relation: Optional[str] = None, + start_after_id: str = "") -> tuple[list[MemoryRecord], str]: + """Read one bounded keyset window and return its next persistent cursor. + + ``Store.list_memories_page`` orders by id. When a bounded window reaches the + end, the empty cursor deliberately makes the *next* sweep wrap to the start; + this rotates maintenance over all eligible rows without materializing or + clustering the full population on every run. + """ + size = max(1, int(batch_size)) + cap = None if max_records is None else max(0, int(max_records)) + if cap == 0: + return [], str(start_after_id or "") + after_id = str(start_after_id or "") + records: list[MemoryRecord] = [] + scoped = _replace(flt, mtypes=mtypes) + next_cursor = "" + while True: + page = store.list_memories_page(scoped, after_id=after_id, limit=size) + if not page: + # The persisted cursor was at the end of the keyspace. Start the next + # sweep from the beginning instead of retrying an empty tail forever. + break + next_after = page[-1].id + page_size = len(page) + if exclude_relation: + excluded = _linked_memory_ids( + store, [memory.id for memory in page], relation=exclude_relation, + ) + page = [memory for memory in page if memory.id not in excluded] + if prompt_only: + records.extend( + memory for memory in page + if prompt_eligible(memory.provenance, memory.metadata) + ) + else: + records.extend(page) + if cap is not None and len(records) >= cap: + next_cursor = next_after + break + if next_after == after_id or page_size < size: + # End-of-keyspace: clear the cursor for the next invocation. + break + after_id = next_after + records.sort( + key=lambda memory: ( + memory.ingested_at if memory.ingested_at is not None else float("-inf"), + memory.id, + ), + reverse=True, + ) + return records[:cap] if cap is not None else records, next_cursor + + +def _scan_memories(store, flt: SearchFilter, *, mtypes: list[MemoryType], + batch_size: int, prompt_only: bool = False, + max_records: Optional[int] = None, + exclude_relation: Optional[str] = None, + start_after_id: str = "") -> list[MemoryRecord]: + """Read every matching row, or one bounded window when ``max_records`` is set.""" + records, _ = _scan_memory_window( + store, flt, mtypes=mtypes, batch_size=batch_size, + prompt_only=prompt_only, max_records=max_records, + exclude_relation=exclude_relation, start_after_id=start_after_id, + ) + return records + + +def _derived_memory_for_sources(store, first: MemoryRecord, source_ids: set[str], + *, provenance_source: str) -> Optional[MemoryRecord]: + """Find a previously inserted but incompletely linked derived memory. + + Memory insertion and link insertion are separate store operations. If a link write + fails after the derived row is committed, a retry must finish that row instead of + creating a second digest and leaving the original sources permanently pending. + """ + flt = SearchFilter( + workspace_id=first.workspace_id, + repo_id=first.repo_id, + scopes=[Scope(first.scope)], + mtypes=[MemoryType.SEMANTIC], + ) + for candidate in store.list_memories(flt, include_invalid=True): + provenance = (candidate.metadata or {}).get("provenance") or {} + if provenance.get("source") != provenance_source: + continue + cited = { + str(memory_id) for memory_id in ( + provenance.get("consolidates") + or provenance.get("profiles") + or [] + ) + } + if cited == source_ids: + return candidate + return None + +def _derived_memories_for_source_subset( + store, first: MemoryRecord, source_ids: set[str], *, provenance_source: str, +) -> list[tuple[MemoryRecord, set[str]]]: + """Find derived rows whose cited sources are a subset of one cluster. + + Structured consolidation may emit several facts per cluster. Recovering each + exact fact before pending detection prevents a partial fact write from either + stranding its remaining sources or being duplicated on retry. + """ + flt = SearchFilter( + workspace_id=first.workspace_id, + repo_id=first.repo_id, + scopes=[Scope(first.scope)], + mtypes=[MemoryType.SEMANTIC], + ) + recovered: list[tuple[MemoryRecord, set[str]]] = [] + for candidate in store.list_memories(flt, include_invalid=True): + provenance = (candidate.metadata or {}).get("provenance") or {} + if provenance.get("source") != provenance_source: + continue + cited = { + str(memory_id) for memory_id in ( + provenance.get("consolidates") + or provenance.get("source_ids") + or [] + ) + } + if cited and cited <= source_ids: + recovered.append((candidate, cited)) + return recovered + + +def _structured_retry_clusters(store, flt: SearchFilter) -> list[list[MemoryRecord]]: + """Recover source clusters for structured rows whose link set was interrupted. + + A structured run can emit several facts from one cluster. If a later fact was + inserted before its first link failed, the sources already linked to an earlier + fact would otherwise be filtered from the bounded scan and the retry would never + see the complete cluster again. + """ + derived_filter = _replace(flt, mtypes=[MemoryType.SEMANTIC]) + source_groups: list[set[str]] = [] + for derived in _scan_memories( + store, derived_filter, mtypes=[MemoryType.SEMANTIC], + batch_size=DISTILL_SCAN_LIMIT, max_records=DISTILL_CLUSTER_LIMIT, + ): + provenance = (derived.metadata or {}).get("provenance") or {} + if provenance.get("source") != "structured_consolidation": + continue + source_ids = { + str(source_id) for source_id in ( + provenance.get("consolidates") or provenance.get("source_ids") or [] + ) if source_id + } + if not source_ids: + continue + attached = { + str(link["b"] if str(link["a"]) == derived.id else link["a"]) + for link in store.get_links(derived.id) + if link["relation"] == "consolidates" + } + if source_ids <= attached: + continue + for group in source_groups: + if group & source_ids: + group.update(source_ids) + break + else: + source_groups.append(set(source_ids)) + + # Merge transitive overlaps (A overlaps B, B overlaps C). + changed = True + while changed: + changed = False + for index, group in enumerate(source_groups): + for other_index in range(index + 1, len(source_groups)): + if group & source_groups[other_index]: + group.update(source_groups.pop(other_index)) + changed = True + break + if changed: + break + + def in_scope(source: Optional[MemoryRecord]) -> bool: + if source is None: + return False + if flt.workspace_id and source.workspace_id != flt.workspace_id: + return False + if flt.repo_id is not None and source.repo_id != flt.repo_id: + return False + try: + return not flt.scopes or Scope(source.scope) in flt.scopes + except (TypeError, ValueError): + return False + + clusters: list[list[MemoryRecord]] = [] + for source_ids in source_groups: + sources = [store.get_memory(source_id) for source_id in source_ids] + records = [source for source in sources if in_scope(source)] + if records: + clusters.append(sorted(records, key=lambda memory: memory.id)) + return clusters + + +def _count_completed_derived(store, flt: SearchFilter, *, source: str, + relation: str) -> int: + """Count completed derived rows for an idempotent maintenance report.""" + derived_filter = _replace(flt, mtypes=[MemoryType.SEMANTIC]) + count = 0 + for derived in _scan_memories( + store, derived_filter, mtypes=[MemoryType.SEMANTIC], + batch_size=DISTILL_SCAN_LIMIT, max_records=DISTILL_CLUSTER_LIMIT, + ): + provenance = (derived.metadata or {}).get("provenance") or {} + if provenance.get("source") != source: + continue + cited = { + str(source_id) for source_id in ( + provenance.get(relation) or provenance.get("source_ids") or [] + ) if source_id + } + if not cited: + continue + attached = { + str(link["b"] if str(link["a"]) == derived.id else link["a"]) + for link in store.get_links(derived.id) + if link["relation"] == relation + } + if cited <= attached: + count += 1 + return count + +def _derived_cited_ids(derived: MemoryRecord, relation: str) -> set[str]: + """Return source ids recorded by a derived row's dedicated or legacy provenance.""" + metadata = derived.metadata if isinstance(derived.metadata, dict) else {} + nested = metadata.get("provenance") + nested = nested if isinstance(nested, dict) else {} + provenance = derived.provenance if isinstance(derived.provenance, dict) else {} + key = "profiles" if relation == PROFILE_RELATION else "consolidates" + for container in (provenance, nested): + values = container.get(key) or container.get("source_ids") or [] + if isinstance(values, str): + values = [values] + if isinstance(values, (list, tuple, set)): + return {str(source_id) for source_id in values if source_id} + return set() + + +def _derived_safety_is_current( + derived: MemoryRecord, sources: list[MemoryRecord], +) -> bool: + """Whether a complete derived row still reflects its source safety labels.""" + from engraphis.core.engine import _SENSITIVITY_RANK + + current_sensitivity = derived.sensitivity or "normal" + expected_sensitivity = max( + [current_sensitivity] + [(source.sensitivity or "normal") for source in sources], + key=lambda value: _SENSITIVITY_RANK.get(value, len(_SENSITIVITY_RANK)), + ) + if current_sensitivity != expected_sensitivity: + return False + # Inheritance is tightening-only: an already-untrusted derived row remains + # untrusted even after all of its sources are later approved. + return not ( + prompt_eligible(derived.provenance, derived.metadata) + and not _sources_are_trusted(sources) + ) + + +def _repair_derived_safety( + engine, flt: SearchFilter, *, provenance_source: str, relation: str, +) -> list[dict]: + """Repair safety on fully linked derived rows before source scans can skip them.""" + from engraphis.core.store import memory_matches_filter + + store = engine.store + derived_filter = _replace(flt, mtypes=[MemoryType.SEMANTIC]) + errors: list[dict] = [] + for derived in store.list_memories(derived_filter, include_invalid=True): + metadata = derived.metadata if isinstance(derived.metadata, dict) else {} + nested = metadata.get("provenance") + nested = nested if isinstance(nested, dict) else {} + provenance = derived.provenance if isinstance(derived.provenance, dict) else {} + if str( + provenance.get("source") or nested.get("source") or "" + ) != provenance_source: + continue + cited = _derived_cited_ids(derived, relation) + if not cited: + continue + attached = { + str(link["b"] if str(link["a"]) == derived.id else link["a"]) + for link in store.get_links(derived.id) + if link["relation"] == relation + } + if not cited <= attached: + continue + sources = [] + for source_id in sorted(cited): + source = store.get_memory(source_id) + if source is None or not memory_matches_filter( + source, flt, include_invalid=True, + ): + break + sources.append(source) + if len(sources) != len(cited) or _derived_safety_is_current(derived, sources): + continue + try: + sensitivity, trusted = _inherit_safety(engine, derived.id, sources) + store.audit( + "consolidation", "safety_repair", derived.id, + f"repaired {relation} safety for {len(sources)} sources " + f"(sensitivity={sensitivity}, trusted={trusted})", + ) + except Exception as exc: + errors.append(_error_entry(sources, exc)) + return errors + + +def _audit_consolidation_once(engine, action: str, target: str, detail: str) -> None: + """Record one completion audit even when a derived write was resumed.""" + exists = engine.store.conn.execute( + "SELECT 1 FROM audit WHERE actor=? AND action=? AND target=? LIMIT 1", + ("consolidation", action, target), + ).fetchone() + if exists is None: + engine.store.audit("consolidation", action, target, detail) + + +def _resume_structured_digests( + engine, cluster: list[MemoryRecord], *, supersede_sources: bool = False, + now: Optional[float] = None, +) -> None: + """Repair every structured fact already committed for this cluster.""" + source_by_id = {memory.id: memory for memory in cluster} + cluster_ids = set(source_by_id) + cited_sources: set[str] = set() + for existing, cited_ids in _derived_memories_for_source_subset( + engine.store, cluster[0], cluster_ids, + provenance_source="structured_consolidation", + ): + sources = [source_by_id[source_id] for source_id in cited_ids] + sensitivity, trusted = _inherit_safety(engine, existing.id, sources) + _ensure_derived_links(engine.store, existing.id, sources, "consolidates") + cited_sources.update(cited_ids) + structured = (existing.metadata or {}).get("structured_consolidation") or {} + audit = structured.get("llm") or {} + try: + confidence = float( + structured.get("confidence", existing.confidence or 0.0) + ) + except (TypeError, ValueError): + confidence = 0.0 + _audit_consolidation_once( + engine, "distill_structured", existing.id, + f"schema-distilled {len(sources)} memories; " + f"confidence={float(confidence):.2f}; sensitivity={sensitivity}; " + f"trusted={trusted}; prompt_sha256={audit.get('prompt_sha256', '')}", + ) + if supersede_sources: + at = time.time() if now is None else now + for memory in cluster: + if memory.id in cited_sources: + engine.store.close_validity( + memory.id, at=at, actor="consolidation", + reason="superseded by structured consolidation", + ) + + +def _ensure_derived_links(store, derived_id: str, sources: list[MemoryRecord], + relation: str) -> None: + """Complete an idempotent source-link set, allowing retries after partial writes.""" + existing = { + (link["a"], link["b"]) + for link in store.get_links(derived_id) + if link["relation"] == relation + } + for source in sources: + if (derived_id, source.id) in existing or (source.id, derived_id) in existing: + continue + store.add_link(derived_id, source.id, relation) + + +def _write_or_resume_digest(engine, cluster: list[MemoryRecord], *, content: str, + subject: str, now: float) -> tuple[str, bool]: + """Write a digest once, or finish one whose links were interrupted.""" + store = engine.store + source_ids = {memory.id for memory in cluster} + existing = _derived_memory_for_sources( + store, cluster[0], source_ids, provenance_source="consolidation", + ) + if existing is not None: + # A previous attempt may have committed the derived row before safety + # inheritance failed. Reapply it before treating the row as complete; otherwise + # the source links make the next sweep skip a secret/poisoned digest forever. + sensitivity, trusted = _inherit_safety(engine, existing.id, cluster) + _ensure_derived_links(store, existing.id, cluster, "consolidates") + _audit_consolidation_once( + engine, "distill", existing.id, + f"digested {len(cluster)} episodic memories " + f"(sensitivity={sensitivity}, trusted={trusted})", + ) + return existing.id, False + return _write_digest(engine, cluster, content=content, subject=subject, now=now), True + + +def _write_or_resume_profile(engine, name: str, etype: str, + sources: list[MemoryRecord], *, content: str, + now: float) -> tuple[str, bool]: + """Write a profile once, or finish one whose links were interrupted.""" + store = engine.store + existing = _derived_memory_for_sources( + store, sources[0], {memory.id for memory in sources}, + provenance_source="profile_consolidation", + ) + if existing is not None: + sensitivity, trusted = _inherit_safety(engine, existing.id, sources) + _ensure_derived_links(store, existing.id, sources, PROFILE_RELATION) + _audit_consolidation_once( + engine, "profile", existing.id, + f"profiled {len(sources)} memories about {name} " + f"(sensitivity={sensitivity}, trusted={trusted})", + ) + return existing.id, False + return _write_profile(engine, name, etype, sources, content=content, now=now), True + + +def _error_entry(cluster: list[MemoryRecord], exc: Exception) -> dict: + # Only the exception TYPE reaches the client-facing report. The message can + # echo internal details or carry stack-trace information; the full error is + # logged server-side by the caller instead. + return { + "source_ids": [memory.id for memory in cluster], + "error": type(exc).__name__, + } + + def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, min_cluster: int = MIN_CLUSTER, subject_jaccard: float = SUBJECT_JACCARD, archive_below: float = ARCHIVE_BELOW, dry_run: bool = False, @@ -130,11 +664,43 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id, scopes=MAINTENANCE_SCOPES) - episodic = store.list_memories( - _replace(flt, mtypes=[MemoryType.EPISODIC]), - limit=DISTILL_SCAN_LIMIT, - prompt_only=True, + report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, + "clusters_found": 0, "digests_created": [], "archived": [], + "skipped_already_consolidated": 0, "errors": []} + if not dry_run: + for provenance_source in ("consolidation", "structured_consolidation"): + report["errors"].extend(_repair_derived_safety( + engine, flt, provenance_source=provenance_source, + relation="consolidates", + )) + if not dry_run: + report["skipped_already_consolidated"] = _count_completed_derived( + store, flt, source="consolidation", relation="consolidates", + ) + if structured: + for retry_cluster in _structured_retry_clusters(store, flt): + try: + _resume_structured_digests( + engine, retry_cluster, + supersede_sources=bool(supersede_sources), now=now, + ) + except Exception as exc: + report["errors"].append(_error_entry(retry_cluster, exc)) + + distill_cursor = store.get_maintenance_cursor( + workspace_id, repo_id, DISTILL_CURSOR_NAME, ) + episodic, next_distill_cursor = _scan_memory_window( + store, flt, mtypes=[MemoryType.EPISODIC], + batch_size=DISTILL_SCAN_LIMIT, prompt_only=True, + max_records=DISTILL_CLUSTER_LIMIT, + exclude_relation="consolidates", + start_after_id=distill_cursor, + ) + if not dry_run: + store.set_maintenance_cursor( + workspace_id, repo_id, DISTILL_CURSOR_NAME, next_distill_cursor, + ) # A digest inherits its owner from its first source. Cluster only records that have # the exact same owner, otherwise a workspace sweep could write one repo's digest with # another repo's content (or mix scope visibility). @@ -146,9 +712,6 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, ) ] - report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, - "clusters_found": 0, "digests_created": [], "archived": [], - "skipped_already_consolidated": 0} if structured: report["structured"] = {"enabled": True, "attempted": 0, "succeeded": 0, "fallbacks": 0, "sources_superseded": 0} @@ -157,9 +720,38 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, # ── pass 1: distill recurring episodes into semantic digests ───────────── for cluster in clusters: + if structured and not dry_run: + try: + # Resume partial structured facts even when their remaining source + # subset is smaller than MIN_CLUSTER. + _resume_structured_digests( + engine, cluster, supersede_sources=bool(supersede_sources), now=now, + ) + except Exception as exc: + report["errors"].append(_error_entry(cluster, exc)) + continue if len(cluster) < min_cluster: continue report["clusters_found"] += 1 + # A prior attempt may have committed the derived row and only some links before + # failing. Complete that exact row first; otherwise the pending-count check below + # could strand the remaining sources forever. + existing = _derived_memory_for_sources( + store, cluster[0], {memory.id for memory in cluster}, + provenance_source="consolidation", + ) if not dry_run else None + if existing is not None: + try: + sensitivity, trusted = _inherit_safety(engine, existing.id, cluster) + _ensure_derived_links(store, existing.id, cluster, "consolidates") + _audit_consolidation_once( + engine, "distill", existing.id, + f"digested {len(cluster)} episodic memories " + f"(sensitivity={sensitivity}, trusted={trusted})", + ) + except Exception as exc: + report["errors"].append(_error_entry(cluster, exc)) + continue pending = [m for m in cluster if not _already_consolidated(store, m.id)] if len(pending) < min_cluster: report["skipped_already_consolidated"] += 1 @@ -199,9 +791,13 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, if supersede_sources: entry["would_supersede_sources"] = source_ids else: - ids = _write_structured_digests( - engine, cluster, structured_facts, subject=subject, now=now, - supersede_sources=bool(supersede_sources)) + try: + ids = _write_structured_digests( + engine, cluster, structured_facts, subject=subject, now=now, + supersede_sources=bool(supersede_sources)) + except Exception as exc: + report["errors"].append(_error_entry(cluster, exc)) + continue entry["ids"] = ids if ids: entry["id"] = ids[0] @@ -221,13 +817,22 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, if dry_run: entry["would_consolidate"] = entry.pop("consolidates") else: - entry["id"] = _write_digest(engine, cluster, content=content, - subject=subject, now=now) + try: + digest_id, created = _write_or_resume_digest( + engine, cluster, content=content, subject=subject, now=now, + ) + except Exception as exc: + report["errors"].append(_error_entry(cluster, exc)) + continue + entry["id"] = digest_id + if not created: + entry["resumed"] = True report["digests_created"].append(entry) # ── pass 2: archive fully-decayed transient memories ───────────────────── - for m in store.list_memories(_replace(flt, mtypes=TRANSIENT_TYPES), - limit=DISTILL_SCAN_LIMIT): + for m in _scan_memories( + store, flt, mtypes=TRANSIENT_TYPES, batch_size=DISTILL_SCAN_LIMIT, + ): if m.pinned: continue r = scoring.retention(m.stability, m.last_access, now) @@ -237,13 +842,18 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, report["archived"].append({"id": m.id, "retention": round(r, 4), "tokens_freed": _mem_tokens(m)}) if not dry_run: - store.close_validity( - m.id, actor="consolidation", - reason=f"retention {r:.4f} below {archive_below} (consolidation sweep)") + try: + store.close_validity( + m.id, at=now, actor="consolidation", + reason=f"retention {r:.4f} below {archive_below} (consolidation sweep)") + except Exception as exc: + report["errors"].append(_error_entry([m], exc)) + report["archived"].pop() + archived_tokens -= _mem_tokens(m) + continue # Preserve the vector as historical evidence. Temporal filtering keeps the # archived row out of current recall while allowing an explicit ``as_of`` # query to reproduce the semantic result from when it was live. - # ── compaction summary: the payoff of the sweep, as a number ───────────── report["compaction"] = { "distilled": _compaction(distilled_before, distilled_after, @@ -396,8 +1006,12 @@ def _inherit_safety(engine, memory_id: str, sources: list[MemoryRecord]) -> tupl and _sources_are_trusted(sources)) provenance = dict(record.provenance or {}) provenance["trusted"] = trusted + if not trusted: + # A source can be downgraded after a derived row was committed. Reopening + # approval keeps retry-repaired metadata truthful instead of leaving an + # approved-looking row that only happens to fail prompt eligibility. + provenance["review_state"] = REVIEW_PENDING metadata = dict(record.metadata or {}) - metadata["provenance"] = dict(provenance) engine.store.conn.execute( "UPDATE memories SET sensitivity=?, metadata=?, provenance=? WHERE id=?", (sensitivity, @@ -415,7 +1029,29 @@ def _sources_are_trusted(sources: list[MemoryRecord]) -> bool: def _already_consolidated(store, memory_id: str) -> bool: - return any(link["relation"] == "consolidates" for link in store.get_links(memory_id)) + for link in store.get_links(memory_id): + if link["relation"] != "consolidates": + continue + other_id = link["b"] if link["a"] == memory_id else link["a"] + derived = store.get_memory(other_id) + if derived is None: + return True + provenance = (derived.metadata or {}).get("provenance") or {} + cited = { + str(source_id) for source_id in ( + provenance.get("consolidates") or provenance.get("source_ids") or [] + ) if source_id + } + if not cited: + return True + attached = { + str(row["b"] if str(row["a"]) == str(other_id) else row["a"]) + for row in store.get_links(other_id) + if row["relation"] == "consolidates" + } + if cited <= attached: + return True + return False def _common_tokens(cluster: list[MemoryRecord], k: int = 5) -> list[str]: @@ -665,11 +1301,11 @@ def _write_digest(engine, cluster: list[MemoryRecord], *, content: str, subject: keywords=_common_tokens(cluster, k=8), metadata={"provenance": {"source": "consolidation", "trusted": trusted, "consolidates": [m.id for m in cluster]}}, + valid_from=now, resolve_conflicts=False, # the digest is new by construction ) sensitivity, trusted = _inherit_safety(engine, digest_id, cluster) - for m in cluster: - engine.store.add_link(digest_id, m.id, "consolidates") + _ensure_derived_links(engine.store, digest_id, cluster, "consolidates") engine.store.audit("consolidation", "distill", digest_id, f"digested {len(cluster)} episodic memories " f"(sensitivity={sensitivity}, trusted={trusted})") @@ -722,15 +1358,15 @@ def _write_structured_digests(engine, cluster: list[MemoryRecord], facts: list[d mtype=MemoryType.SEMANTIC, scope=Scope(first.scope), title=(fact.get("title") or f"Consolidated: {subject}")[:200], importance=importance, + confidence=fact.get("confidence", 0.0), keywords=fact.get("keywords") or _common_tokens(sources, k=8), - metadata=metadata, resolve_conflicts=False, + metadata=metadata, valid_from=now, resolve_conflicts=False, _trusted_graph_keys=frozenset( key for key in ("entities", "relations") if key in metadata ), ) sensitivity, trusted = _inherit_safety(engine, mid, sources) - for memory in sources: - engine.store.add_link(mid, memory.id, "consolidates") + _ensure_derived_links(engine.store, mid, sources, "consolidates") audit = fact.get("llm") or {} engine.store.audit("consolidation", "distill_structured", mid, f"schema-distilled {len(sources)} memories; " @@ -764,9 +1400,9 @@ def consolidate_profiles(engine, *, workspace_id: str, repo_id: Optional[str] = Deterministic and offline: entities come from the knowledge graph (``store.list_entities``); a memory belongs to an entity's profile if the entity's - name occurs in its title/content (case-insensitive), within the same scope and the - default (live) validity window. A profile is a ``semantic`` memory linked to every - source via ``profiles`` and provenance ``source='profile_consolidation'``. + name's bounded memory↔entity incidence rows identify the sources within the same + scope and the default (live) validity window. A profile is a ``semantic`` memory + linked to every source via ``profiles`` and provenance ``source='profile_consolidation'``. Idempotent (mirrors the distill pass): if any candidate source is already in a profile, the entity is skipped rather than re-summarized. Governed like every other @@ -777,29 +1413,73 @@ def consolidate_profiles(engine, *, workspace_id: str, repo_id: Optional[str] = flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id, scopes=MAINTENANCE_SCOPES) report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, - "entities_considered": 0, "profiles_created": [], "skipped_existing": 0} - - live = [ - memory for memory in store.list_memories( - _replace(flt, mtypes=DURABLE_TYPES), - limit=PROFILE_SCAN_LIMIT, - prompt_only=True, + "entities_considered": 0, "profiles_created": [], "skipped_existing": 0, + "errors": []} + if not dry_run: + report["errors"].extend(_repair_derived_safety( + engine, flt, provenance_source="profile_consolidation", + relation=PROFILE_RELATION, + )) + + profile_cursor = store.get_maintenance_cursor( + workspace_id, repo_id, PROFILE_CURSOR_NAME, + ) + profile_memories, next_profile_cursor = _scan_memory_window( + store, flt, mtypes=DURABLE_TYPES, + batch_size=PROFILE_SCAN_LIMIT, prompt_only=True, + max_records=PROFILE_MEMORY_LIMIT, exclude_relation=PROFILE_RELATION, + start_after_id=profile_cursor, + ) + if not dry_run: + store.set_maintenance_cursor( + workspace_id, repo_id, PROFILE_CURSOR_NAME, next_profile_cursor, ) + live = [ + memory for memory in profile_memories if memory.metadata.get("provenance", {}).get("source") != "profile_consolidation" ] p_before = p_after = 0 - for ent in store.list_entities(flt, limit=2000): + entities = store.list_entities(flt, limit=PROFILE_ENTITY_LIMIT) + entity_ids = {entity.id for entity in entities} + live_by_id = {memory.id: memory for memory in live} + linked_by_entity: dict[str, set[str]] = {} + if live_by_id and entity_ids: + for link in store.list_memory_entities(flt, memory_ids=list(live_by_id)): + entity_id = str(link.get("entity_id") or "") + memory_id = str(link.get("memory_id") or "") + if entity_id in entity_ids and memory_id in live_by_id: + linked_by_entity.setdefault(entity_id, set()).add(memory_id) + + for ent in entities: name = (ent.name or "").strip() if len(name) < PROFILE_MIN_NAME_LEN: continue - pattern = _entity_pattern(name) - matching = [m for m in live if pattern.search(f"{m.title} {m.content}")] + matching = [live_by_id[memory_id] + for memory_id in linked_by_entity.get(ent.id, set())] for sources in _partition_by_visibility_owner(matching): if len(sources) < min_mentions: continue report["entities_considered"] += 1 + existing = _derived_memory_for_sources( + store, sources[0], {memory.id for memory in sources}, + provenance_source="profile_consolidation", + ) if not dry_run else None + if existing is not None: + try: + sensitivity, trusted = _inherit_safety(engine, existing.id, sources) + _ensure_derived_links(store, existing.id, sources, PROFILE_RELATION) + _audit_consolidation_once( + engine, "profile", existing.id, + f"profiled {len(sources)} memories about {name} " + f"(sensitivity={sensitivity}, trusted={trusted})", + ) + except Exception as exc: + report["errors"].append(_error_entry(sources, exc)) + continue + report["skipped_existing"] += 1 + continue if any(_in_profile(store, m.id) for m in sources): report["skipped_existing"] += 1 continue @@ -813,8 +1493,16 @@ def consolidate_profiles(engine, *, workspace_id: str, repo_id: Optional[str] = if dry_run: entry["would_profile"] = [m.id for m in sources] else: - entry["id"] = _write_profile(engine, name, ent.ntype, sources, - content=content, now=now) + try: + profile_id, created = _write_or_resume_profile( + engine, name, ent.ntype, sources, content=content, now=now, + ) + except Exception as exc: + report["errors"].append(_error_entry(sources, exc)) + continue + entry["id"] = profile_id + if not created: + entry["resumed"] = True report["profiles_created"].append(entry) report["compaction"] = _compaction(p_before, p_after, len(report["profiles_created"])) @@ -854,11 +1542,11 @@ def _write_profile(engine, name: str, etype: str, sources: list[MemoryRecord], metadata={"provenance": {"source": "profile_consolidation", "trusted": trusted, "entity": name, "etype": etype, "profiles": [m.id for m in sources]}}, + valid_from=now, resolve_conflicts=False, # a profile is new by construction ) sensitivity, trusted = _inherit_safety(engine, profile_id, sources) - for m in sources: - engine.store.add_link(profile_id, m.id, PROFILE_RELATION) + _ensure_derived_links(engine.store, profile_id, sources, PROFILE_RELATION) engine.store.audit("consolidation", "profile", profile_id, f"profiled {len(sources)} memories about {name} " f"(sensitivity={sensitivity}, trusted={trusted})") diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index 7a999f9e..0e51cdbf 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -30,6 +30,7 @@ from engraphis.backends.vector_sqlitevec import get_vector_index from engraphis.core import scoring from engraphis.core.adaptive_context import AdaptiveContextResult, fit_recent_history +from engraphis.core.conflicts import detect_conflicts from engraphis.core.interfaces import ( MemoryRecord, MemoryType, @@ -55,9 +56,15 @@ CANDIDATE_DEPTH_MODES, RETRIEVAL_PROFILES, ) -from engraphis.core.resolve import RELATED_SIM_FLOOR, Resolution, ResolutionOp, resolve +from engraphis.core.resolve import ( + CONFLICT_RELATION, + RELATED_SIM_FLOOR, + Resolution, + ResolutionOp, + resolve, +) from engraphis.core.secrets import reject_secrets -from engraphis.core.store import Store, memory_matches_filter, now_ts +from engraphis.core.store import Store, _dumps, memory_matches_filter, now_ts from engraphis.core.textutil import estimate_tokens, jaccard, tokenize logger = logging.getLogger("engraphis.core.engine") @@ -76,6 +83,17 @@ # Bounded so hub memories don't accrete unbounded link lists (link quality > quantity). EVOLVE_MAX_LINKS = 3 +# The deterministic detector's contradiction/obsolete reports below this severity are +# too weak to justify a durable ``conflicts_with`` relation. The detector floors its +# own reports at 0.74 (numeric) / 0.78 (polarity) / 0.82 (assertion), so this only +# filters out margin-of-error edge cases, keeping the repair trigger conservative. +CONFLICT_MIN_SEVERITY = 0.7 + +# Deterministic confidence penalty applied to both sides of a persisted conflict +# repair. Bounded and explainable: the ``conflicts_with`` link + audit row make the +# discount auditable, and an explicit human resolution can restore confidence. +CONFLICT_CONFIDENCE_FACTOR = 0.8 + # Metadata keys that feed the entity/edge graph under the *trusted* # provenance.source="structured_extractor" label — i.e. "a configured Extractor produced # this". See _has_structured_graph_metadata / _trusted_graph_hints. @@ -430,7 +448,8 @@ def _rebuild_versioned_embeddings(self) -> None: def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = None, session_id: Optional[str] = None, mtype: MemoryType = MemoryType.SEMANTIC, scope: Optional[Scope] = None, title: str = "", importance: float = 0.0, - keywords: Optional[list] = None, metadata: Optional[dict] = None, + confidence: Optional[float] = None, keywords: Optional[list] = None, + metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, candidate_k: int = 5, subject_key: str = "", claim_kind: str = "", _trusted_graph_keys: Optional[frozenset] = None) -> str: @@ -440,7 +459,8 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = """ return self.remember_with_resolution( content, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, + mtype=mtype, scope=scope, title=title, importance=importance, + confidence=confidence, keywords=keywords, metadata=metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, candidate_k=candidate_k, subject_key=subject_key, claim_kind=claim_kind, _trusted_graph_keys=_trusted_graph_keys, @@ -449,7 +469,8 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = def remember_with_resolution(self, content: str, *, workspace_id: str, repo_id: Optional[str] = None, session_id: Optional[str] = None, mtype: MemoryType = MemoryType.SEMANTIC, scope: Optional[Scope] = None, - title: str = "", importance: float = 0.0, keywords: Optional[list] = None, + title: str = "", importance: float = 0.0, + confidence: Optional[float] = None, keywords: Optional[list] = None, metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, candidate_k: int = 5, subject_key: str = "", claim_kind: str = "", @@ -560,7 +581,8 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, return self._resolve_and_store( content, text=text, vec=vec, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, mtype=mtype, scope=scope, title=title, - importance=importance, keywords=keywords, metadata=write_metadata, + importance=importance, confidence=confidence, keywords=keywords, + metadata=write_metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, candidate_k=candidate_k, subject_key=subject_key, claim_kind=claim_kind, trusted_graph_keys=_trusted_graph_keys, @@ -575,7 +597,8 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarray], workspace_id: str, repo_id: Optional[str], session_id: Optional[str], mtype: MemoryType, scope: Scope, - title: str, importance: float, keywords: Optional[list], + title: str, importance: float, confidence: Optional[float], + keywords: Optional[list], metadata: Optional[dict], valid_from: Optional[float], resolve_conflicts: bool, candidate_k: int, subject_key: str, claim_kind: str, @@ -589,12 +612,12 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra genuinely inherited from an ``Extractor``; everything else is treated as caller-supplied — see ``_rehome_untrusted_graph_hints``.""" poisoning = poisoning or PoisoningDecision(False) - decision, neighbors = None, [] + decision, neighbors, conflicted_with = None, [], None # Untrusted records are retained as passive inspection evidence. They may # not deduplicate into, invalidate, relate to, reinforce, or otherwise # mutate higher-trust memory; that is a trust lattice, not a detector score. if resolve_conflicts and trusted_write and not poisoning.quarantined: - decision, neighbors = self._resolve_against_neighbors( + decision, neighbors, conflicted_with = self._resolve_against_neighbors( text, vec, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, scope=scope, mtype=mtype, candidate_k=candidate_k, subject_key=subject_key, @@ -648,6 +671,12 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra "valid_from cannot predate the memory it supersedes; " "record the historical interval separately or correct the older memory" ) + if decision is not None and decision.op == ResolutionOp.INVALIDATE: + # A keyed/temporal supersession is the resolution: the detector may have + # flagged the superseded predecessor earlier, but a closed record is no + # longer a live conflict — drop the repair so no ``conflicts_with`` link, + # metadata marker, or confidence discount is written for this write. + conflicted_with = None if decision is not None and decision.op == ResolutionOp.NOOP: self.store.reinforce(decision.target_id, boost=scoring.INTERACTION_BOOST["create"]) @@ -670,7 +699,12 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra # Persist the supersession pointer on the new record so the chain is # queryable later (why/timeline/inspector), not only in the audit log. meta["supersedes"] = [decision.target_id] - + if conflicted_with: + # Surface the deterministic conflict repair on the new record so + # downstream (recall/why/inspector) can explain the lowered confidence + # without a separate graph walk. The neighbor side already carries the + # durable ``conflicts_with`` link; this is a minimal, queryable mirror. + meta["conflict_with"] = [conflicted_with] if poisoning.quarantined: # Retained only for governance inspection: an untrusted payload must not # elevate itself through caller-supplied retention supervision. @@ -683,10 +717,25 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra meta["retention_supervision"] = retention_signal quarantine_at = valid_from if valid_from is not None else now_ts() + # Confidence defaults to 1.0 (no scoring change for ordinary writes); a + # caller-supplied value wins, and the structured-extraction metadata hint is + # honored when present so persisted verdicts actually reach scoring. + if confidence is None: + confidence = meta.get("confidence", 1.0) + try: + confidence = float(confidence) + except (TypeError, ValueError): + confidence = 1.0 + confidence = max(0.0, min(1.0, confidence)) if math.isfinite(confidence) else 1.0 + if conflicted_with: + # The new fact directly contradicts a live memory without a safe + # supersession; neither side gets to claim full confidence. + confidence = round(confidence * CONFLICT_CONFIDENCE_FACTOR, 4) rec = MemoryRecord( id="", content=content, mtype=mtype, scope=scope, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, title=title, importance=importance, - stability=stability, subject_key=subject_key, claim_kind=claim_kind, + stability=stability, confidence=confidence, + subject_key=subject_key, claim_kind=claim_kind, keywords=keywords or [], metadata=meta, # A zero-length validity interval retains the record/audit trail while the # existing temporal filters keep it out of every normal recall arm. @@ -815,6 +864,31 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra return out linked = self._evolve(mid, neighbors) if trusted_write else [] + if conflicted_with: + # Deterministic conflict repair: persist the ``conflicts_with`` relation + # (with the real new-memory id), the audit row, and a bounded confidence + # discount on BOTH sides. Non-fatal: a storage hiccup here must not fail + # the write — the conflict metadata on the new record already surfaced it. + try: + self.store.add_link( + mid, conflicted_with, CONFLICT_RELATION, + reason=( + "detector=contradiction; deterministic contradiction " + "(no safe supersession)" + ), + valid_from=valid_from, + ) + self.store.audit( + "resolver", "conflict_detected", conflicted_with, + f"new_memory={mid}; deterministic contradiction (no safe supersession)", + ) + self.store.conn.execute( + "UPDATE memories SET confidence=MIN(confidence, ?) WHERE id=?", + (round(CONFLICT_CONFIDENCE_FACTOR, 4), conflicted_with), + ) + self.store.conn.commit() + except Exception: # noqa: BLE001 — best-effort repair, never fail the write + pass if decision is not None and decision.op == ResolutionOp.RELATE: related_to = decision.target_id if related_to and not self.store.has_link(mid, related_to): @@ -825,6 +899,8 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra } else: out = {"id": mid, "op": "add", "reason": decision.reason if decision else ""} + if conflicted_with: + out["conflict_with"] = conflicted_with if linked: out["linked"] = linked return out @@ -985,9 +1061,10 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id valid_at: Optional[float] = None, content: Optional[str] = None): """Fetch same-scope neighbors via the vector index and run the deterministic - resolver (``core.resolve``). Returns ``(decision, neighbors)`` so the caller can - also evolve the neighborhood. Never raises — a broken/missing index degrades to - "no neighbors found" (ADD), not a write failure.""" + resolver (``core.resolve``). Returns ``(decision, neighbors, conflicted_with)`` + so the caller can also evolve the neighborhood and persist a conflict repair. + Never raises — a broken/missing index degrades to "no neighbors found" (ADD), + not a write failure.""" flt = SearchFilter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id if scope == Scope.SESSION else None, @@ -1074,10 +1151,59 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id for record in authoritative: if record.id not in known_ids: neighbors.append((1.0, record)) - return resolve( + decision = resolve( text, neighbors, subject_key=subject_key, claim_kind=claim_kind, candidate_content=content, - ), neighbors + ) + # Repair trigger: when the resolver cannot safely supersede (INVALIDATE/NOOP), + # surface a genuine high-severity contradiction as a persisted relation instead + # of a silent coin-flip ADD. ``_repair_conflicts`` is a pure detector (self- + # guarding, no-op on any failure); persistence happens in ``_resolve_and_store`` + # once the new memory exists and its real id is known. + conflicted_with: Optional[str] = None + if decision.op not in (ResolutionOp.INVALIDATE, ResolutionOp.NOOP): + conflicted_with = self._repair_conflicts( + "", text, neighbors, workspace_id=workspace_id, + repo_id=repo_id, valid_at=valid_at, + ) + return decision, neighbors, conflicted_with + + def _repair_conflicts(self, new_id: str, new_text: str, neighbors: list, *, + workspace_id: str, repo_id: Optional[str], + valid_at: Optional[float]) -> Optional[str]: + """Detect a deterministic, high-severity contradiction among the neighbors the + resolver could not safely supersede. + + The resolver only INVALIDATEs on a shared claim key or on strong joint + lexical+semantic evidence; a true semantic contradiction with little token + overlap otherwise lands as a plain ADD — a silent coin-flip between two live + facts. This hook runs ``core.conflicts.detect_conflicts`` over the same scoped, + prompt-eligible neighbor set that resolution already saw and returns the + highest-severity genuine contradiction (``contradiction`` or ``obsolete``) + when no safe supersession happened. The caller persists the ``conflicts_with`` + relation, audit row, and confidence discount after the new memory exists. + + Conservative by construction: the detector is deterministic and precision-first, + this runs only for trusted, non-quarantined writes whose resolution produced no + INVALIDATE/NOOP, only top-K neighbors are considered (the same bounded set the + resolver saw), duplicates/refinements never create a link, and any failure + degrades to a no-op — never a write error. + """ + try: + conflicts = detect_conflicts(new_text, (rec for _, rec in neighbors)) + except Exception: + return None + if not conflicts: + return None + conflict = conflicts[0] + if conflict.type not in ("contradiction", "obsolete"): + return None + if conflict.severity < CONFLICT_MIN_SEVERITY: + return None + target_id = conflict.memory_id + if not target_id or target_id == new_id: + return None + return target_id # ── ingest: extract-then-remember ─────────────────────────────────────────── def ingest(self, text: str, *, workspace_id: str, repo_id: Optional[str] = None, @@ -1592,7 +1718,21 @@ def recall_proactive(self, *, workspace_id: str, repo_id: Optional[str] = None, ) now = now_ts() scored = [] - for rec in self.store.list_memories(flt, limit=500, prompt_only=prompt_only): + always: list = [] + candidates = self.store.list_memories(flt, limit=500, prompt_only=prompt_only) + overrides = self.store.list_proactive_overrides(flt, prompt_only=prompt_only) + records = {rec.id: rec for rec in [*candidates, *overrides]}.values() + # SQLite's timestamp ordering is not total when records share an ingestion + # timestamp. Use the id as a stable final key so the agenda does not change + # between calls (or between the bounded and override queries). + def stable_record_key(rec: MemoryRecord) -> tuple: + ingested_at = rec.ingested_at + return ( + ingested_at is None, + -(float(ingested_at) if ingested_at is not None else 0.0), + rec.id, + ) + for rec in records: eligible = ( prompt_eligible(rec.provenance, rec.metadata) if prompt_only @@ -1600,9 +1740,24 @@ def recall_proactive(self, *, workspace_id: str, repo_id: Optional[str] = None, ) if not eligible: continue + # Per-memory proactive rules: a user-flagged ``metadata["proactive"]`` value + # ("always" | "never") overrides the score, and ``pinned`` always includes + # (the Mem0-style add-to-context analogue). "never" excludes regardless of + # importance so a user can silence a memory from the agenda. + proactive = (rec.metadata or {}).get("proactive") + if str(proactive).lower() == "never": + continue + if str(proactive).lower() == "always" or rec.pinned: + always.append(rec) + continue scored.append((scoring.score_proactive(rec, now=now), rec)) - scored.sort(key=lambda t: t[0], reverse=True) + always.sort(key=stable_record_key) + scored.sort(key=lambda t: (-t[0], *stable_record_key(t[1]))) top = [r for _, r in scored[:k]] + if always: + # Keep the user's explicit choices first, then the score-ranked remainder. + top = always + [r for r in top if r.id not in {a.id for a in always}] + top = top[:k] last_session: dict = {} if repo_id: @@ -1744,11 +1899,18 @@ def approve_for_prompt(self, memory_id: str, *, reviewer: str, old = self.store.get_memory(memory_id) if old is None: raise KeyError(f"no memory with id '{memory_id}'") - # Approval is a one-way ceremony for pending/quarantined evidence. Repeating it - # on an approved successor only duplicates prompt-visible content and weakens the - # audit story; a human correction must use the governed correction path instead. + # Normal local-agent writes are already approved and do not need an owner + # ceremony. Treat an explicit retry against such a record as an idempotent + # no-op so older clients that still call the former approval step do not fail + # after upgrading. A human correction still uses the governed correction path. if provenance_is_approved(old.provenance): - raise ValueError("memory is already approved") + return { + "id": old.id, + "approved_from": old.provenance.get("approved_from"), + "reviewer": str( + old.metadata.get("approval", {}).get("reviewer", reviewer) + ), + } now = now_ts() if ( @@ -1835,6 +1997,18 @@ def approve_for_prompt(self, memory_id: str, *, reviewer: str, self.store.conn.commit() if old.pinned: self.store.set_pinned(result["id"], True) + # Carry the proactive-agenda flag ("always"/"never") onto the approved + # successor. The user's explicit agenda choice is a governance decision, + # not review-dependent content, so it must survive the approval ceremony. + if (old.metadata or {}).get("proactive"): + successor = self.store.get_memory(result["id"]) + successor_meta = dict(successor.metadata or {}) if successor else {} + successor_meta.setdefault("proactive", old.metadata["proactive"]) + self.store.conn.execute( + "UPDATE memories SET metadata=? WHERE id=?", + (_dumps(successor_meta), result["id"]), + ) + self.store.conn.commit() self.store.audit( "human_review", "approve", result["id"], f"from={old.id}; reviewer={reviewer[:200]}; reason={reason[:500]}", diff --git a/engraphis/core/interfaces.py b/engraphis/core/interfaces.py index 4cd0538a..496b8816 100644 --- a/engraphis/core/interfaces.py +++ b/engraphis/core/interfaces.py @@ -88,6 +88,11 @@ class MemoryRecord: provenance: dict[str, Any] = field(default_factory=dict) embedding: Optional[np.ndarray] = None valid_to_recorded_at: Optional[float] = None # when valid_to was learned + # Keep additions after the established positional fields; callers may construct + # records positionally even though keyword construction is preferred. + pinned_at: Optional[float] = None # system-time when a pin last became effective + unpinned_at: Optional[float] = None # system-time when an unpin became effective + confidence: float = 1.0 # 0..1, extraction/model confidence (scoring multiplier) @dataclass @@ -312,10 +317,15 @@ def embedder_capabilities(embedder: Any) -> dict[str, Any]: @runtime_checkable class VectorIndex(Protocol): - """Approximate nearest-neighbour index over embeddings (§6.2).""" - def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None) -> None: ... + """Approximate nearest-neighbour index over embeddings (§6.2). + + ``commit=False`` keeps derived-index writes inside a caller-owned transaction; + existing callers retain the historical committing default. + """ + def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = None, + *, commit: bool = True) -> None: ... def search(self, vec: np.ndarray, k: int, *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]: ... - def delete(self, ids: list[str]) -> None: ... + def delete(self, ids: list[str], *, commit: bool = True) -> None: ... @runtime_checkable diff --git a/engraphis/core/poisoning.py b/engraphis/core/poisoning.py index 48ab10a3..25c2c903 100644 --- a/engraphis/core/poisoning.py +++ b/engraphis/core/poisoning.py @@ -456,6 +456,10 @@ def apply_quarantine_metadata(metadata: Mapping[str, Any], provenance.update({ "trusted": False, "quarantined": True, + # Quarantine always overrides approval: a detected payload must never keep + # an approved review state (e.g. a local-agent write) that would hide it + # from the review inbox. + "review_state": QUARANTINE_STATE, "quarantine_policy": decision.policy, "quarantine_reasons": list(decision.reasons), }) diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py index faa10aab..dd1171b4 100644 --- a/engraphis/core/recall.py +++ b/engraphis/core/recall.py @@ -72,6 +72,22 @@ PROMPT_ONLY_MIN_CANDIDATES = 256 PROMPT_ONLY_MAX_CANDIDATES = 1024 +# Provenance sources that mark a memory as the durable *product* of consolidation +# (sleep-time distill digests, schema-distilled facts, and entity profiles — see +# core/consolidate.py). Recall gives these consolidated summaries a small, +# deterministic additive bonus after normalization so a digest retrieved alongside +# its raw source episodes is preferred, while an ordinary memory is never penalized. +CONSOLIDATION_SOURCES = frozenset({ + "consolidation", + "structured_consolidation", + "profile_consolidation", +}) +# Additive bonus applied to the fused score of consolidated digests/profiles once +# every arm contribution has been min-max normalized. Deliberately small: it is a +# preference signal, not a relevance substitute — a raw episode that actually matches +# the query keeps outranking a digest the query merely grazes. +CONSOLIDATION_BONUS = 0.05 + @dataclass class RecallResult: @@ -323,7 +339,8 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, memory_id for run in query_runs for arm in ("vector", "lexical", "graph", "code") - for memory_id in run[arm] + for memory_id, _score in _finite_arm_items(run.get(arm)) + if isinstance(memory_id, str) and memory_id }) fetched = self.store.get_memories(candidate_ids) recs: dict[str, MemoryRecord] = {} @@ -341,7 +358,7 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, recs[mid] = rec can_expand = any( - enabled and len(run[arm]) >= arm_candidate_k + len(_finite_arm_items(run.get(arm))) >= arm_candidate_k for run in query_runs for arm, enabled in ( ("vector", run["config"].vector), @@ -349,6 +366,7 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, ("graph", run["config"].graph), ("code", run["config"].code), ) + if enabled ) if ( not prompt_only @@ -423,6 +441,15 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, if mid in arm_state["raw"][name] ] fusion_score = base + 0.5 * rrf.get(mid, 0.0) + if _consolidated_source(rec): + # Small deterministic preference for consolidated digests/profiles + # (post-normalization constant; see CONSOLIDATION_BONUS). Kept out + # of the base score so raw evidence comparisons stay untouched. + fusion_score += CONSOLIDATION_BONUS + evidence = ( + _consolidation_evidence(rec, store=self.store, flt=flt) + if _consolidated_source(rec) else [] + ) arm = ( "code" if "code" in arms else (arms[0] if len(arms) == 1 else ("hybrid" if arms else "fused")) @@ -456,6 +483,10 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, "calibrated_score": fusion_score, "arm_agreement": len(arms), "arms": arms, + "consolidation_bonus": ( + CONSOLIDATION_BONUS if _consolidated_source(rec) else 0.0 + ), + "consolidation_source_ids": evidence, } # Tie-break on id so equal scores get a stable, process-independent order. scored.sort(key=lambda c: (-c.score, c.id)) @@ -546,6 +577,12 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, "claim_kind": c.record.claim_kind, "retention": round(scoring.retention(c.record.stability, c.record.last_access, now), 4), "provenance": c.record.provenance, + # Consolidated digests/profiles expose the ids of the source memories + # they summarize as citable evidence (never their bodies — see + # ``_consolidation_evidence``). Ordinary memories carry no such field. + "consolidation_source_ids": ( + _consolidation_evidence(c.record, store=self.store, flt=flt) + ), } for c in final] context, packed_chunks, usage = self.context_packer.pack(query, final, budget) trace = None @@ -591,7 +628,15 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, ), token_counter=getattr(self.context_packer, "count_tokens", None), source_metadata={ - candidate.id: _source_safety_metadata(candidate.record) + candidate.id: { + **_source_safety_metadata(candidate.record), + **( + {"consolidation_source_ids": _consolidation_evidence( + candidate.record, store=self.store, flt=flt + )} + if _consolidated_source(candidate.record) else {} + ), + } for candidate in final if candidate.record is not None }, @@ -1176,7 +1221,13 @@ def _graph_arm_1hop( def _seed_entity_map( self, query: str, flt: SearchFilter, *, limit: int = 2048, ) -> dict[str, str]: - """Return a bounded, scoped set of entity names that may occur in ``query``.""" + """Return a bounded, scoped set of entity names that may occur in ``query``. + + Direct name matches come first. When they are thin, a second pass resolves the + query against canonical entity names so an alias member ("Open AI") seeds the + whole canonical group whose representative ("OpenAI") appears in the query — + the graph arm otherwise returns nothing on paraphrases. + """ terms = sorted({ term.casefold() for term in re.findall(r"[\w@#.+-]+", query) if len(term) >= 2 @@ -1208,10 +1259,82 @@ def _seed_entity_map( sql += " WHERE " + " AND ".join(clauses) sql += " ORDER BY id LIMIT ?" params.append(max(0, int(limit))) - return { + seeds = { r["id"]: r["name"] for r in self.store.conn.execute(sql, params).fetchall() } + if seeds: + # Expand to the full canonical group: when a query matches one member of a + # canonical alias group, every member is a valid seed (the graph arm should + # not depend on which spelling the query happened to use). The expansion + # must stay inside the caller's scope — an unscoped JOIN here would let a + # scoped recall pull another workspace's members of the same canonical + # group into the seeds. Use the same include_ancestors semantics as the + # initial seed query and the canonical fallback below. + group_clauses = [] + if flt.workspace_id: + if flt.include_ancestors: + group_clauses.append("(e.workspace_id=? OR e.workspace_id IS NULL)") + else: + group_clauses.append("e.workspace_id=?") + params_group = [flt.workspace_id] + else: + params_group = [] + if flt.repo_id: + if flt.include_ancestors: + group_clauses.append("(e.repo_id=? OR e.repo_id IS NULL)") + else: + group_clauses.append("e.repo_id=?") + params_group.append(flt.repo_id) + marks = ",".join("?" for _ in seeds) + expanded = self.store.conn.execute( + "SELECT e.id, e.name FROM entities e WHERE e.canonical_id IN (" + "SELECT COALESCE(NULLIF(e2.canonical_id, ''), e2.id) FROM entities e2 " + f"WHERE e2.id IN ({marks})" + + ((" AND " + " AND ".join(group_clauses)) if group_clauses else "") + + ") AND " + + (" AND ".join(group_clauses) if group_clauses else "1=1") + + " LIMIT ?", + # Placeholder order matches the SQL text: subquery marks, subquery + # scope clauses, outer scope clauses, then the LIMIT. + list(seeds) + params_group + params_group + [max(0, int(limit))], + ).fetchall() + return {r["id"]: r["name"] for r in expanded} or seeds + # Canonical fallback: an entity whose representative name appears in the query + # (even when the stored member spelling differs) seeds the whole group. The + # JOIN introduces a second `entities` alias, so every scope clause must be + # qualified with `e.` to avoid an ambiguous-column error. + canonical_clauses = [] + if flt.workspace_id: + if flt.include_ancestors: + canonical_clauses.append("(e.workspace_id=? OR e.workspace_id IS NULL)") + else: + canonical_clauses.append("e.workspace_id=?") + if flt.repo_id: + if flt.include_ancestors: + canonical_clauses.append("(e.repo_id=? OR e.repo_id IS NULL)") + else: + canonical_clauses.append("e.repo_id=?") + canonical_clauses.append( + "(" + " OR ".join("instr(lower(c.name), ?) > 0" for _ in terms) + ")" + ) + sql2 = ( + "SELECT DISTINCT e.id, e.name FROM entities e " + "JOIN entities c ON c.id = COALESCE(NULLIF(e.canonical_id, ''), e.id) " + "WHERE " + " AND ".join(canonical_clauses) + + " ORDER BY e.id LIMIT ?" + ) + # Scope params (workspace_id/repo_id) come first, then the name terms. + scope_params = [] + if flt.workspace_id: + scope_params.append(flt.workspace_id) + if flt.repo_id: + scope_params.append(flt.repo_id) + canonical_params = scope_params + terms + [max(0, int(limit))] + return { + r["id"]: r["name"] + for r in self.store.conn.execute(sql2, canonical_params).fetchall() + } def _entity_map(self, flt: SearchFilter, *, limit: int = 2048) -> dict[str, str]: """Compatibility view of scoped entities without restoring unbounded recall scans. @@ -1338,6 +1461,19 @@ def _planned_filter( return replace(flt, mtypes=ordered) +def _finite_arm_value(value: object) -> Optional[float]: + try: + score = float(value) + except (TypeError, ValueError, OverflowError): + return None + return score if math.isfinite(score) else None + + +def _finite_arm_score(value: object) -> float: + score = _finite_arm_value(value) + return score if score is not None else 0.0 + + def _fuse_query_runs( query_runs: list[dict[str, Any]], recs: dict[str, MemoryRecord], @@ -1359,19 +1495,31 @@ def _fuse_query_runs( for category in ("raw", "normalized", "adjusted") } rrf: dict[str, float] = {} - for run in query_runs: + for run in query_runs or []: item = run["query"] config = run["config"] priority_weight = 1.0 / max(1, int(item.priority)) for source_name, output_name in names.items(): - raw = {mid: score for mid, score in run[source_name].items() if mid in recs} + raw = {} + for mid, number in _finite_arm_items(run.get(source_name)): + if mid not in recs: + continue + raw[mid] = number normalized = scoring.normalize(raw) - scale = getattr(config, f"{output_name}_scale") - bonus = getattr(config, f"{output_name}_presence_bonus", 0.0) + scale = max( + 0.0, + _finite_arm_score(getattr(config, f"{output_name}_scale", 0.0)), + ) + bonus = max( + 0.0, + _finite_arm_score( + getattr(config, f"{output_name}_presence_bonus", 0.0) + ), + ) for mid, value in raw.items(): state["raw"][output_name][mid] = max( state["raw"][output_name].get(mid, float("-inf")), - float(value), + value, ) state["normalized"][output_name][mid] = max( state["normalized"][output_name].get(mid, 0.0), @@ -1536,7 +1684,8 @@ def _graph_traversal_details(query_runs: list[dict[str, Any]]) -> list[dict[str, if not isinstance(plan, GraphTraversalPlan): continue candidates = sorted( - run["graph"].items(), key=lambda item: (-item[1], item[0]) + _finite_arm_items(run.get("graph")), + key=lambda item: (-item[1], str(item[0])), )[:50] details.append({ "query": run["query"].text, @@ -1571,6 +1720,89 @@ def _source_safety_metadata(record: MemoryRecord) -> dict: return out +def _consolidated_source(record: MemoryRecord) -> bool: + """Whether a candidate is a consolidated digest/profile (provenance-based). + + Reads the projected provenance field first, then falls back to the same marker + inside ``metadata`` for legacy/synced rows that predate the dedicated column + (the write path copies, not pops, so both views agree on current rows). + """ + provenance = record.provenance if isinstance(record.provenance, dict) else {} + source = str(provenance.get("source") or "").strip().casefold() + if not source: + metadata = record.metadata if isinstance(record.metadata, dict) else {} + nested = metadata.get("provenance") + nested = nested if isinstance(nested, dict) else {} + source = str(nested.get("source") or "").strip().casefold() + return source in CONSOLIDATION_SOURCES + + +def _consolidation_evidence( + record: MemoryRecord, *, store=None, flt: Optional[SearchFilter] = None, +) -> list[str]: + """Source memory ids a consolidated digest/profile summarizes (citable evidence). + + Returns the union of the persisted ``consolidates``/``profiles`` memory links and + any equivalent id lists in the record's provenance/metadata. When a caller + supplies the active filter, every endpoint is reloaded and checked against that + filter before its id is exposed; this prevents a cross-repository link or forged + provenance list from widening a recall response. This surfaces the digest's + sources as evidence ids for citation without duplicating their bodies; ordinary + memories have no such links and yield ``[]``. + """ + evidence: list[str] = [] + seen: set[str] = set() + + def append_visible(value: object) -> None: + memory_id = str(value or "").strip() + if not memory_id or memory_id in seen: + return + if store is not None and flt is not None: + try: + source = store.get_memory(memory_id) + except Exception: + return + if source is None or not memory_matches_filter(source, flt): + return + seen.add(memory_id) + evidence.append(memory_id) + + metadata = record.metadata if isinstance(record.metadata, dict) else {} + provenance = record.provenance if isinstance(record.provenance, dict) else {} + nested = metadata.get("provenance") + nested = nested if isinstance(nested, dict) else {} + for container in (provenance, nested): + for key in ("consolidates", "profiles"): + values = container.get(key) + if isinstance(values, str): + values = [values] + if isinstance(values, (list, tuple, set)): + for value in values: + append_visible(value) + if record.id and store is not None and hasattr(store, "get_links"): + try: + try: + links = store.get_links(record.id, flt=flt) + except TypeError: + # Keep compatibility with older store adapters that do not yet + # accept the temporal filter keyword; endpoint scope validation + # below still applies when a filter is active. + links = store.get_links(record.id) + for link in links: + relation = str(link.get("relation") or "") + if relation not in ("consolidates", "profiles"): + continue + other = link.get("b") if link.get("a") == record.id else link.get("a") + append_visible(other) + except Exception: + # Link lookup is best-effort evidence enrichment, never a recall failure. + pass + return evidence + + + + + def _absolute_retrieval_support( query: str, content: str, @@ -1585,7 +1817,10 @@ def _absolute_retrieval_support( outside the vector arm's top-k. Unlike fused rank, neither component is min-max normalised against the other candidates in this response. """ - raw_semantic = float(semantic_cosine) + try: + raw_semantic = float(semantic_cosine) + except (TypeError, ValueError, OverflowError): + raw_semantic = 0.0 semantic = max(0.0, min(1.0, raw_semantic)) if math.isfinite(raw_semantic) else 0.0 # Titles improve candidate discovery, but are metadata rather than answer-bearing # evidence. Keeping them out of the absolute gate aligns adaptive routing with @@ -1599,10 +1834,28 @@ def _entity_pattern(name: str) -> re.Pattern[str]: return re.compile(r"(? list[tuple[object, float]]: + if not isinstance(arm, dict): + return [] + return [ + (memory_id, score) + for memory_id, raw_score in arm.items() + if (score := _finite_arm_value(raw_score)) is not None + ] + + def _ranked(arm: dict[str, float], recs: dict) -> list[str]: # Tie-break on id: RRF depends on rank position, so equal arm scores must not order - # differently between runs (they feed the final score). - return [i for i, _ in sorted(arm.items(), key=lambda x: (-x[1], x[0])) if i in recs] + # differently between runs (they feed the final score). Adapters can return + # malformed scores; those are absent evidence, not zero-scored memories. + return [ + memory_id + for memory_id, _ in sorted( + _finite_arm_items(arm), + key=lambda item: (-item[1], str(item[0])), + ) + if memory_id in recs + ] def _call_temporal_store( diff --git a/engraphis/core/resolve.py b/engraphis/core/resolve.py index 6994050b..26fa759c 100644 --- a/engraphis/core/resolve.py +++ b/engraphis/core/resolve.py @@ -32,10 +32,23 @@ # Token Jaccard: at/above this (but below DUP) it's the same subject with new content. SUBJECT_TOKEN_JACCARD = 0.40 # Supersession without an explicit claim key is intentionally stricter than a -# generic "related" judgment. Both independent signals must agree before a -# write hides a currently-live fact from ordinary recall. STRONG_SUBJECT_TOKEN_JACCARD = 0.55 STRONG_JOINT_EMBED_SIM = 0.45 +# Equal or near-equal lexical evidence from multiple live memories is not enough +# to retire one of them. The hash-vector score is only a discovery/joint signal +# (see resolve()), so a margin keeps an ambiguous write from becoming a +# supersession merely because of candidate ordering. +AMBIGUITY_EPSILON = 1e-9 +AMBIGUITY_MARGIN = 0.05 + +# Relation persisted on ``mem_links`` when the deterministic detector finds a genuine +# high-severity contradiction that the resolver cannot safely supersede (no shared +# claim key and not enough joint lexical/semantic evidence). ``conflicts_with`` is a +# free-form relation label on an already-bi-temporal table: ``mem_links.relation`` is +# TEXT and every read/write path treats it as opaque, so no schema change is needed. +# The graph layer inference in ``core/graph_layers.py`` classifies unknown labels as +# the generic SEMANTIC overlay, which is the correct conservative default. +CONFLICT_RELATION = "conflicts_with" def _normalise_claim_text(value: str) -> str: @@ -84,9 +97,11 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]], *, already scoped to the same workspace/repo/scope/mtype as the candidate (conflict resolution must not silently cross a scope boundary — promotion is explicit, §5.1) and filtered to currently-visible memories. Order doesn't matter; every neighbor - above ``RELATED_SIM_FLOOR`` is checked and the best token-overlap match wins. Cosine - is candidate-discovery and *joint* evidence only: the dependency-free hashing - embedder is lexical, not a sound paraphrase/contradiction classifier. + above ``RELATED_SIM_FLOOR`` is checked, and the best token-overlap match wins unless + another live memory is a near-equal strong match, in which case resolution relates + without superseding either one. Cosine is candidate-discovery and *joint* evidence + only: the dependency-free hashing embedder is lexical, not a sound + paraphrase/contradiction classifier. """ cand_tokens = tokenize(candidate_text) candidate_subject = str(subject_key or "").strip() @@ -109,12 +124,26 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]], *, fallback_neighbors.append((sim, rec)) considered = exact_claim_neighbors or fallback_neighbors - best: Optional[tuple[float, MemoryRecord, float]] = None # (overlap, rec, sim) + scored: list[tuple[float, MemoryRecord, float]] = [] for sim, rec in considered: overlap = jaccard(cand_tokens, tokenize(f"{rec.title} {rec.content}")) - if best is None or overlap > best[0]: - best = (overlap, rec, sim) - + scored.append((overlap, rec, sim)) + # Retrieval order is not part of the resolution contract. Stable tie-breaking + # makes repeated writes idempotent even when a vector backend returns equal-score + # neighbors in a different order. When a claim has multiple visible versions, + # prefer the latest world-time version before falling back to its id so a + # supersession follows the temporal chain rather than arbitrary retrieval order. + scored.sort( + key=lambda item: ( + -item[0], + -item[2], + -(item[1].valid_from if item[1].valid_from is not None else float("-inf")), + str(item[1].id), + ) + ) + best: Optional[tuple[float, MemoryRecord, float]] = ( + scored[0] if scored else None + ) if best is None: return Resolution(ResolutionOp.ADD, reason="no related memory in scope") @@ -168,6 +197,19 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]], *, # neighbor rather than a contradiction, so it does not change either fact. if (not candidate_subject and overlap >= STRONG_SUBJECT_TOKEN_JACCARD and sim >= STRONG_JOINT_EMBED_SIM): + ambiguous = [ + item for item in scored[1:] + if item[0] >= STRONG_SUBJECT_TOKEN_JACCARD + and item[2] >= STRONG_JOINT_EMBED_SIM + and overlap - item[0] <= AMBIGUITY_MARGIN + AMBIGUITY_EPSILON + ] + if ambiguous: + ids = ", ".join(sorted({rec.id, *(item[1].id for item in ambiguous)})) + return Resolution( + ResolutionOp.RELATE, + reason=f"ambiguous strong match among {ids}; no memory superseded " + f"(best overlap={overlap:.2f}, similarity={sim:.2f})", + ) return Resolution(ResolutionOp.INVALIDATE, target_id=rec.id, reason=f"supersedes {rec.id} (strong joint evidence: " f"token overlap={overlap:.2f}, similarity={sim:.2f})") diff --git a/engraphis/core/schema.py b/engraphis/core/schema.py index 24c9e84a..9cee7831 100644 --- a/engraphis/core/schema.py +++ b/engraphis/core/schema.py @@ -8,7 +8,7 @@ """ from __future__ import annotations -SCHEMA_VERSION = 7 +SCHEMA_VERSION = 9 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -69,6 +69,7 @@ importance REAL DEFAULT 0.0, surprise REAL DEFAULT 1.0, stability REAL DEFAULT 1.0, + confidence REAL NOT NULL DEFAULT 1.0, -- 0..1 model/extraction confidence (scoring multiplier) access_count INTEGER DEFAULT 0, last_access REAL, valid_from REAL, -- world-time validity @@ -81,6 +82,8 @@ pinned INTEGER DEFAULT 0, sensitivity TEXT DEFAULT 'normal', provenance TEXT DEFAULT '{}', + pinned_at REAL, -- system-time when a pin last became effective + unpinned_at REAL, -- system-time when an unpin became effective sort_order REAL -- manual drag-to-reorder position (dashboard); NULL = unordered ); CREATE INDEX IF NOT EXISTS idx_mem_scope ON memories(workspace_id, repo_id, scope, mtype); @@ -481,6 +484,43 @@ value TEXT, updated_at REAL ); +-- ── Maintenance cursors (local bounded-sweep progress) ─────────────────────── +-- Consolidation scans are intentionally bounded. Persist their keyset cursor so +-- recurring sweeps rotate past rows that are not currently clusterable instead of +-- restarting at the same oldest window forever. This is local bookkeeping and is +-- never included in sync bundles. +CREATE TABLE IF NOT EXISTS maintenance_cursors ( + workspace_id TEXT NOT NULL, + repo_id TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL, + cursor TEXT NOT NULL DEFAULT '', + updated_at REAL NOT NULL, + PRIMARY KEY (workspace_id, repo_id, name) +); +CREATE INDEX IF NOT EXISTS idx_maintenance_cursors_workspace + ON maintenance_cursors(workspace_id, repo_id, name); + +-- Durable per-memory tombstones (sync deletion markers, v9). +-- +-- ``secure_erase`` hard-deletes the memory row and all local derivatives, but the +-- deletion must still PROPAGATE: without a tombstone, a peer that still holds the +-- row keeps pushing it and the next apply re-adds it (the erased memory +-- resurrects). This table records, with no user content, that a memory id is dead. +-- +-- It is sync state in the same sense as ``sync_state``: durable, additive, and +-- shared with no other subsystem. It lives in its own table (not the KV) so the +-- sync layer can read/merge it in bulk and cap it like the other bundle payloads. +CREATE TABLE IF NOT EXISTS memory_tombstones ( + memory_id TEXT PRIMARY KEY, + deleted_at REAL NOT NULL, -- system-time when the erasure happened + device_id TEXT NOT NULL, -- origin device (sync attribution only) + workspace_id TEXT, -- sync scope (may be NULL for legacy rows) + repo_id TEXT, -- repo scope; NULL means workspace scope/legacy + created_at REAL NOT NULL +); +-- Sync exports scope tombstones by workspace; keep that read bounded as erasures grow. +CREATE INDEX IF NOT EXISTS idx_memory_tombstones_workspace + ON memory_tombstones(workspace_id, repo_id, memory_id); """ # FTS5 if available, else a plain fallback table with the same columns. diff --git a/engraphis/core/scoring.py b/engraphis/core/scoring.py index 099ceb1d..ab0c544d 100644 --- a/engraphis/core/scoring.py +++ b/engraphis/core/scoring.py @@ -53,6 +53,25 @@ class Weights: def weights_for(mtype: MemoryType) -> Weights: return DEFAULT_WEIGHTS.get(mtype, Weights()) +def _finite_number(value: object, default: float = 0.0) -> float: + try: + number = float(value) + except (TypeError, ValueError, OverflowError): + return default + return number if math.isfinite(number) else default + + +def _bounded(value: object, *, default: float = 0.0, + lower: float = 0.0, upper: float = 1.0) -> float: + number = _finite_number(value, default) + return max(lower, min(upper, number)) + + +def _confidence(value: object) -> float: + if value is None: + return 1.0 + return _bounded(value, default=1.0) + def retention(stability: float, last_access: Optional[float], now: float) -> float: """Ebbinghaus R(t) = exp(-Δt_days / S). @@ -65,10 +84,17 @@ def retention(stability: float, last_access: Optional[float], now: float) -> flo """ try: supplied = float(stability) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): supplied = DEFAULT_STABILITY_DAYS S = supplied if math.isfinite(supplied) and supplied > 0 else DEFAULT_STABILITY_DAYS - dt_days = max((now - (last_access if last_access is not None else now)) / 86400.0, 0.0) + current = _finite_number(now, float("nan")) + if not math.isfinite(current): + return 0.0 + accessed = ( + current if last_access is None + else _finite_number(last_access, current) + ) + dt_days = max((current - accessed) / 86400.0, 0.0) return math.exp(-dt_days / S) @@ -76,8 +102,15 @@ def recency(t_ref: Optional[float], now: float, tau_days: float = 30.0) -> float """Exponential recency on world-time, for tie-breaking and temporal queries.""" if t_ref is None: return 0.0 - dt_days = max((now - t_ref) / 86400.0, 0.0) - return math.exp(-dt_days / max(tau_days, 1e-6)) + current = _finite_number(now, float("nan")) + reference = _finite_number(t_ref, float("nan")) + if not math.isfinite(current) or not math.isfinite(reference): + return 0.0 + tau = _finite_number(tau_days, 30.0) + if tau <= 0: + tau = 1e-6 + dt_days = max((current - reference) / 86400.0, 0.0) + return math.exp(-dt_days / tau) def staleness_penalty(valid_to: Optional[float], now: float, @@ -85,31 +118,81 @@ def staleness_penalty(valid_to: Optional[float], now: float, """1.0 once a fact is past its validity; ramps up in the ``ramp_days`` before.""" if valid_to is None: return 0.0 - if now >= valid_to: + current = _finite_number(now, float("nan")) + expiry = _finite_number(valid_to, float("nan")) + if not math.isfinite(current) or not math.isfinite(expiry): + return 0.0 + if current >= expiry: return 1.0 - days_left = (valid_to - now) / 86400.0 - if days_left >= ramp_days: + ramp = _finite_number(ramp_days, 7.0) + if ramp <= 0: + return 0.0 + days_left = (expiry - current) / 86400.0 + if days_left >= ramp: return 0.0 - return 1.0 - (days_left / ramp_days) + return max(0.0, min(1.0, 1.0 - (days_left / ramp))) def normalize(scores: dict[str, float]) -> dict[str, float]: - """Min-max normalize to [0, 1]; flat inputs map to 1.0.""" + """Min-max normalize to [0, 1]; flat inputs map to 1.0. + + Retrieval adapters are external inputs in practice. Non-finite values are + treated as missing evidence instead of allowing NaN/Infinity to poison the + fused ranking or its deterministic sort. When every value is non-finite the + arm contributes no evidence at all (empty result) rather than granting every + key the maximum score. + """ if not scores: return {} - vals = list(scores.values()) - lo, hi = min(vals), max(vals) - if hi - lo < 1e-12: - return {k: 1.0 for k in scores} - return {k: (v - lo) / (hi - lo) for k, v in scores.items()} + finite: dict[str, float] = {} + for key, value in scores.items(): + try: + number = float(value) + except (TypeError, ValueError, OverflowError): + continue # unparseable evidence is missing evidence + if math.isfinite(number): + finite[key] = number + if not finite: + return {} + lo, hi = min(finite.values()), max(finite.values()) + span = hi - lo + if not math.isfinite(span): + scale = max(abs(lo), abs(hi)) + if not math.isfinite(scale) or scale == 0.0: + return {key: 1.0 for key in finite} + scaled_lo = lo / scale + scaled_hi = hi / scale + span = scaled_hi - scaled_lo + if not math.isfinite(span) or span < 1e-12: + return {key: 1.0 for key in finite} + return { + key: max(0.0, min(1.0, (value / scale - scaled_lo) / span)) + for key, value in finite.items() + } + if span < 1e-12: + return {key: 1.0 for key in finite} + return { + key: max(0.0, min(1.0, (value - lo) / span)) + for key, value in finite.items() + } def reciprocal_rank_fusion(rankings: list[list[str]], k: int = 60) -> dict[str, float]: """RRF across arms — rewards items ranked highly by multiple retrieval arms.""" + try: + base = int(k) + except (TypeError, ValueError): + base = 60 + base = max(1, base) fused: dict[str, float] = {} - for ranking in rankings: - for rank, mid in enumerate(ranking): - fused[mid] = fused.get(mid, 0.0) + 1.0 / (k + rank + 1) + for ranking in rankings or []: + seen: set[str] = set() + for mid in ranking or []: + if not isinstance(mid, str) or not mid or mid in seen: + continue + seen.add(mid) + rank = len(seen) - 1 + fused[mid] = fused.get(mid, 0.0) + 1.0 / (base + rank + 1) return fused @@ -129,8 +212,16 @@ def score_memory(rec: MemoryRecord, *, now: float, weights: Weights, w = weights r = retention(rec.stability, rec.last_access, now) x = staleness_penalty(rec.valid_to, now) - return (w.r * r + w.s * semantic + w.l * lexical + w.g * graph - + w.i * (rec.importance or 0.0) - w.x * x) + confidence = _confidence(getattr(rec, "confidence", 1.0)) + importance = _bounded(getattr(rec, "importance", 0.0)) + return ( + _finite_number(getattr(w, "r", 0.0)) * r + + _finite_number(getattr(w, "s", 0.0)) * _finite_number(semantic) + + _finite_number(getattr(w, "l", 0.0)) * _finite_number(lexical) + + _finite_number(getattr(w, "g", 0.0)) * _finite_number(graph) + + _finite_number(getattr(w, "i", 0.0)) * importance * confidence + - _finite_number(getattr(w, "x", 0.0)) * x + ) def score_proactive(rec: MemoryRecord, *, now: float, weights: Optional[Weights] = None, @@ -142,14 +233,17 @@ def score_proactive(rec: MemoryRecord, *, now: float, weights: Optional[Weights] for call compatibility and deliberately no longer alters scoring. """ w = weights or weights_for(rec.mtype) - importance = min(max(float(rec.importance or 0.0), 0.0), 1.0) + importance = _bounded(getattr(rec, "importance", 0.0)) del importance_retention_floor r = retention(rec.stability, rec.last_access, now) rec_ref = rec.valid_from if rec.valid_from is not None else rec.ingested_at - importance_signal = importance * r + confidence = _confidence(getattr(rec, "confidence", 1.0)) + importance_signal = importance * r * confidence return ( - w.i * importance_signal - + w.c * recency(rec_ref, now) - + w.r * r - - w.x * staleness_penalty(rec.valid_to, now) + _finite_number(getattr(w, "i", 0.0)) * importance_signal + + _finite_number(getattr(w, "c", 0.0)) * recency(rec_ref, now) + + _finite_number(getattr(w, "r", 0.0)) * r + - _finite_number(getattr(w, "x", 0.0)) * staleness_penalty( + rec.valid_to, now + ) ) diff --git a/engraphis/core/secrets.py b/engraphis/core/secrets.py index d0fd85fa..309f64e6 100644 --- a/engraphis/core/secrets.py +++ b/engraphis/core/secrets.py @@ -73,6 +73,12 @@ def __init__(self, field: str, kind: str) -> None: """, ) _REDACTION = re.compile(r"^?$", re.I) +_PEM_BLOCK = re.compile( + r"-----BEGIN(?: [A-Z0-9]+)? PRIVATE KEY-----[\s\S]*?" + r"-----END(?: [A-Z0-9]+)? PRIVATE KEY-----", + re.I, +) +_REDACTED = "" def _text(value: Any) -> str: @@ -139,6 +145,26 @@ def secret_kind(value: Any) -> str | None: return None +def redact_secrets(text: str) -> str: + """Return *text* with credential-shaped values replaced by a safe marker. + + This is intentionally separate from :func:`reject_secrets`: product writes + must still fail closed. It is for callers that explicitly need a safe copy + of untrusted text, such as an evaluation corpus that must not persist an + incidental credential found in source material. The returned text is + suitable for the normal capture boundary and never includes the original + matching value. + """ + if not isinstance(text, str) or not text: + return text + safe = _PEM_BLOCK.sub(_REDACTED, text) + for _kind, pattern in _PATTERNS: + safe = pattern.sub(_REDACTED, safe) + safe = _DSN.sub(_REDACTED, safe) + safe = _ASSIGNMENT.sub(_REDACTED, safe) + return safe + + def reject_secrets(fields: Iterable[tuple[str, Any]]) -> None: """Reject the first secret found in persisted memory/event payload fields. diff --git a/engraphis/core/store.py b/engraphis/core/store.py index 9e489dd9..99e7accc 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -50,6 +50,11 @@ # Bound placeholders per ``IN (...)`` so a batched lookup stays under SQLite's # SQLITE_MAX_VARIABLE_NUMBER (999 before 3.32, 32766 after) on every build. IN_CLAUSE_CHUNK = 500 +# Keep dynamic blocking predicates well below SQLite's conservative 999-variable +# and expression-depth limits. Each token contributes two LIKE parameters. +ENTITY_BLOCK_TOKEN_CHUNK = 200 +# Do not materialize unbounded common-token buckets during migration/live writes. +ENTITY_BLOCK_BUCKET_LIMIT = 1024 def now_ts() -> float: return time.time() @@ -165,6 +170,44 @@ def normalize_entity_name(value: str) -> str: return re.sub(r"\s+", " ", text).strip() +def _entity_token_set(name: Any) -> set[str]: + """Return conservative blocking tokens for one entity spelling.""" + return { + token + for token in re.split(r"[^a-z0-9]+", str(name or "").casefold()) + if len(token) >= 2 + } + + +def _entity_compact_name(name: Any) -> str: + """Return the punctuation-preserving, whitespace-insensitive spelling.""" + return re.sub(r"\s+", "", normalize_entity_name(str(name or ""))) + + +def _entity_punctuation_signature(name: Any) -> str: + """Return meaningful punctuation so token blocking cannot cross its boundary.""" + normalized = normalize_entity_name(str(name or "")) + return "".join( + character for character in normalized + if not character.isalnum() and not character.isspace() + ) + + +def _entity_overlap(left: Any, right: Any) -> Optional[float]: + """Return the token-blocking score, or ``None`` when no safe match exists.""" + left_compact = _entity_compact_name(left) + right_compact = _entity_compact_name(right) + if left_compact and left_compact == right_compact: + return 1.0 + if _entity_punctuation_signature(left) != _entity_punctuation_signature(right): + return None + left_tokens = _entity_token_set(left) + right_tokens = _entity_token_set(right) + if not left_tokens or not right_tokens: + return None + return len(left_tokens & right_tokens) / max(len(left_tokens), len(right_tokens)) + + _SUPPORT_CONFIDENCE = { "manual": 1.0, "schema": 1.0, @@ -755,6 +798,12 @@ def __init__(self, path: str = ":memory:", *, and "fts5" in str(row["sql"] or "").casefold() ) else: + # Keep deleted pages scrubbed even when an emergency erase cannot run a + # final VACUUM because another connection has the database busy. The + # per-erase helper sets this too for legacy connections and backups; + # setting it at writable-store startup makes the protection durable for + # every normal v2 connection without changing the schema or data model. + self.conn.execute("PRAGMA secure_delete=ON") self.conn.execute("PRAGMA synchronous=NORMAL") self.init_schema() # journal_mode is persistent state, so set it only after a required backup @@ -1060,6 +1109,9 @@ def _apply_schema(self, previous_version: int) -> None: # explicit, idempotent ALTER TABLE here (SQLite has no "ADD COLUMN IF NOT EXISTS"). for stmt in ( "ALTER TABLE memories ADD COLUMN sort_order REAL", + "ALTER TABLE memories ADD COLUMN pinned_at REAL", + "ALTER TABLE memories ADD COLUMN unpinned_at REAL", + "ALTER TABLE memories ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0", "ALTER TABLE memories ADD COLUMN subject_key TEXT DEFAULT ''", "ALTER TABLE memories ADD COLUMN claim_kind TEXT DEFAULT ''", "ALTER TABLE memories ADD COLUMN valid_to_recorded_at REAL", @@ -1094,11 +1146,26 @@ def _apply_schema(self, previous_version: int) -> None: "ALTER TABLE operation_receipts ADD COLUMN sequence INTEGER", "ALTER TABLE jobs ADD COLUMN runner_id TEXT", "ALTER TABLE jobs ADD COLUMN heartbeat_at REAL", + "ALTER TABLE memory_tombstones ADD COLUMN repo_id TEXT", ): try: self.conn.execute(stmt) except sqlite3.OperationalError: pass # column already exists + tombstone_index_columns = [ + str(row["name"]) + for row in self.conn.execute( + "PRAGMA index_info('idx_memory_tombstones_workspace')" + ).fetchall() + ] + if tombstone_index_columns != ["workspace_id", "repo_id", "memory_id"]: + self.conn.execute( + "DROP INDEX IF EXISTS idx_memory_tombstones_workspace" + ) + self.conn.execute( + "CREATE INDEX idx_memory_tombstones_workspace " + "ON memory_tombstones(workspace_id, repo_id, memory_id)" + ) # This cannot live in SCHEMA_SQL: CREATE TABLE IF NOT EXISTS leaves an # early-v5 ``mem_links`` table untouched, so the index would reference # temporal columns before the additive ALTERs above install them. @@ -1153,6 +1220,21 @@ def _apply_schema(self, previous_version: int) -> None: "VALUES (?,?,?)", ("deterministic_hashing", "v1_legacy", now_ts()), ) + if previous_version < 8: + # v7 memories predate first-class confidence. ``confidence`` is a + # scoring multiplier with a 1.0 default, so existing rows need no + # backfill — the NOT NULL DEFAULT 1.0 column already covers them + # (the additive ALTER above is one-shot on reopens). + # v7 pin state has no clock. Synthesize earliest-wins markers so a + # legacy pinned row still participates in the new pin lattice: a pinned + # row without ``pinned_at`` is treated as pinned since the epoch (it + # can never be beaten by a peer's unpin, which matches the old + # OR-semantics), and a legacy unpinned row carries no marker at all + # (a peer's pin simply applies). Rows with real clocks are untouched. + self.conn.execute( + "UPDATE memories SET pinned_at=0.0 " + "WHERE pinned=1 AND pinned_at IS NULL" + ) # Classify pre-v3 edges. Existing rows defaulted to semantic during ALTER TABLE; # infer their more specific logical layer from the relationship label. if previous_version < 3: @@ -1170,9 +1252,15 @@ def _apply_schema(self, previous_version: int) -> None: (inferred, row["rowid"]), ) # v4 makes canonical identity and edge evidence explicit and indexed. Run the - # backfills before creating representative-only uniqueness indexes so exact - # normalized aliases can safely converge onto one deterministic canonical id. - self._backfill_entity_canonicalization() + # backfill only when the database crosses the migration that introduced the + # canonical fields. Running the all-pairs token pass on every fresh/opened + # database turns startup into an O(n²) scan of the entire entity table. + if previous_version < 4: + self._backfill_entity_canonicalization() + elif previous_version < 9: + # v8 databases may have canonical fields but never received the token + # overlap pass; v9 is the one-time repair for that gap. + self._backfill_entity_canonicalization() self._execute_script_transactional( "CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_workspace_canonical " "ON entities(workspace_id, normalized_name, etype) " @@ -1441,12 +1529,110 @@ def backfill_memory_entities_for_memory(self, memory_id: str) -> None: """Materialize the evidence incidence for one freshly written memory.""" self._backfill_memory_entities_v5(memory_id) + def _entity_blocking_candidates(self, *, entity_id: Optional[str], + workspace_id: Optional[str], + etype: Optional[str], name: Any) -> list[sqlite3.Row]: + """Select lexical peers without making one unbounded SQL expression. + Ordinary token blocks return every matching peer; unusually broad blocks are + deliberately discarded rather than materialized. The compact-alias query always + runs. The Python score below then applies the exact compact/Jaccard rule. + Matching both normalized_name and the legacy name column lets a partially + upgraded database participate before its next migration completes. + """ + tokens = sorted(_entity_token_set(name)) + if not tokens: + return [] + base_sql = ( + "SELECT id, workspace_id, repo_id, name, etype, canonical_id, " + "normalized_name, canonical_method, canonical_confidence " + "FROM entities WHERE workspace_id IS ? AND etype IS ? AND (" + ) + found: dict[str, sqlite3.Row] = {} + + def collect(clauses: list[str], patterns: list[str], *, + guard_broad: bool) -> None: + params: list[Any] = [workspace_id, etype, *patterns] + sql = base_sql + " OR ".join(clauses) + ")" + if entity_id is not None: + sql += " AND id<>?" + params.append(entity_id) + if guard_broad: + sql += " LIMIT ?" + params.append(ENTITY_BLOCK_BUCKET_LIMIT + 1) + rows = self.conn.execute(sql, params).fetchall() + if guard_broad and len(rows) > ENTITY_BLOCK_BUCKET_LIMIT: + # A common token is not useful as a blocking key. Do not retain + # an arbitrarily large bucket; the exact compact query still runs. + return + for row in rows: + found[str(row["id"])] = row + + for start in range(0, len(tokens), ENTITY_BLOCK_TOKEN_CHUNK): + clauses: list[str] = [] + patterns: list[str] = [] + for token in tokens[start:start + ENTITY_BLOCK_TOKEN_CHUNK]: + pattern = "%" + _escape_like(token) + "%" + clauses.append( + "(normalized_name LIKE ? ESCAPE '\\' OR lower(name) LIKE ? ESCAPE '\\')" + ) + patterns.extend((pattern, pattern)) + collect(clauses, patterns, guard_broad=True) + + # Whitespace-separated aliases such as OpenAI/Open AI have no shared token, + # but their compact spellings are still an exact canonical match. + compact = _entity_compact_name(name) + if compact: + compact_pattern = "%" + _escape_like(compact) + "%" + collect( + [ + "(replace(lower(normalized_name), ' ', '') LIKE ? ESCAPE '\\' " + "OR replace(lower(name), ' ', '') LIKE ? ESCAPE '\\')" + ], + [compact_pattern, compact_pattern], guard_broad=False, + ) + return [found[key] for key in sorted(found)] + def _backfill_entity_canonicalization(self) -> None: rows = [dict(row) for row in self.conn.execute( "SELECT id, workspace_id, name, etype, canonical_id, normalized_name, " "canonical_method, canonical_confidence FROM entities " "ORDER BY workspace_id, etype, id" ).fetchall()] + # Close canonical chains to their root FIRST. A legacy database can carry a + # two-hop chain (A→B, B→C) when an earlier pass merged B into C after A had + # already pointed at B; the group pass below keeps "any existing canonical + # wins", so A would otherwise dangle at B while B points at C. Resolve every + # id to its transitive root (an id whose canonical is itself, or a + # non-existent id — caller-provided roots are authoritative) and persist one + # hop, so the group pass and the singleton-reset logic below see roots only. + # Deterministic and idempotent. + root_of: dict[str, str] = {row["id"]: row["id"] for row in rows} + for row in rows: + cid = str(row.get("canonical_id") or "") + if cid: + root_of[row["id"]] = cid + for mid in root_of: + seen: set[str] = set() + cursor = root_of[mid] + while cursor in root_of and root_of[cursor] != cursor: + if cursor in seen: # cycle safety (should not happen) + break + seen.add(cursor) + cursor = root_of[cursor] + root_of[mid] = cursor + for row in rows: + root = root_of.get(row["id"]) + cid = str(row.get("canonical_id") or "") + if cid and root and root != cid: + self.conn.execute( + "UPDATE entities SET canonical_id=? WHERE id=?", + (root, row["id"]), + ) + rows = [dict(row) for row in self.conn.execute( + "SELECT id, workspace_id, name, etype, canonical_id, normalized_name, " + "canonical_method, canonical_confidence FROM entities " + "ORDER BY workspace_id, etype, id" + ).fetchall()] groups: dict[tuple[str, str, str], list[dict]] = {} for row in rows: normalized = normalize_entity_name(row.get("name") or "") @@ -1490,6 +1676,58 @@ def _backfill_entity_canonicalization(self) -> None: (row["_normalized"], canonical_id, method, confidence, row["id"]), ) + # Token-overlap blocking is deliberately query-backed rather than an in-memory + # all-pairs pass. It is still a one-time migration transform, but a workspace + # with many unrelated entities should not turn an upgrade into quadratic work. + rows = [dict(row) for row in self.conn.execute( + "SELECT id, workspace_id, repo_id, name, etype, canonical_id, normalized_name, " + "canonical_method, canonical_confidence FROM entities " + "ORDER BY workspace_id, etype, id" + ).fetchall()] + row_by_id = {str(row["id"]): row for row in rows} + seen_pairs: set[tuple[str, str]] = set() + for row in rows: + if not _entity_token_set(row.get("name")): + continue + candidates = self._entity_blocking_candidates( + entity_id=row["id"], workspace_id=row.get("workspace_id"), + etype=row.get("etype"), name=row.get("name"), + ) + for candidate in candidates: + other = dict(candidate) + pair = tuple(sorted((str(row["id"]), str(other["id"])))) + if pair in seen_pairs: + continue + seen_pairs.add(pair) + overlap = _entity_overlap(row.get("name"), other.get("name")) + if overlap is None or overlap < 0.6: + continue + # Existing canonical ids win when either side has one; otherwise the + # lexicographically oldest typed id is deterministic. + other_state = row_by_id.get(str(other["id"])) + if other_state is not None: + other["canonical_id"] = other_state.get("canonical_id") + other["canonical_method"] = other_state.get("canonical_method") + existing = sorted({ + str(row.get("canonical_id") or ""), + str(other.get("canonical_id") or ""), + }) + existing = [value for value in existing if value] + canonical = existing[0] if existing else min(pair) + for member in (row, other): + state = row_by_id.get(str(member["id"]), member) + if state.get("canonical_id") != canonical or \ + state.get("canonical_method") != "token_overlap": + self.conn.execute( + "UPDATE entities SET canonical_id=?, canonical_method=? " + "WHERE id=?", + (canonical, "token_overlap", member["id"]), + ) + state["canonical_id"] = canonical + state["canonical_method"] = "token_overlap" + member["canonical_id"] = canonical + member["canonical_method"] = "token_overlap" + def _backfill_edge_supports(self) -> None: rows = self.conn.execute( "SELECT id, relation, valid_from, valid_to, ingested_at, expired_at, provenance " @@ -1958,17 +2196,28 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, f"existing provenance={existing['provenance']}, " f"incoming provenance={_dumps(rec.provenance)}", commit=False) ts = now_ts() + # A "closed history" record may legitimately carry only a past ``valid_to`` with + # ``valid_from`` defaulting to ingest time (the fixture/backfill convention). The + # empty-interval invariant therefore applies only when the caller explicitly + # supplied BOTH endpoints — a caller-authored inversion is always a bug, whereas + # a defaulted ``valid_from`` with a past ``valid_to`` is an accepted closed window. + valid_from_was_explicit = rec.valid_from is not None rec.ingested_at = rec.ingested_at if rec.ingested_at is not None else ts rec.valid_from = rec.valid_from if rec.valid_from is not None else ts rec.last_access = rec.last_access if rec.last_access is not None else ts + if (valid_from_was_explicit and rec.valid_to is not None + and rec.valid_to < rec.valid_from): + raise ValueError( + "valid_to cannot predate valid_from; the validity interval would be empty" + ) self.conn.execute( """INSERT INTO memories (id, workspace_id, repo_id, session_id, scope, mtype, title, content, summary, keywords, metadata, importance, surprise, stability, access_count, last_access, valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, subject_key, claim_kind, - pinned, sensitivity, provenance) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + pinned, sensitivity, provenance, confidence, pinned_at, unpinned_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, repo_id=excluded.repo_id, session_id=excluded.session_id, scope=excluded.scope, mtype=excluded.mtype, @@ -1982,7 +2231,9 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, ingested_at=excluded.ingested_at, expired_at=excluded.expired_at, subject_key=excluded.subject_key, claim_kind=excluded.claim_kind, pinned=excluded.pinned, - sensitivity=excluded.sensitivity, provenance=excluded.provenance""", + sensitivity=excluded.sensitivity, provenance=excluded.provenance, + confidence=excluded.confidence, + pinned_at=excluded.pinned_at, unpinned_at=excluded.unpinned_at""", (rec.id, rec.workspace_id, rec.repo_id, rec.session_id, _enum(rec.scope), _enum(rec.mtype), rec.title, rec.content, rec.summary, _dumps(rec.keywords), _dumps(rec.metadata), rec.importance, rec.surprise, @@ -1990,13 +2241,24 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, rec.valid_to_recorded_at, rec.ingested_at, rec.expired_at, rec.subject_key, rec.claim_kind, int(rec.pinned), rec.sensitivity, - _dumps(rec.provenance)), + _dumps(rec.provenance), rec.confidence, + rec.pinned_at, rec.unpinned_at), ) - # full-text mirror - self._fts_upsert(rec.id, rec.title, rec.content, " ".join(rec.keywords)) - # vector mirror (L2-normalized for cosine-as-dot) - if rec.embedding is not None: - self.put_vector(rec.id, rec.embedding, model=str(rec.metadata.get("embed_model", ""))) + try: + # Keep the row, FTS mirror, and vector mirror atomic for the normal + # single-write path. Once the main INSERT succeeds, a mirror failure + # otherwise leaves this connection pinned in a partial transaction and + # lets a later commit publish an unindexed memory. + self._fts_upsert(rec.id, rec.title, rec.content, " ".join(rec.keywords)) + # vector mirror (L2-normalized for cosine-as-dot) + if rec.embedding is not None: + self.put_vector( + rec.id, rec.embedding, model=str(rec.metadata.get("embed_model", "")) + ) + except BaseException: + if commit: + self.conn.rollback() + raise # ``commit=False`` lets a bulk writer (sync's bundle apply) amortize one commit over # a batch of rows instead of paying a durability fsync per memory. The caller then # owns the transaction and MUST commit or roll back — see SyncEngine.apply_bundle. @@ -2074,6 +2336,32 @@ def count_memories(self, flt: Optional[SearchFilter] = None, row = self.conn.execute(sql, params).fetchone() return int(row["count"] if row is not None else 0) + def list_proactive_overrides(self, flt: Optional[SearchFilter] = None, + *, prompt_only: bool = False) -> list[MemoryRecord]: + """Return pinned/``proactive=always`` rows outside the normal scan window. + + The proactive agenda intentionally bounds its ordinary scan, but explicit user + choices are not bounded by recency. Keep this query separate so a very old pin + cannot disappear behind 500 newer memories without making every proactive call + materialize the entire store. + """ + sql = "SELECT * FROM memories" + where, params = self._where(flt, include_invalid=False) + where.append("(pinned=1 OR lower(metadata) LIKE ?)") + params.append('%"proactive"%') + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY ingested_at DESC" + out: list[MemoryRecord] = [] + for row in self.conn.execute(sql, params): + rec = _row_to_record(row) + proactive = str((rec.metadata or {}).get("proactive") or "").lower() + if not rec.pinned and proactive != "always": + continue + if prompt_only and not _row_is_prompt_eligible(row["provenance"], row["metadata"]): + continue + out.append(rec) + return out + def list_live_claims(self, *, workspace_id: str, repo_id: Optional[str], session_id: Optional[str], scope: Scope, mtype: MemoryType, subject_key: str, claim_kind: str) -> list[MemoryRecord]: @@ -2169,8 +2457,32 @@ def close_validity(self, memory_id: str, *, at: Optional[float] = None, def set_pinned(self, memory_id: str, pinned: bool) -> None: """Pinned memories are exempt from automatic decay/pruning (AGENTS.md §3.2); - governance (explicit forget/correct) can still act on them.""" - self.conn.execute("UPDATE memories SET pinned=? WHERE id=?", (int(pinned), memory_id)) + governance (explicit forget/correct) can still act on them. + + Every pin-state transition stamps the system time into the row so sync can + merge the state as a latest-transition lattice instead of an OR-set: + ``pinned_at`` records the latest pin and ``unpinned_at`` the latest unpin. + A re-pin preserves the unpin marker, so peers converge on whichever + transition happened last instead of allowing a stale pin to resurrect. + """ + row = self.conn.execute( + "SELECT pinned FROM memories WHERE id=?", (memory_id,) + ).fetchone() + if row is None: + return + now = now_ts() + if pinned: + self.conn.execute( + "UPDATE memories SET pinned=1, pinned_at=? " + "WHERE id=? AND pinned=0", + (now, memory_id), + ) + else: + self.conn.execute( + "UPDATE memories SET pinned=0, unpinned_at=? " + "WHERE id=? AND pinned=1", + (now, memory_id), + ) self.conn.commit() def reinforce(self, memory_id: str, *, alpha: float = 0.3, boost: float = 0.0) -> None: @@ -2292,7 +2604,15 @@ def _erase_memory_rows(cls, conn, memory_id: str, *, actor: str = "user") -> dic """ if not cls._has_table(conn, "memories"): return {"present": False, "removed": False} - row = conn.execute("SELECT id FROM memories WHERE id=?", (memory_id,)).fetchone() + memory_columns = { + item["name"] for item in conn.execute("PRAGMA table_info(memories)").fetchall() + } + row = conn.execute( + ("SELECT id, workspace_id, repo_id FROM memories WHERE id=?" + if "workspace_id" in memory_columns + else "SELECT id FROM memories WHERE id=?"), + (memory_id,), + ).fetchone() if row is None: return {"present": False, "removed": False} @@ -2423,6 +2743,8 @@ def _erase_memory_rows(cls, conn, memory_id: str, *, actor: str = "user") -> dic return { "present": True, "removed": True, + "workspace_id": row["workspace_id"] if "workspace_id" in row.keys() else None, + "repo_id": row["repo_id"] if "repo_id" in row.keys() else None, "graph_edges_considered": len(supported_edges), "entities_considered": len(incident_entities), } @@ -2490,10 +2812,35 @@ def secure_erase_memory(self, memory_id: str, *, actor: str = "user") -> dict: and recognised local SQLite recovery backups. OS snapshots, copies, remote sync peers, and a process that already read the secret cannot be recalled or erased. """ - current = self._erase_memory_rows(self.conn, memory_id, actor=actor) - if not current["present"]: - raise KeyError(f"no memory with id '{memory_id}'") - self.conn.commit() + owns_transaction = not self.conn.transaction_owned_by_current_thread() + try: + # Mint the origin before opening the erase transaction. ``device_id`` may + # need to write sync metadata on a new database; keeping that write outside + # the destructive transaction means the deletion and terminal tombstone + # commit (or roll back) as one unit. + device_id = self.device_id() + current = self._erase_memory_rows(self.conn, memory_id, actor=actor) + if not current["present"]: + raise KeyError(f"no memory with id '{memory_id}'") + # Durable sync tombstone: the local row is hard-deleted, but the *deletion* + # must survive in sync state so a peer that still holds the row is told this + # id is dead instead of re-adding it on the next round. No content travels — + # only the id, the erasure time, and this device's id. Scope is captured from + # the erased row so an export restricted to a repo still tells that repo's + # peers the id is gone (a tombstone scoped to the workspace is never + # exported, mirroring how an erased row can no longer be scoped). + self.add_memory_tombstone( + memory_id, deleted_at=now_ts(), + device_id=device_id, + workspace_id=current.get("workspace_id"), + repo_id=current.get("repo_id"), + ) + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.commit() + except BaseException: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.rollback() + raise durable = self.path not in (":memory:", "") and not self.path.startswith("file::memory:") maintenance = self._checkpoint_and_vacuum(self.conn, durable=durable) @@ -2590,6 +2937,16 @@ def search_like( # ── graph ───────────────────────────────────────────────────────────────── def upsert_entity(self, node: Node, *, commit: bool = True) -> str: + """Persist an entity and its derived incidence atomically.""" + owns_transaction = not self.conn.transaction_owned_by_current_thread() + try: + return self._upsert_entity_impl(node, commit=commit) + except BaseException: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.rollback() + raise + + def _upsert_entity_impl(self, node: Node, *, commit: bool = True) -> str: normalized = normalize_entity_name(node.name) existing = self.conn.execute( "SELECT id FROM entities WHERE workspace_id=? AND repo_id IS ? " @@ -2623,10 +2980,45 @@ def upsert_entity(self, node: Node, *, commit: bool = True) -> str: self._backfill_entity_text_mentions( nid, name=node.name, workspace_id=node.workspace_id, repo_id=node.repo_id, ) + self._live_canonicalize_entity( + nid, name=node.name, workspace_id=node.workspace_id, repo_id=node.repo_id, + ) if commit: self.conn.commit() return nid + def _live_canonicalize_entity(self, entity_id: str, *, name: str, + workspace_id: Optional[str], + repo_id: Optional[str]) -> None: + """Merge a freshly-written entity into a token-overlap alias group.""" + name = (name or "").strip() + if len(name) < 2 or not workspace_id: + return + entity = self.conn.execute( + "SELECT etype FROM entities WHERE id=?", (entity_id,) + ).fetchone() + if entity is None: + return + candidates = self._entity_blocking_candidates( + entity_id=entity_id, workspace_id=workspace_id, + etype=entity["etype"], name=name, + ) + best: Optional[dict] = None + best_overlap = 0.0 + for peer in candidates: + overlap = _entity_overlap(name, peer["name"]) + if overlap is None or overlap < 0.6 or overlap <= best_overlap: + continue + best_overlap = overlap + best = dict(peer) + if best is None: + return + peer_canonical = best["canonical_id"] or best["id"] + self.conn.execute( + "UPDATE entities SET canonical_id=?, canonical_method=? WHERE id=?", + (peer_canonical, "token_overlap", entity_id), + ) + def _backfill_entity_text_mentions(self, entity_id: str, *, name: str, workspace_id: Optional[str], repo_id: Optional[str]) -> None: @@ -2639,14 +3031,17 @@ def _backfill_entity_text_mentions(self, entity_id: str, *, name: str, name = (name or "").strip() if len(name) < 2: return - scope_sql = "repo_id IS NULL" - scope_params: list[Any] = [] - if repo_id is not None: - # A contextual repo read includes its workspace/user ancestors. Do not - # lose a legacy workspace mention merely because the matching entity was - # introduced later in a repository; sibling repositories stay isolated. + if repo_id is None: + # A workspace-owned entity is the shared identity across its repositories. + # Include every repo-owned memory in this workspace, then partition profile + # writes by the memory owner so a workspace sweep remains repo-isolated. + scope_sql = "1=1" + scope_params: list[Any] = [] + else: + # A repo-owned entity may use workspace-level memories as shared evidence, + # but must not reach a sibling repository. scope_sql = "(repo_id=? OR repo_id IS NULL)" - scope_params.append(repo_id) + scope_params = [repo_id] rows = self.conn.execute( "SELECT id, title, content, workspace_id, repo_id, valid_from, valid_to, " "valid_to_recorded_at, ingested_at, expired_at FROM memories " @@ -2670,7 +3065,6 @@ def _backfill_entity_text_mentions(self, entity_id: str, *, name: str, ingested_at=row["ingested_at"], expired_at=row["expired_at"], provenance={"source": "exact_text_backfill"}, commit=False, ) - def list_entities(self, flt: Optional[SearchFilter] = None, *, limit: Optional[int] = None) -> list[Node]: """Entities in scope, newest first — the seed set the profile-consolidation @@ -2889,6 +3283,21 @@ def list_memory_entities(self, flt: Optional[SearchFilter] = None, *, return rows def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: + """Atomically persist an edge and its normalized support rows. + + The implementation performs several writes. If a later support write fails, + roll back a transaction opened by this call so a partial edge cannot remain + pending on the shared connection. + """ + owns_transaction = not self.conn.transaction_owned_by_current_thread() + try: + return self._upsert_edge_impl(edge, commit=commit) + except BaseException: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.rollback() + raise + + def _upsert_edge_impl(self, edge: Edge, *, commit: bool = True) -> str: eid = edge.id or ids.new_id("edge") layer = normalize_graph_layer(edge.layer, edge.relation).value source, target = edge.src, edge.dst @@ -3137,6 +3546,22 @@ def add_edge_support(self, edge_id: str, provenance: dict, *, valid_from: Optional[float] = None, ingested_at: Optional[float] = None, commit: bool = True) -> None: + """Record support and edge provenance as one write unit.""" + owns_transaction = not self.conn.transaction_owned_by_current_thread() + try: + self._add_edge_support_impl( + edge_id, provenance, valid_from=valid_from, + ingested_at=ingested_at, commit=commit, + ) + except BaseException: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.rollback() + raise + + def _add_edge_support_impl(self, edge_id: str, provenance: dict, *, + valid_from: Optional[float] = None, + ingested_at: Optional[float] = None, + commit: bool = True) -> None: """Record another source memory supporting an existing graph edge.""" incoming = _provenance_memory_ids(provenance) if not incoming: @@ -3399,8 +3824,8 @@ def add_link(self, a: str, b: str, relation: str = "related", if layer is not None else None ) graph_layer = requested_layer or normalize_graph_layer(None, relation).value - started_transaction = not self.conn.in_transaction - if started_transaction: + owns_transaction = not self.conn.transaction_owned_by_current_thread() + if owns_transaction: self.conn.execute("BEGIN IMMEDIATE") try: # A sync bundle may carry a closed link interval. It has no live row to @@ -3418,7 +3843,7 @@ def add_link(self, a: str, b: str, relation: str = "related", ), ).fetchone() if exact is not None: - if started_transaction: + if owns_transaction: self.conn.commit() return existing = self.conn.execute( @@ -3471,7 +3896,7 @@ def add_link(self, a: str, b: str, relation: str = "related", ) if commit: self.conn.commit() - elif started_transaction: + elif owns_transaction: # The pre-read reservation has no write to batch. Release it even # for ``commit=False``; the old no-op path never opened a transaction. self.conn.commit() @@ -3490,7 +3915,7 @@ def add_link(self, a: str, b: str, relation: str = "related", if commit: self.conn.commit() except BaseException: - if started_transaction and self.conn.in_transaction: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): self.conn.rollback() raise @@ -3514,8 +3939,8 @@ def add_link_version(self, a: str, b: str, relation: str = "related", stamp = now_ts() world_start = stamp if valid_from is None else valid_from system_start = stamp if ingested_at is None else ingested_at - started_transaction = not self.conn.in_transaction - if started_transaction: + owns_transaction = not self.conn.transaction_owned_by_current_thread() + if owns_transaction: self.conn.execute("BEGIN IMMEDIATE") try: exact = self.conn.execute( @@ -3530,7 +3955,7 @@ def add_link_version(self, a: str, b: str, relation: str = "related", ), ).fetchone() if exact is not None: - if started_transaction: + if owns_transaction: self.conn.commit() return False self.conn.execute( @@ -3545,7 +3970,7 @@ def add_link_version(self, a: str, b: str, relation: str = "related", self.conn.commit() return True except BaseException: - if started_transaction and self.conn.in_transaction: + if owns_transaction and self.conn.transaction_owned_by_current_thread(): self.conn.rollback() raise @@ -4909,23 +5334,169 @@ def get_sync_state(self, key: str) -> Optional[str]: row = self.conn.execute("SELECT value FROM sync_state WHERE key=?", (key,)).fetchone() return row["value"] if row else None - def set_sync_state(self, key: str, value: str) -> None: + def set_sync_state(self, key: str, value: str, *, commit: bool = True) -> None: self.conn.execute( "INSERT INTO sync_state(key, value, updated_at) VALUES (?,?,?) " "ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (key, value, now_ts()), ) - self.conn.commit() + if commit: + self.conn.commit() + + # ── bounded maintenance cursors (local, never synced) ────────────────────── + def get_maintenance_cursor(self, workspace_id: str, repo_id: Optional[str], + name: str) -> str: + """Return the last keyset id visited by one scoped maintenance sweep.""" + row = self.conn.execute( + "SELECT cursor FROM maintenance_cursors " + "WHERE workspace_id=? AND repo_id=? AND name=?", + (workspace_id, repo_id or "", name), + ).fetchone() + return str(row["cursor"]) if row else "" + + def set_maintenance_cursor(self, workspace_id: str, repo_id: Optional[str], + name: str, cursor: str, *, commit: bool = True) -> None: + """Persist bounded-sweep progress without exposing it to sync peers.""" + normalized_cursor = str(cursor or "") + scope = (workspace_id, repo_id or "", name) + existing = self.conn.execute( + "SELECT cursor FROM maintenance_cursors " + "WHERE workspace_id=? AND repo_id=? AND name=?", + scope, + ).fetchone() + if existing is not None and str(existing["cursor"] or "") == normalized_cursor: + return + if existing is None: + self.conn.execute( + "INSERT INTO maintenance_cursors(" + "workspace_id, repo_id, name, cursor, updated_at" + ") VALUES (?,?,?,?,?)", + (*scope, normalized_cursor, now_ts()), + ) + else: + self.conn.execute( + "UPDATE maintenance_cursors SET cursor=?, updated_at=? " + "WHERE workspace_id=? AND repo_id=? AND name=?", + (normalized_cursor, now_ts(), *scope), + ) + if commit: + self.conn.commit() + + # ── sync tombstones (durable deletion markers that propagate) ─────────────── + def add_memory_tombstone(self, memory_id: str, *, deleted_at: Optional[float] = None, + device_id: Optional[str] = None, + workspace_id: Optional[str] = None, + repo_id: Optional[str] = None) -> None: + """Record that a memory id is dead (secure-erased) so sync can propagate it. + + Carries no user content — only the id, the erasure time, and the origin + device. Earliest ``deleted_at`` wins, exactly like the ``valid_to`` closure + lattice, so a replayed or stale erasure can never resurrect a memory or move + a tombstone later in time. The caller owns the transaction/commit. + """ + ts = now_ts() if deleted_at is None else deleted_at + did = device_id or self.device_id() + existing = self.conn.execute( + "SELECT deleted_at, device_id, workspace_id, repo_id " + "FROM memory_tombstones WHERE memory_id=?", + (memory_id,), + ).fetchone() + if existing is None: + self.conn.execute( + "INSERT INTO memory_tombstones(" + "memory_id, deleted_at, device_id, workspace_id, repo_id, created_at" + ") VALUES (?,?,?,?,?,?)", + (memory_id, ts, did, workspace_id, repo_id, ts), + ) + return + existing_workspace = existing["workspace_id"] + if ( + existing_workspace is not None + and workspace_id is not None + and existing_workspace != workspace_id + ): + raise ValueError("tombstone workspace scope conflicts with existing marker") + existing_repo = existing["repo_id"] + if ( + existing_repo is not None + and repo_id is not None + and existing_repo != repo_id + ): + raise ValueError("tombstone repository scope conflicts with existing marker") + earlier = float(ts) < float(existing["deleted_at"]) + merged_workspace = ( + None + if existing_workspace is None or workspace_id is None + else (workspace_id if earlier else existing_workspace) + ) + # A repo-less marker is legacy global state. Never narrow it to a repo; + # conversely, a legacy marker arriving after a known repo marker widens + # the terminal scope rather than allowing sibling-specific overwrite. + merged_repo = ( + None + if existing_repo is None or repo_id is None + else existing_repo + ) + self.conn.execute( + "UPDATE memory_tombstones SET deleted_at=?, device_id=?, " + "workspace_id=?, repo_id=? WHERE memory_id=?", + ( + ts if earlier else existing["deleted_at"], + did if earlier else existing["device_id"], + merged_workspace, + merged_repo, + memory_id, + ), + ) + + def list_memory_tombstones(self, workspace_id: Optional[str] = None, + repo_id: Optional[str] = None) -> list[dict]: + """Return tombstones scoped to a workspace and, when selected, one repo. + + Workspace-scoped tombstones remain visible to every repo in that workspace; + repo-scoped tombstones never cross a repo-only export boundary. + """ + if workspace_id is None and repo_id is not None: + raise ValueError("repo_id requires workspace_id") + if workspace_id is None: + rows = self.conn.execute( + "SELECT memory_id, deleted_at, device_id, workspace_id, repo_id " + "FROM memory_tombstones ORDER BY memory_id" + ).fetchall() + elif repo_id is None: + rows = self.conn.execute( + "SELECT memory_id, deleted_at, device_id, workspace_id, repo_id " + "FROM memory_tombstones WHERE workspace_id=? " + "ORDER BY memory_id", + (workspace_id,), + ).fetchall() + else: + rows = self.conn.execute( + "SELECT memory_id, deleted_at, device_id, workspace_id, repo_id " + "FROM memory_tombstones WHERE workspace_id=? AND (repo_id=? OR repo_id IS NULL) " + "ORDER BY memory_id", + (workspace_id, repo_id), + ).fetchall() + return [ + { + "id": str(row["memory_id"]), "deleted_at": float(row["deleted_at"]), + "device": str(row["device_id"] or ""), + "workspace_id": row["workspace_id"], + "repo_id": row["repo_id"], + } + for row in rows + ] def device_id(self) -> str: """Stable per-database device id (minted once, then persistent). Attributes sync bundles to their origin device so a store never re-applies its own writes; it is local metadata, never memory, and only ever leaves the machine inside a bundle header.""" + owns_transaction = not self.conn.transaction_owned_by_current_thread() did = self.get_sync_state("device_id") if not did: did = ids.new_id("device") - self.set_sync_state("device_id", did) + self.set_sync_state("device_id", did, commit=owns_transaction) return did # ── helpers ─────────────────────────────────────────────────────────────── @@ -5007,6 +5578,10 @@ def _row_to_record(row: sqlite3.Row) -> MemoryRecord: title=row["title"] or "", summary=row["summary"] or "", keywords=_loads(row["keywords"], []), metadata=_loads(row["metadata"], {}), importance=row["importance"], surprise=row["surprise"], stability=row["stability"], + confidence=( + row["confidence"] + if "confidence" in row.keys() and row["confidence"] is not None else 1.0 + ), access_count=row["access_count"], last_access=row["last_access"], valid_from=row["valid_from"], valid_to=row["valid_to"], valid_to_recorded_at=( @@ -5018,6 +5593,8 @@ def _row_to_record(row: sqlite3.Row) -> MemoryRecord: claim_kind=row["claim_kind"] if "claim_kind" in row.keys() else "", pinned=bool(row["pinned"]), sensitivity=row["sensitivity"], provenance=_loads(row["provenance"], {}), + pinned_at=row["pinned_at"] if "pinned_at" in row.keys() else None, + unpinned_at=row["unpinned_at"] if "unpinned_at" in row.keys() else None, ) diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index f4876a5b..ccd861e1 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -22,7 +22,15 @@ device invalidates everywhere — never resurrected). - ``stability`` / ``access_count`` / ``last_access``: ``max`` (reinforcement is monotone; the spacing effect only ever grows stability). - - ``pinned``: logical OR. + - ``pinned``: a per-field pin lattice — ``pinned_at``/``unpinned_at`` markers + merge latest-wins (the newest transition dominates), so a re-pin on one device + beats a stale unpin on another instead of losing a legitimate toggle. + ``pinned`` itself is derived from the merged markers. + - ``deleted_at``: secure-erase tombstones are terminal within their known + repository scope. An erased id is carried in the bundle's ``tombstones`` list + (id + erasure time + origin device, never content) and merged earliest-wins; + legacy repo-less markers remain global for compatibility, while a known marker + cannot erase a same-id row from a sibling repository. - descriptive fields (title/content/keywords/…): last-writer-wins under a **deterministic total order** — ``(last_access, ingested_at, content-hash)`` — so the winner is a function of the data, never of arrival order. @@ -68,6 +76,9 @@ SYNC_VERSION = 2 SYNC_ACCEPTED_VERSIONS = frozenset({1, 2}) +# ── tombstone bundle constants ──────────────────────────────────────────────── +MAX_TOMBSTONES = 200_000 # same cap as MAX_MEMORIES (ids only, no content) + # ── validation caps (untrusted bundle → clamp, don't trust) ─────────────────── MAX_MEMORIES = 200_000 MAX_LINKS = 500_000 @@ -100,7 +111,7 @@ # handled separately and are NOT part of this set. _LWW_FIELDS = ( "title", "content", "summary", "keywords", "metadata", "mtype", "scope", - "importance", "surprise", "sensitivity", "valid_from", "ingested_at", + "importance", "surprise", "confidence", "sensitivity", "valid_from", "ingested_at", "session_id", "provenance", "subject_key", "claim_kind", ) @@ -145,7 +156,7 @@ def _label_tuple(rec: MemoryRecord) -> list: return [ rec.title, rec.content, rec.summary, sorted(rec.keywords or []), _enum(rec.mtype), _enum(rec.scope), rec.importance, rec.surprise, - rec.sensitivity, rec.valid_from, rec.session_id, + rec.confidence, rec.sensitivity, rec.valid_from, rec.session_id, rec.subject_key, rec.claim_kind, json.dumps(rec.metadata or {}, sort_keys=True, default=str), json.dumps(rec.provenance or {}, sort_keys=True, default=str), @@ -169,6 +180,7 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: """ winner = local if _version_key(local) >= _version_key(incoming) else incoming valid_to, valid_to_recorded_at = _merge_closure(local, incoming) + pinned, pinned_at, unpinned_at = _pin_lattice(local, incoming) return MemoryRecord( id=local.id, # scope pointers are always local — never merged from the remote @@ -178,7 +190,8 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: content=winner.content, title=winner.title, summary=winner.summary, keywords=list(winner.keywords or []), metadata=dict(winner.metadata or {}), mtype=winner.mtype, scope=winner.scope, importance=winner.importance, - surprise=winner.surprise, sensitivity=winner.sensitivity, + surprise=winner.surprise, confidence=winner.confidence, + sensitivity=winner.sensitivity, session_id=winner.session_id, provenance=dict(winner.provenance or {}), subject_key=winner.subject_key, claim_kind=winner.claim_kind, valid_from=winner.valid_from, @@ -199,7 +212,9 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: stability=max(local.stability, incoming.stability), access_count=max(local.access_count, incoming.access_count), last_access=_max_nonnull(local.last_access, incoming.last_access), - pinned=bool(local.pinned or incoming.pinned), + pinned=pinned, + pinned_at=pinned_at, + unpinned_at=unpinned_at, valid_to_recorded_at=valid_to_recorded_at, ) @@ -231,6 +246,71 @@ def _merge_closure( ) +def _merge_closure_ts(a: Optional[float], b: Optional[float]) -> Optional[float]: + """Latest non-null wins for one pin-transition marker. + + Pin state is a toggle, not a closure. The newest transition must therefore + dominate an older peer marker; retaining only the earliest marker makes a + legitimate re-pin impossible to propagate after an unpin. + """ + if a is None: + return b + if b is None: + return a + return a if a >= b else b + + +def _normalise_pin_state(rec: MemoryRecord) -> tuple[bool, Optional[float], Optional[float]]: + """Drop contradictory marker combinations before merging untrusted rows. + + A pinned row may be legacy (no markers), or carry a pin marker newer than its + unpin marker. An unpinned row may carry the history of a pin, but a pin marker + without an unpin marker is inconsistent with ``pinned=False`` and must not grant + authority merely because a peer serialized a timestamp. + """ + pinned = bool(rec.pinned) + pinned_at = rec.pinned_at + unpinned_at = rec.unpinned_at + if pinned: + if pinned_at is None: + # Legacy pinned rows have no trustworthy transition clock. Preserve the + # legacy state and ignore a marker-only forged unpin. + unpinned_at = None + elif unpinned_at is not None and pinned_at <= unpinned_at: + # The markers describe an unpinned state; the explicit boolean cannot + # override the newer unpin event. + pinned = False + elif pinned_at is not None and ( + unpinned_at is None or pinned_at > unpinned_at): + # A false row must not become pinned from a lone peer-controlled marker. + pinned_at = None + return pinned, pinned_at, unpinned_at + + +def _pin_lattice(local: MemoryRecord, incoming: MemoryRecord) -> tuple[bool, Optional[float], Optional[float]]: + """Merge pin state as a latest-transition lattice. + + The two markers retain the latest pin and unpin events. Deriving the boolean + from their newest values makes pin/unpin/re-pin convergence commutative, + associative, and idempotent while rejecting contradictory marker-only authority. + """ + local_pinned, local_pinned_at, local_unpinned_at = _normalise_pin_state(local) + incoming_pinned, incoming_pinned_at, incoming_unpinned_at = _normalise_pin_state(incoming) + pinned_at = _merge_closure_ts(local_pinned_at, incoming_pinned_at) + unpinned_at = _merge_closure_ts(local_unpinned_at, incoming_unpinned_at) + # A legacy pinned row carries no marker. Treat it as pinned since the epoch so + # an explicit unpin can still propagate to peers that have not migrated it. + if pinned_at is None and (local_pinned or incoming_pinned): + pinned_at = 0.0 + if unpinned_at is None: + pinned = pinned_at is not None + elif pinned_at is None: + pinned = False + else: + pinned = pinned_at > unpinned_at + return pinned, pinned_at, unpinned_at + + # Fields ``Store.add_memory`` fills in from the SERVER clock when they arrive as ``None`` # (store.py: ``ingested_at``/``valid_from``/``last_access`` are each defaulted to ``now_ts()``). # For these, "omitted by the bundle" is NOT a competing value — the store has no way to @@ -284,6 +364,7 @@ def _same_sync_payload(left: MemoryRecord, right: MemoryRecord) -> bool: and left.scope == right.scope and left.importance == right.importance and left.surprise == right.surprise + and left.confidence == right.confidence and left.sensitivity == right.sensitivity and left.valid_from == right.valid_from and left.ingested_at == right.ingested_at @@ -299,6 +380,7 @@ def _signature(rec: MemoryRecord) -> str: rec.valid_to, rec.valid_to_recorded_at, rec.expired_at, rec.ingested_at, rec.stability, rec.access_count, rec.last_access, bool(rec.pinned), + rec.pinned_at, rec.unpinned_at, ]) @@ -311,11 +393,13 @@ def record_to_dict(rec: MemoryRecord) -> dict: "title": rec.title, "content": rec.content, "summary": rec.summary, "keywords": list(rec.keywords or []), "metadata": rec.metadata or {}, "importance": rec.importance, "surprise": rec.surprise, "stability": rec.stability, - "access_count": rec.access_count, "last_access": rec.last_access, + "confidence": rec.confidence, "access_count": rec.access_count, + "last_access": rec.last_access, "valid_from": rec.valid_from, "valid_to": rec.valid_to, "valid_to_recorded_at": rec.valid_to_recorded_at, "ingested_at": rec.ingested_at, "expired_at": rec.expired_at, "pinned": bool(rec.pinned), "sensitivity": rec.sensitivity, + "pinned_at": rec.pinned_at, "unpinned_at": rec.unpinned_at, "subject_key": rec.subject_key, "claim_kind": rec.claim_kind, "provenance": rec.provenance or {}, } @@ -500,6 +584,7 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]: importance=_clamp_num(d.get("importance"), 0.0, 1.0, 0.0), surprise=_clamp_num(d.get("surprise"), 0.0, 100.0, 1.0), stability=_clamp_num(d.get("stability"), 0.0, MAX_STABILITY, 1.0), + confidence=_clamp_num(d.get("confidence"), 0.0, 1.0, 1.0), access_count=min(MAX_ACCESS_COUNT, max(0, _as_int(d.get("access_count"), 0))), last_access=_clamp_ts(d.get("last_access"), now), # World-time validity may be in the future; the system timestamps below may not @@ -512,6 +597,8 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]: # Authority-bearing booleans are strict. In particular ``"false"`` must # not become truthy and then remain permanently pinned through the CRDT OR. pinned=d.get("pinned") is True, sensitivity=sens, + pinned_at=_clamp_ts(d.get("pinned_at"), now), + unpinned_at=_clamp_ts(d.get("unpinned_at"), now), subject_key=_clamp_str(d.get("subject_key"), 512), claim_kind=_clamp_str(d.get("claim_kind"), 256), provenance=_safe_json_obj(d.get("provenance")), @@ -577,6 +664,7 @@ def export_bundle(self, workspace_id: str, *, repo_id: Optional[str] = None) -> "workspace_name": ws_name, "repos": {r["id"]: r["name"] for r in repo_rows}, "memories": [record_to_dict(m) for m in mems], + "tombstones": self.store.list_memory_tombstones(workspace_id, repo_id), "mem_links": [ { "a": ln["a"], "b": ln["b"], "relation": ln["relation"], @@ -612,9 +700,12 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, mem_dicts = bundle.get("memories") or [] link_dicts = bundle.get("mem_links") or [] - if not isinstance(mem_dicts, list) or not isinstance(link_dicts, list): - raise SyncError("bundle memories/mem_links must be lists") - if len(mem_dicts) > MAX_MEMORIES or len(link_dicts) > MAX_LINKS: + tomb_dicts = bundle.get("tombstones") or [] + if not isinstance(mem_dicts, list) or not isinstance(link_dicts, list) \ + or not isinstance(tomb_dicts, list): + raise SyncError("bundle memories/mem_links/tombstones must be lists") + if len(mem_dicts) > MAX_MEMORIES or len(link_dicts) > MAX_LINKS \ + or len(tomb_dicts) > MAX_TOMBSTONES: raise SyncError("bundle exceeds size caps") raw_ws_name = into_workspace if into_workspace is not None else bundle.get("workspace_name") @@ -626,7 +717,7 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, if self.allowed_workspaces is not None and ws_name not in self.allowed_workspaces: raise SyncError("workspace %r is not authorized for sync" % ws_name) report = {"added": 0, "updated": 0, "unchanged": 0, "rejected": 0, - "links_added": 0, "links_updated": 0, + "links_added": 0, "links_updated": 0, "tombstones_applied": 0, "workspace": ws_name, "dry_run": bool(dry_run)} # Resolve scope by NAME (per-device ids differ; names are the sync key). A @@ -645,18 +736,119 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, if dry_run: row = self.store.conn.execute( "SELECT id FROM workspaces WHERE name=?", (ws_name,)).fetchone() - local_ws = row["id"] if row else None + # Use non-persisted scope sentinels when the target does not exist yet. + # Dry-run must evaluate the same repo-scoped acceptance path as a real + # apply; ``None`` would incorrectly reject rows that the real apply would + # accept after creating the workspace/repository. + local_ws = row["id"] if row else f"__dry_run_workspace__:{ws_name}" for rid, rname in valid_remote_repos.items(): repo_row = (self.store.conn.execute( "SELECT id FROM repos WHERE workspace_id=? AND name=?", - (local_ws, rname)).fetchone() if local_ws is not None else None) - repo_remap[rid] = repo_row["id"] if repo_row else None + (row["id"], rname)).fetchone() if row else None) + repo_remap[rid] = ( + repo_row["id"] if repo_row + else f"__dry_run_repo__:{ws_name}:{rid}" + ) else: local_ws = self.store.get_or_create_workspace(ws_name) for rid, rname in valid_remote_repos.items(): repo_remap[rid] = self.store.get_or_create_repo(local_ws, rname) accepted: dict[str, MemoryRecord] = {} + parsed_tombstones = self._parse_tombstones(tomb_dicts, src_device) + accepted_tombstones: list[dict] = [] + + # Tombstones are scoped before they are applied. A bundle authorized for one + # workspace must never hard-delete a known id owned by another workspace. + for tomb in parsed_tombstones: + remote_tomb_repo = tomb.get("repo_id") + mapped_tomb_repo = ( + repo_remap.get(remote_tomb_repo) + if remote_tomb_repo is not None else None + ) + if remote_tomb_repo is not None and mapped_tomb_repo is None: + report["rejected"] += 1 + continue + existing = ( + self.store.get_memory(tomb["id"]) + if local_ws is not None else None + ) + # Tombstone scope is durable even after the erased row disappears. Do not + # let a same-id marker from another workspace overwrite or poison the local + # workspace's deletion state when the id is no longer present locally. + tombstone_row = self.store.conn.execute( + "SELECT workspace_id, repo_id, deleted_at " + "FROM memory_tombstones WHERE memory_id=?", + (tomb["id"],) + ).fetchone() + if (tombstone_row is not None + and tombstone_row["workspace_id"] is not None + and tombstone_row["workspace_id"] != local_ws): + report["rejected"] += 1 + continue + # Once a tombstone has a repository identity, a marker from a sibling + # repository must not overwrite it. A NULL marker is legacy global + # state and must not be upgraded from an incoming repository identity. + if (tombstone_row is not None + and tombstone_row["repo_id"] is not None + and mapped_tomb_repo is not None + and tombstone_row["repo_id"] != mapped_tomb_repo): + report["rejected"] += 1 + continue + if (existing is not None and existing.workspace_id != local_ws): + report["rejected"] += 1 + continue + # A repo-scoped tombstone can only erase a row in that same repo. + # Legacy repo-less markers retain their historical global-id behavior. + if (existing is not None and mapped_tomb_repo is not None + and existing.repo_id != mapped_tomb_repo): + report["rejected"] += 1 + continue + if (existing is not None and only_repo_id is not None + and existing.repo_id != only_repo_id): + report["rejected"] += 1 + continue + if (only_repo_id is not None and mapped_tomb_repo is not None + and mapped_tomb_repo != only_repo_id): + report["rejected"] += 1 + continue + # Preserve an already-known repository identity, but never infer one + # from the live row for a legacy marker: doing so narrows a global marker + # and permits a same-id row from a sibling repository to resurrect. + stored_tomb_repo = mapped_tomb_repo + if stored_tomb_repo is None and tombstone_row is not None: + stored_tomb_repo = tombstone_row["repo_id"] + marker_changed = ( + tombstone_row is None + or float(tomb["deleted_at"]) < float(tombstone_row["deleted_at"]) + or tombstone_row["repo_id"] != stored_tomb_repo + ) + accepted_tombstones.append({ + **tomb, "_mapped_repo_id": stored_tomb_repo, + }) + if not dry_run: + self.store.add_memory_tombstone( + tomb["id"], deleted_at=tomb["deleted_at"], + device_id=tomb["device"], workspace_id=local_ws, + repo_id=stored_tomb_repo, + ) + # A peer's secure erase must remove a row this device still holds + # immediately, not only block a future re-add. + if existing is not None: + try: + self.store._erase_memory_rows( + self.store.conn, tomb["id"], actor="sync_tombstone" + ) + except Exception: # noqa: BLE001 — never leave erased data resident + # The tombstone must not be treated as successfully applied if + # local derivative cleanup failed. Roll back this tombstone batch + # so a retry can recover instead of leaving stale content behind. + self.store.conn.rollback() + raise + if marker_changed or dry_run: + report["tombstones_applied"] += 1 + if not dry_run and accepted_tombstones: + self.store.conn.commit() # Bulk apply. Previously this was N+1: a SELECT per id to test existence, then a # Store.add_memory that did its own dupe-check SELECT, INSERT, FTS delete+insert, @@ -672,7 +864,8 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, # than retrying; one wide transaction would silently roll the whole bundle back. try: self._apply_memories(mem_dicts, report, accepted, local_ws, - repo_remap, only_repo_id, src_device, dry_run) + repo_remap, only_repo_id, src_device, dry_run, + accepted_tombstones) self._apply_links(link_dicts, report, accepted, local_ws, only_repo_id, src_device, dry_run) except BaseException: @@ -687,8 +880,33 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, return report def _apply_memories(self, mem_dicts: list, report: dict, - accepted: dict, local_ws, repo_remap: dict, - only_repo_id, src_device, dry_run: bool) -> None: + accepted: dict[str, MemoryRecord], local_ws, repo_remap: dict, + only_repo_id, src_device, dry_run: bool, + tombstones: Optional[list[dict]] = None) -> None: + # Keep repository identity with the terminal marker. A known repo marker + # must not reject a same-id memory from a sibling repo; a legacy NULL repo + # marker remains global for backward compatibility. + live_tombstones = ( + { + t["id"]: (float(t["deleted_at"]), t.get("repo_id")) + for t in self.store.list_memory_tombstones(local_ws) + } + if local_ws is not None else {} + ) + for tomb in tombstones or []: + timestamp = float(tomb["deleted_at"]) + mapped_repo = tomb.get("_mapped_repo_id") + if mapped_repo is None and tomb.get("repo_id") is not None: + mapped_repo = repo_remap.get(tomb["repo_id"]) + existing = live_tombstones.get(tomb["id"]) + if existing is None: + live_tombstones[tomb["id"]] = (timestamp, mapped_repo) + elif existing[1] is None: + # A legacy marker is global. Never upgrade it to a repository + # identity merely because a newer peer also knows a repo scope. + continue + elif mapped_repo is not None and timestamp < existing[0]: + live_tombstones[tomb["id"]] = (timestamp, mapped_repo) for start in range(0, len(mem_dicts), APPLY_BATCH): batch = mem_dicts[start:start + APPLY_BATCH] parsed = [dict_to_record(d) for d in batch] @@ -700,13 +918,14 @@ def _apply_memories(self, mem_dicts: list, report: dict, [rec.id for rec in parsed if rec is not None]) for d, rec in zip(batch, parsed): self._apply_one(d, rec, report, accepted, known, local_ws, - repo_remap, only_repo_id, src_device, dry_run) + repo_remap, only_repo_id, src_device, dry_run, + live_tombstones) if not dry_run: self.store.conn.commit() def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, local_ws, repo_remap: dict, only_repo_id, src_device, - dry_run: bool) -> None: + dry_run: bool, live_tombstones: Optional[dict] = None) -> None: if rec is None: report["rejected"] += 1 return @@ -743,15 +962,24 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, # synced-in memory stays auditable ("why is this known?" — AGENTS.md §3.6). rec.workspace_id = local_ws if remote_repo_id: - if remote_repo_id not in repo_remap: + if (remote_repo_id not in repo_remap + or repo_remap[remote_repo_id] is None): + # Dry-run does not create missing repositories. A repo-scoped row whose + # source repo cannot be resolved must still be rejected; accepting it + # with repo_id=None would silently broaden visibility to the workspace. report["rejected"] += 1 return rec.repo_id = repo_remap[remote_repo_id] - if rec.repo_id is None and only_repo_id is not None: - report["rejected"] += 1 - return else: rec.repo_id = None + # A known repository tombstone is terminal only for that repository. Legacy + # repo-less tombstones intentionally retain their historical global-id behavior. + tombstone = (live_tombstones or {}).get(rec.id) + if tombstone is not None: + tombstone_repo = tombstone[1] if isinstance(tombstone, tuple) else None + if tombstone_repo is None or tombstone_repo == rec.repo_id: + report["rejected"] += 1 + return if only_repo_id is not None and rec.repo_id != only_repo_id: report["rejected"] += 1 return @@ -867,8 +1095,9 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, "synced record quarantined by deterministic policy", commit=False, ) - known[rec.id] = rec # write-through: a duplicate id later in this - # batch must see what we just persisted + # Keep the dry-run view write-through as well: duplicate ids in one + # bundle must be evaluated against the first row, not the pre-bundle store. + known[rec.id] = rec report["added"] += 1 accepted[rec.id] = rec else: @@ -891,7 +1120,8 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, "sync_overwrite", merged.id, "content replaced by synced bundle (last-writer-wins)", commit=False) - known[rec.id] = merged + # Keep duplicate processing deterministic during dry-run too. + known[rec.id] = merged report["updated"] += 1 accepted[rec.id] = merged @@ -1006,6 +1236,56 @@ def _apply_links(self, link_dicts: list, report: dict, accepted: dict, if not dry_run: self.store.conn.commit() + def _parse_tombstones(self, tomb_dicts: list, src_device: object) -> list[dict]: + """Validate + clamp untrusted bundle tombstones. Never raises. + + A tombstone is ``{id, deleted_at, device, repo_id}`` — no content — so there is + nothing to quarantine; it is clamped like any other untrusted input and a + malformed entry is silently dropped (counted by the caller only for entries + that survive). ``deleted_at`` is bounded to ``[0, now + skew]`` so a hostile + far-future erasure cannot permanently tombstone a memory id. A missing + ``repo_id`` is a legacy global marker. + """ + # Scope is part of tombstone identity now. Keep the earliest event for + # each (memory id, repository) pair, but a legacy repo-less marker is + # global and therefore suppresses every repo-scoped marker for that id. + best: dict[tuple[str, Optional[str]], dict] = {} + positions: dict[tuple[str, Optional[str]], int] = {} + now = now_ts() + for t in tomb_dicts: + if not isinstance(t, dict): + continue + mid = t.get("id") + deleted_at = _as_float(t.get("deleted_at"), None) + if not isinstance(mid, str) or not mid or deleted_at is None: + continue + mid = _clamp_str(mid, 128) + if not mid: + continue + deleted_at = max(0.0, min(deleted_at, now + TS_FUTURE_SKEW)) + device = _clamp_str(t.get("device"), 128) if t.get("device") else "" + repo_id = ( + _clamp_str(t.get("repo_id"), 128) + if isinstance(t.get("repo_id"), str) and t.get("repo_id") + else None + ) + key = (mid, repo_id) + previous = best.get(key) + if previous is not None and previous["deleted_at"] <= deleted_at: + continue + best[key] = { + "id": mid, "deleted_at": deleted_at, + "device": device or (_clamp_str(src_device, 128) if src_device else ""), + "repo_id": repo_id, + } + if key not in positions: + positions[key] = len(positions) + global_ids = {mid for mid, repo_id in best if repo_id is None} + return [ + best[key] for key in positions + if key[1] is None or key[0] not in global_ids + ] + def _write(self, rec: MemoryRecord, *, commit: bool = True) -> None: """Persist a merged/new record verbatim (ids + timestamps preserved) and keep derived state coherent: re-embed for the vector arm when an embedder is wired. @@ -1102,7 +1382,7 @@ def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, applied: list[dict] = [] totals = { "added": 0, "updated": 0, "unchanged": 0, "rejected": 0, - "links_added": 0, "links_updated": 0, + "links_added": 0, "links_updated": 0, "tombstones_applied": 0, } # Fetch each bundle inside its own try: a transport that raises while producing # bundle N (a relay 404 on a bundle deleted mid-round, an oversized blob) used to diff --git a/engraphis/dashboard_app.py b/engraphis/dashboard_app.py index 220a7508..9e416268 100644 --- a/engraphis/dashboard_app.py +++ b/engraphis/dashboard_app.py @@ -6,8 +6,10 @@ """ from __future__ import annotations +import asyncio import importlib.util import hmac +import logging from pathlib import Path from urllib.parse import urlsplit @@ -16,7 +18,7 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field @@ -34,12 +36,41 @@ from engraphis.routes import v2_api from engraphis.service import MemoryService +logger = logging.getLogger("engraphis") + _STATIC = Path(__file__).resolve().parent / "static" _CLASSIC_ASSETS = Path(__file__).resolve().parent / "classic_assets" _V2_ASSETS = Path(__file__).resolve().parent / "dashboard_assets" _INDEX = _V2_ASSETS / "index.html" +async def _dashboard_consolidation_loop(service: MemoryService) -> None: + """Run opt-in v2 consolidation from the dashboard's actual lifespan. + + The retired compatibility app owns the historical consciousness loop, but the + supported dashboard is the process that serves the v2 MemoryService. Keep this + maintenance task v2-only and dispatch both the candidate scan and SQLite writes to + worker threads so request handling never shares the event loop with a sweep. + """ + from engraphis.app import _consolidation_candidates_exist, _run_loop_consolidation + + ticks = 0 + while True: + try: + await asyncio.sleep(settings.loop_interval) + ticks += 1 + interval = int(settings.loop_consolidate) + if interval <= 0 or ticks % interval: + continue + if not await asyncio.to_thread(_consolidation_candidates_exist, service.engine): + continue + await asyncio.to_thread(_run_loop_consolidation, service.engine) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - maintenance must not kill the server + logger.error("Dashboard consolidation loop error (%s)", type(exc).__name__) + + class _FreshStaticFiles(StaticFiles): """Revalidate local dashboard assets so a running UI cannot pin an old renderer. @@ -48,7 +79,21 @@ class _FreshStaticFiles(StaticFiles): an older graph engine alive after a source/package update. """ + @staticmethod + def _is_private_asset(path: str) -> bool: + """Keep package implementation files out of the public asset mounts.""" + parts = [part for part in path.replace("\\", "/").split("/") if part] + return ( + any(part == "__pycache__" or part.startswith(".") for part in parts) + or any( + part.lower().endswith((".py", ".pyc", ".pyo", ".pyi")) + for part in parts + ) + ) + async def get_response(self, path, scope): + if self._is_private_asset(path): + return Response(status_code=404) response = await super().get_response(path, scope) response.headers["Cache-Control"] = "no-cache, must-revalidate" return response @@ -165,6 +210,7 @@ def create_app() -> FastAPI: @_contextlib.asynccontextmanager async def _lifespan(app: FastAPI): + background_task = None try: # one-line "update available" notice (background, fail-silent, opt-out) import logging as _logging @@ -172,11 +218,25 @@ async def _lifespan(app: FastAPI): update_check.emit_startup_notice(_logging.getLogger("engraphis").info) except Exception: # noqa: BLE001 - never block dashboard startup pass - if _mcp_asgi is not None: - async with _mcp_mgr.run(): + if settings.loop_interval > 0 and settings.loop_consolidate > 0: + background_task = asyncio.create_task(_dashboard_consolidation_loop(svc)) + logger.info( + "Dashboard consolidation loop started (interval=%ds)", + settings.loop_interval, + ) + try: + if _mcp_asgi is not None: + async with _mcp_mgr.run(): + yield + else: yield - else: - yield + finally: + if background_task is not None: + background_task.cancel() + try: + await background_task + except asyncio.CancelledError: + pass # FastAPI's interactive docs execute CDN-hosted JavaScript with same-origin # authority. Do not expose that supply-chain surface on an authenticated memory @@ -215,6 +275,7 @@ async def _license_error(request: Request, exc: licensing.LicenseError): svc = MemoryService.create( settings.db_path, embed_model=settings.embed_model, embed_dim=settings.embed_dim or 384, + vector_backend=settings.vector_backend, allowed_workspaces=settings.allowed_workspaces) app.state.service = svc # The review token is intentionally process-local and is never a general API diff --git a/engraphis/dashboard_assets/engraphis-graph.js b/engraphis/dashboard_assets/engraphis-graph.js index 77283fbf..3a20153a 100644 --- a/engraphis/dashboard_assets/engraphis-graph.js +++ b/engraphis/dashboard_assets/engraphis-graph.js @@ -97,8 +97,15 @@ `linkCanvasObject` does — they only appear once the user has zoomed in past this scale. */ const LINK_LABEL_MIN_SCALE = 2.4; + function hasOwn(value, key) { + return value != null && Object.prototype.hasOwnProperty.call(value, key); + } function idOf(value) { return value && typeof value === 'object' ? value.id : value; } - function nodeName(node) { return String(node.name || node.label || node.id || ''); } + function nodeName(node) { + if (node === undefined || node === null) return ''; + if (typeof node !== 'object' && typeof node !== 'function') return String(node); + return String(node.name || node.label || node.id || ''); + } function showRelationLabel(label) { return Boolean(label) && String(label).toLowerCase() !== 'co_occurs'; } @@ -141,11 +148,21 @@ const radius = size * 0.45 * (0.55 + Math.min(1.6, normalized * 1.9)); return Math.max(0.8, Math.min(size * 1.1, radius)); } + function validNodeId(value) { + const type = typeof value; + return type === 'string' || type === 'boolean' + || (type === 'number' && Number.isFinite(value)); + } function linkEndpoint(link, side) { - return idOf(link[side] !== undefined ? link[side] : link[side === 'source' ? 'from' : 'to']); + if (!link || (typeof link !== 'object' && typeof link !== 'function')) return null; + const value = link[side] !== undefined ? link[side] : link[side === 'source' ? 'from' : 'to']; + return idOf(value); } function asOfValue(value) { - if (value instanceof Date) return value.getTime(); + if (value instanceof Date) { + const parsed = value.getTime(); + return Number.isFinite(parsed) ? parsed : null; + } if (typeof value === 'number') return Number.isFinite(value) ? value * (value < 1e11 ? 1000 : 1) : null; if (typeof value === 'string' && value.trim()) { const numeric = Number(value); @@ -156,9 +173,11 @@ return null; } function temporalValue(item, key, fallback) { + if (!item || (typeof item !== 'object' && typeof item !== 'function')) return fallback; const value = item[key] !== undefined ? item[key] : item[key === 'valid_from' ? 'born' : 'closed']; if (value === undefined || value === null || value === '') return fallback; - return asOfValue(value); + const parsed = asOfValue(value); + return parsed === null ? fallback : parsed; } /* Node and link labels come from ingested memories, i.e. untrusted text. force-graph's @@ -172,15 +191,21 @@ } function hexRgb(c) { - if (!c) return [140, 131, 232]; - if (c[0] === '#') { - const hex = c.length === 4 ? c[1] + c[1] + c[2] + c[2] + c[3] + c[3] : c.slice(1, 7); + const fallback = [140, 131, 232]; + if (typeof c !== 'string') return fallback; + const value = c.trim(); + if (!value) return fallback; + if (value[0] === '#') { + const hex = value.length === 4 + ? value[1] + value[1] + value[2] + value[2] + value[3] + value[3] + : value.slice(1, 7); + if (!/^[0-9a-f]{6}$/i.test(hex)) return fallback; const n = parseInt(hex, 16); - if (!Number.isFinite(n)) return [140, 131, 232]; return [n >> 16 & 255, n >> 8 & 255, n & 255]; } - const m = c.match(/\d+/g) || [140, 131, 232]; - return [+m[0], +m[1], +m[2]]; + const matches = value.match(/-?\d+(?:\.\d+)?/g) || []; + if (matches.length < 3) return fallback; + return matches.slice(0, 3).map(component => Math.max(0, Math.min(255, Math.round(Number(component))))); } function alpha(c, a) { const [r, g, b] = hexRgb(c); return 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')'; } function mixColours(a, b, amount) { @@ -694,14 +719,14 @@ for both. Same semantics here. */ const CLUSTER_EXCLUDED_LABELS = { influences: true }; function clustersAcross(link) { - return !!(link && CLUSTER_EXCLUDED_LABELS[link.label]); + return !!(link && hasOwn(CLUSTER_EXCLUDED_LABELS, link.label)); } function communities(nodes, links) { - const adj = {}; + const adj = Object.create(null); // Traversal adjacency (hover neighbourhood, focus depth, bridges, betweenness) keeps every // relation; only the community BFS below reads `clusterAdj`. - const clusterAdj = {}; + const clusterAdj = Object.create(null); const nodesById = new Map(nodes.map(node => [node.id, node])); nodes.forEach(n => { adj[n.id] = []; clusterAdj[n.id] = []; }); links.forEach(l => { @@ -761,7 +786,7 @@ const BETWEENNESS_PIVOTS = 220; const BETWEENNESS_BUDGET = 1.5e6; function betweenness(nodes, adj) { - const bc = {}; + const bc = Object.create(null); nodes.forEach(n => { bc[n.id] = 0; }); // Each pivot costs O(V) just to initialise its bookkeeping, so cap pivots by total work // as well as by count: without the budget a 60k-entity store blocks the main thread for @@ -773,7 +798,8 @@ const stride = nodes.length > pivots ? Math.ceil(nodes.length / pivots) : 1; for (let index = 0; index < nodes.length; index += stride) { const src = nodes[index]; - const stack = [], pred = {}, sigma = {}, dist = {}, delta = {}; + const stack = [], pred = Object.create(null), sigma = Object.create(null); + const dist = Object.create(null), delta = Object.create(null); nodes.forEach(n => { pred[n.id] = []; sigma[n.id] = 0; dist[n.id] = -1; delta[n.id] = 0; }); sigma[src.id] = 1; dist[src.id] = 0; const queue = [src.id]; @@ -798,12 +824,18 @@ } /* Bridge edges (Tarjan): removing one disconnects part of the store. */ + function edgeKey(a, b) { + const left = JSON.stringify([typeof a, String(a)]); + const right = JSON.stringify([typeof b, String(b)]); + return left < right ? left + '|' + right : right + '|' + left; + } function findBridges(nodes, links, adj) { - const disc = {}, low = {}, parent = {}, bridges = new Set(); - const multiplicity = {}; + const disc = Object.create(null), low = Object.create(null); + const parent = Object.create(null), bridges = new Set(); + const multiplicity = Object.create(null); links.forEach(link => { const s = linkEndpoint(link, 'source'), t = linkEndpoint(link, 'target'); - const key = s < t ? s + '|' + t : t + '|' + s; + const key = edgeKey(s, t); multiplicity[key] = (multiplicity[key] || 0) + 1; }); let timer = 0; @@ -831,10 +863,9 @@ const p = parent[u]; if (p !== undefined) { low[p] = Math.min(low[p], low[u]); - const key = p < u ? p + '|' + u : u + '|' + p; + const key = edgeKey(p, u); if (low[u] > disc[p] && multiplicity[key] === 1) { - bridges.add(p + '|' + u); - bridges.add(u + '|' + p); + bridges.add(edgeKey(p, u)); } } } @@ -842,7 +873,7 @@ nodes.forEach(n => { if (!disc[n.id]) visit(n.id); }); links.forEach(l => { const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); - l.bridge = bridges.has(s + '|' + t); + l.bridge = bridges.has(edgeKey(s, t)); }); return bridges; } @@ -855,13 +886,14 @@ // Named `styleName`, not `style`: scripts/externalize_dashboard_assets.py scans this // asset for runtime inline-style mutation with a text pattern, and a plain data field // by the shorter name reads as one. The longer name keeps that gate honest. - styleName: 'cyber', colorBy: 'community', palette: 'theme', overrides: {}, themeColors: {}, + styleName: 'cyber', colorBy: 'community', palette: 'theme', + overrides: Object.create(null), themeColors: Object.create(null), settings: Object.assign({}, PRESETS.communities, { mode: 'communities', labels: false, flow: true, frozen: false }), minDegree: 1, showUnlinked: false, focusId: null, depth: 2, layers: { temporal: true, entity: true, causal: true, semantic: true, code: false }, path: null, asOf: null, ghost: true, sizeBy: 'degree', bridges: false, suggestions: false, collapse: 'auto', renderMode: opts.renderMode === 'full' ? 'full' : 'overview' }; - let raw = { nodes: [], links: [], suggestions: [] }, adj = {}, hilite = null, hoverSet = null, maxDeg = 1; + let raw = { nodes: [], links: [], suggestions: [] }, adj = Object.create(null), hilite = null, hoverSet = null, maxDeg = 1; // The classic renderer treats label density as a hard ranked cap, not merely a looser // degree threshold. Keeping chosen IDs outside the paint callback bounds fillText work. let labelIds = new Set(); @@ -885,6 +917,53 @@ let betweennessReady = false; const fg = ForceGraph()(el); const api = {}; + let activeDragNode = null, activeDragLinks = [], dragFollowForce = null; + + /* The ordinary link force is degree-normalised. That is useful for settling a graph, but + it makes a high-degree node a weak anchor during a manual drag: the centre force moves + the rest of the graph while each individual relation barely follows. Keep a separate, + drag-only one-hop force so every directly connected node follows the pointer. It pulls + toward the dragged node's position while preserving the configured link distance. */ + function setActiveDragNode(node) { + activeDragNode = node || null; + if (!activeDragNode) { + activeDragLinks = []; + return; + } + const activeId = activeDragNode.id; + activeDragLinks = (fg.graphData().links || []).filter(link => { + const source = linkEndpoint(link, 'source'), target = linkEndpoint(link, 'target'); + return source === activeId || target === activeId; + }); + } + + function makeDragFollowForce() { + const force = alpha => { + if (!activeDragNode || state.settings.frozen || staticFullLayout) return; + const nodes = fg.graphData().nodes || []; + const byId = new Map(nodes.map(node => [node.id, node])); + const targetDistance = Math.max(8, Number(state.settings.link) || 16); + const strength = 0.28 + Math.min(0.16, (Number(state.settings.gravity) || 0) / 500); + activeDragLinks.forEach(link => { + const source = linkEndpoint(link, 'source'), target = linkEndpoint(link, 'target'); + const otherId = source === activeDragNode.id ? target + : target === activeDragNode.id ? source : null; + const other = otherId == null ? null : byId.get(otherId); + if (!other || other === activeDragNode + || !Number.isFinite(other.x) || !Number.isFinite(other.y) + || !Number.isFinite(activeDragNode.x) || !Number.isFinite(activeDragNode.y)) return; + const dx = activeDragNode.x - other.x, dy = activeDragNode.y - other.y; + const distance = Math.hypot(dx, dy); + const gap = distance - targetDistance; + if (distance < 1e-6 || gap <= 0) return; + const impulse = (gap / distance) * strength * (Number.isFinite(alpha) ? alpha : 1); + other.vx = (other.vx || 0) + dx * impulse; + other.vy = (other.vy || 0) + dy * impulse; + }); + }; + force.initialize = nodes => { force.nodes = nodes; }; + return force; + } function autoFit(duration, padding) { const bbox = fg.getGraphBbox && fg.getGraphBbox(); @@ -959,20 +1038,29 @@ the resolved values through setThemeColors() on every applyTheme()/graphRecolor(); THEME_ETYPE stays only as the standalone-embed fallback for a caller that never does. */ function etypeColor(type) { - if (state.overrides[type]) return state.overrides[type]; - if (state.styleName !== 'classic' && STYLE_PAL[state.styleName] && STYLE_PAL[state.styleName][type]) return STYLE_PAL[state.styleName][type]; - return state.themeColors[type] || THEME_ETYPE[type] || '#8c83e8'; + const override = hasOwn(state.overrides, type) ? state.overrides[type] : null; + if (typeof override === 'string' && override) return override; + const stylePalette = state.styleName !== 'classic' ? STYLE_PAL[state.styleName] : null; + const styled = stylePalette && hasOwn(stylePalette, type) ? stylePalette[type] : null; + if (typeof styled === 'string' && styled) return styled; + const themed = hasOwn(state.themeColors, type) ? state.themeColors[type] : null; + if (typeof themed === 'string' && themed) return themed; + return hasOwn(THEME_ETYPE, type) ? THEME_ETYPE[type] : '#8c83e8'; } function selectedPalette() { - const palette = PALETTES[state.palette]; - return palette ? Object.values(palette) : null; + const palette = hasOwn(PALETTES, state.palette) ? PALETTES[state.palette] : null; + if (!palette) return null; + const values = Object.values(palette).filter(value => typeof value === 'string' && value); + return values.length ? values : null; } /* A palette is a colour family, not merely an entity-type override. Previously the default Community and Connections modes skipped `overrides`, so choosing Aurora, Ocean, Ember, or High contrast changed no pixels unless the user also discovered the separate Entity type selector. Use the selected family in every node-colour mode; Theme retains the active style's deliberately tuned defaults. */ - function commPal() { return selectedPalette() || COMMUNITY_PALS[state.styleName] || COMMUNITY_PALS.classic; } + function commPal() { + return selectedPalette() || COMMUNITY_PALS[state.styleName] || COMMUNITY_PALS.classic; + } function heatColor(node) { const t = (node.rank || 0) / Math.max(1, raw.nodes.length - 1); const colors = selectedPalette() || GRAPH_HEAT; @@ -983,7 +1071,10 @@ if (state.colorBy === 'connections') return heatColor(node); return etypeColor(node.etype); } - function layerColor(layer) { return (STYLE_LAYERS[state.styleName] || STYLE_LAYERS.classic)[layer] || '#8c83e8'; } + function layerColor(layer) { + const layers = STYLE_LAYERS[state.styleName] || STYLE_LAYERS.classic; + return (hasOwn(layers, layer) && layers[layer]) || '#8c83e8'; + } function born(item) { return temporalValue(item, 'valid_from', -Infinity); } function closed(item) { return temporalValue(item, 'valid_to', null); } @@ -993,7 +1084,7 @@ } function collapsedData(nodes, links) { - const groups = {}; + const groups = Object.create(null); nodes.forEach(n => { const c = n.community || 0; if (!groups[c]) groups[c] = { id: 'cluster-' + c, cluster: true, community: c, name: (n.topic || 'Cluster ' + (c + 1)), etype: n.etype, members: 0, degree: 0, betweenness: 0 }; @@ -1002,7 +1093,7 @@ groups[c].betweenness = Math.max(groups[c].betweenness, n.betweenness || 0); }); const cnodes = Object.values(groups); - const seen = {}; + const seen = Object.create(null); const clinks = []; // Indexed lookup, not Array#find per endpoint: auto-collapse fires on every zoom-out, // and the scan made that O(nodes x links) — a visible freeze on a real store. @@ -1023,7 +1114,10 @@ } function visible() { - const keepLayer = l => state.layers[l.layer] !== false; + const keepLayer = l => { + const layers = state.layers; + return !layers || !hasOwn(layers, l.layer) || layers[l.layer] !== false; + }; let nodes = raw.nodes.filter(n => (n.degree > 0 && n.degree >= state.minDegree) || (state.showUnlinked && n.degree === 0)); if (state.repo) { @@ -1082,6 +1176,7 @@ fg.d3Force('y', null); fg.d3Force('radial', null); fg.d3Force('collide', null); + fg.d3Force('dragFollow', null); return; } const s = state.settings, mode = s.mode || 'compact'; @@ -1095,19 +1190,72 @@ link = d3.forceLink().id(node => node.id); fg.d3Force('link', link); } - if (charge && charge.strength) charge.strength(-s.repel); + if (charge && charge.strength) charge.strength(-(mode === 'communities' ? Math.max(10, s.repel * 0.68) : s.repel)); if (link && link.distance) link.distance(s.link); if (typeof d3 === 'undefined') return; + if (!dragFollowForce) { + dragFollowForce = makeDragFollowForce(); + fg.d3Force('dragFollow', dragFollowForce); + } fg.d3Force('radial', null); - /* Community detection still controls colour and link structure, but it must not give - each community a separate orbit target. The default used those scattered targets and - made a connected graph settle as a giant ring around empty space. Every standard - layout now shares the origin as its gravitational centre; repulsion and link distance - retain the useful local separation without sacrificing a coherent overview. */ - const centering = mode === 'radial' ? Math.max(0.04, s.gravity / 300) : s.gravity / 100; - fg.d3Force('x', d3.forceX(0).strength(centering)); - fg.d3Force('y', d3.forceY(0).strength(centering)); - if (mode === 'radial' && d3.forceRadial) fg.d3Force('radial', d3.forceRadial(n => Math.max(0, 5 - Math.min(5, n.degree || 0)) * Math.max(8, s.link * 0.72)).strength(0.32)); + const layoutNodes = fg.graphData().nodes || []; + /* The layout buttons are arrangements, not just five nearby slider presets. Keep the + ordinary force settings as the local texture, then give each named mode its own + geometry so switching modes is visible even when the graph has only one component. + Centering must stay gentle and origin-based: a function target at a distant grid + slot would fight an explicit drag, and a released node must stay where the user + dropped it (the e2e drag-release contract). */ + if (mode === 'communities') { + const communityKeys = [], seenCommunities = new Set(); + layoutNodes.forEach(node => { + const key = Number.isFinite(node.community) ? node.community : 0; + if (!seenCommunities.has(key)) { seenCommunities.add(key); communityKeys.push(key); } + }); + communityKeys.sort((a, b) => a - b); + const columns = Math.max(1, Math.ceil(Math.sqrt(communityKeys.length))); + const rows = Math.max(1, Math.ceil(communityKeys.length / columns)); + const gap = Math.max(180, (Number(s.link) || 16) * 10); + const targets = new Map(); + communityKeys.forEach((key, index) => { + const column = index % columns, row = Math.floor(index / columns); + targets.set(key, { + x: (column - (columns - 1) / 2) * gap, + y: (row - (rows - 1) / 2) * gap * 0.72, + }); + }); + /* A gentle origin-based centering keeps the layout coherent without fighting a + drag; the community grid is still visible through the charge/repel and link + structure installed above. */ + const centering = Math.max(0.04, (Number(s.gravity) || 0) / 100); + fg.d3Force('x', d3.forceX(0).strength(centering)); + fg.d3Force('y', d3.forceY(0).strength(centering)); + } else if (mode === 'radial' && d3.forceRadial) { + const outerRadius = Math.max(180, Math.min(360, Math.sqrt(Math.max(1, layoutNodes.length)) * 18 + (Number(s.link) || 16) * 4)); + const degreeScale = Math.max(1, maxOf(layoutNodes.map(node => node.degree || 0), 1)); + fg.d3Force('x', d3.forceX(0).strength(Math.max(0.05, (Number(s.gravity) || 0) / 500))); + fg.d3Force('y', d3.forceY(0).strength(Math.max(0.05, (Number(s.gravity) || 0) / 500))); + fg.d3Force('radial', d3.forceRadial(node => { + const hubness = Math.max(0, Math.min(1, (node.degree || 0) / degreeScale)); + return 34 + (outerRadius - 34) * (1 - hubness); + }).strength(0.72)); + } else if (mode === 'constellation') { + const positions = new Map(), total = Math.max(1, layoutNodes.length - 1); + const reach = Math.max(160, Math.min(330, 80 + Math.sqrt(Math.max(1, layoutNodes.length)) * 10)); + layoutNodes.forEach((node, index) => { + const rank = Number.isFinite(node.rank) ? node.rank : index; + const fraction = Math.max(0, Math.min(1, rank / total)); + const angle = index * 2.399963229728653; + const radius = 48 + fraction * reach; + positions.set(node.id, { x: Math.cos(angle) * radius * 1.18, y: Math.sin(angle) * radius * 0.76 }); + }); + const target = node => positions.get(node.id) || { x: 0, y: 0 }; + fg.d3Force('x', d3.forceX(node => target(node).x).strength(0.18)); + fg.d3Force('y', d3.forceY(node => target(node).y).strength(0.18)); + } else { + const centering = mode === 'compact' ? Math.max(0.24, (Number(s.gravity) || 0) / 100) : Math.max(0.06, (Number(s.gravity) || 0) / 100); + fg.d3Force('x', d3.forceX(0).strength(centering)); + fg.d3Force('y', d3.forceY(0).strength(centering)); + } /* One collision pass on a large graph, two otherwise — the classic path's `.iterations(GPERF.large?1:2)`. The second pass costs another full quadtree traversal per node on every tick, and a large store pays that on the initial layout and on every @@ -1392,6 +1540,21 @@ if (fg.warmupTicks) fg.warmupTicks(simulate ? (large ? 18 : 40) : 0); } + /* A pointer drag is an active interaction, not a normal layout run. The normal cooldown + protects settled graphs from repainting forever, but it also cuts off link attraction + after roughly two seconds while the user is still holding a node. Keep the simulation + alive until pointer-up, then finishNodeDrag() restores the bounded settling budget. */ + function setDragSimulationBudget(active) { + if (active && !staticFullLayout && !state.settings.frozen) { + if (fg.cooldownTime) fg.cooldownTime(Infinity); + if (fg.cooldownTicks) fg.cooldownTicks(Infinity); + if (fg.warmupTicks) fg.warmupTicks(0); + if (fg.d3AlphaDecay) fg.d3AlphaDecay(0); + return; + } + setSimulationBudget(!state.settings.frozen); + } + function render(fit, reheat) { if (destroyed) return; if (suspended) { @@ -1496,6 +1659,33 @@ if (opts.onNodeClick) opts.onNodeClick(node); } + /* A drag uses fx/fy while the pointer is down. Those anchors are only persistent when the + explicit Freeze control is on; leaving them behind in live mode makes one dragged node + look frozen even though the switch is off. Reheat as soon as a live drag starts too, so + the link force can pull connected nodes along with the pointer instead of waiting for + pointer-up. */ + function reheatLiveLayout(dragging = false) { + if (state.settings.frozen || staticFullLayout) return; + applyForces(); + if (dragging) setDragSimulationBudget(true); + else setSimulationBudget(true); + if (fg.d3AlphaDecay && !dragging) fg.d3AlphaDecay(alphaDecay()); + if (fg.d3ReheatSimulation) fg.d3ReheatSimulation(); + } + + function finishNodeDrag(node) { + setActiveDragNode(null); + if (state.settings.frozen || staticFullLayout) { + node.fx = node.x; + node.fy = node.y; + node.vx = 0; + node.vy = 0; + return; + } + raw.nodes.forEach(item => { item.fx = undefined; item.fy = undefined; }); + reheatLiveLayout(); + } + fg.backgroundColor('rgba(0,0,0,0)').nodeRelSize(1) .enableNodeDrag(false).autoPauseRedraw(true) /* force-graph's default `nodeLabel`/`linkLabel` is the literal accessor "name", and its @@ -1541,14 +1731,19 @@ invalidate(); }) .onNodeClick(handleNodeClick) - // Kept as the pinning contract for embedders that opt back into vendor dragging; + // Kept as the drag contract for embedders that opt back into vendor dragging; // Ledger itself disables that path and uses the scoped pointer controller below. - .onNodeDragEnd(node => { node.fx = node.x; node.fy = node.y; suppressNodeClick(); }) + .onNodeDragEnd(node => { finishNodeDrag(node); suppressNodeClick(); }) .onBackgroundClick(() => { if (opts.onBackgroundClick) opts.onBackgroundClick(); }) .onZoom(z => { zoom = z.k || 1; if (state.collapse !== 'auto') return; - const next = zoom < 0.55; + /* Layout presets can legitimately occupy more of the canvas than the compact default. + Keep auto-collapse for true zoom-out, but do not hide a freshly selected arrangement + merely because its fit scale is below the old, overly eager threshold. */ + const collapseThreshold = state.settings.mode === 'communities' ? 0.22 : 0.42; + const canAutoCollapse = raw.nodes.length > 500; + const next = canAutoCollapse && zoom < collapseThreshold; if (next !== collapsed) { collapsed = next; render(false, true); @@ -1556,9 +1751,19 @@ } }); - /* force-graph's built-in drag always reheats the entire simulation. Ledger treats manual - placement as a pin, so install a small scoped drag controller and leave global physics - changes to the explicit Reheat control. Capturing pointer-down prevents the vendor's + /* Some force-graph releases expose a vendor drag-start callback, while the dashboard's + current bundle does not. Use it only when present; Ledger's manual pointer controller + below remains the canonical path and does not depend on this optional API. */ + if (typeof fg.onNodeDragStart === 'function') { + fg.onNodeDragStart(node => { + setActiveDragNode(node); + reheatLiveLayout(true); + }); + } + + /* force-graph's built-in drag always reheats the entire simulation. Ledger uses a scoped + controller: a frozen graph keeps a deliberate manual pin, while a live graph releases + the temporary drag anchor and reheats. Capturing pointer-down prevents the vendor's drag handler from seeing node gestures while preserving its background pan/zoom path. */ let detachManualDrag = null; if (typeof window !== 'undefined' && typeof window.addEventListener === 'function' @@ -1578,10 +1783,7 @@ window.removeEventListener('pointerup', endManualDrag, true); window.removeEventListener('pointercancel', endManualDrag, true); if (current.dragged) { - current.node.fx = current.node.x; - current.node.fy = current.node.y; - current.node.vx = 0; - current.node.vy = 0; + finishNodeDrag(current.node); suppressNodeClick(); } else if (event.type !== 'pointercancel') { // Our capture listener owns the direct click. Suppress force-graph's @@ -1596,6 +1798,7 @@ if (!point || !Number.isFinite(point.x) || !Number.isFinite(point.y)) return; const dx = event.clientX - manualDrag.startClientX; const dy = event.clientY - manualDrag.startClientY; + let started = false; if (!manualDrag.dragged) { if (Math.hypot(dx, dy) < 3) { event.preventDefault(); @@ -1603,12 +1806,17 @@ return; } manualDrag.dragged = true; + started = true; } const node = manualDrag.node; node.x = node.fx = point.x + manualDrag.offsetX; node.y = node.fy = point.y + manualDrag.offsetY; - node.vx = 0; - node.vy = 0; + node.vx = 0; + node.vy = 0; + if (started) { + setActiveDragNode(node); + reheatLiveLayout(true); + } invalidate(); event.preventDefault(); event.stopPropagation(); @@ -1646,34 +1854,56 @@ window.removeEventListener('pointercancel', endManualDrag, true); }; } - api.setData = data => { + if (destroyed) return; const inputNodes = Array.isArray(data && data.nodes) ? data.nodes : []; - const nodes = inputNodes - .filter(node => node && node.id != null) - .map(node => Object.assign({}, node, { name: nodeName(node) })); - const nodeIds = new Set(nodes.map(node => node.id)); - const links = (Array.isArray(data && (data.links || data.edges)) ? (data.links || data.edges) : []) + const nodes = [], nodeIds = new Set(); + inputNodes.forEach(node => { + if (!node || (typeof node !== 'object' && typeof node !== 'function') + || !validNodeId(node.id) || nodeIds.has(node.id)) return; + nodeIds.add(node.id); + nodes.push(Object.assign({}, node, { name: nodeName(node) })); + }); + const linkInput = Array.isArray(data && data.links) + ? data.links + : (Array.isArray(data && data.edges) ? data.edges : []); + const links = linkInput + .filter(link => link && (typeof link === 'object' || typeof link === 'function')) .map(link => { const source = linkEndpoint(link, 'source'), target = linkEndpoint(link, 'target'); return Object.assign({}, link, { source, target }); }) - .filter(link => link.source != null && link.target != null && nodeIds.has(link.source) && nodeIds.has(link.target)); + .filter(link => link.source != null && link.target != null + && nodeIds.has(link.source) && nodeIds.has(link.target)); const suggestions = (Array.isArray(data && data.suggestions) ? data.suggestions : []) - .map(link => Object.assign({}, link, { source: linkEndpoint(link, 'source'), target: linkEndpoint(link, 'target') })) + .filter(link => link && (typeof link === 'object' || typeof link === 'function')) + .map(link => Object.assign({}, link, { + source: linkEndpoint(link, 'source'), target: linkEndpoint(link, 'target') + })) .filter(link => link.source != null && link.target != null); /* A fresh payload means fresh node objects, so the cached seed is stale even when the ids are identical — force-graph must be re-pointed at the new objects or the render below would style ones nobody is painting from. */ seeded = null; + fullLayoutDirty = true; raw = { nodes, links, suggestions }; adj = communities(raw.nodes, raw.links); - const deg = {}; - raw.links.forEach(l => { const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); deg[s] = (deg[s] || 0) + 1; deg[t] = (deg[t] || 0) + 1; }); + const deg = Object.create(null); + raw.links.forEach(l => { + const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); + deg[s] = (deg[s] || 0) + 1; + deg[t] = (deg[t] || 0) + 1; + }); raw.nodes.forEach(n => { n.degree = deg[n.id] || 0; n.betweenness = 0; }); maxDeg = maxOf(raw.nodes.map(n => n.degree), 1); const ranked = [...raw.nodes].sort((a, b) => b.degree - a.degree); ranked.forEach((n, i) => { n.rank = i; n.hub = i < 6; }); + // A refresh can replace the workspace while a prior focus/highlight still names an old id. + // Drop those references before visible() so the next render cannot isolate an empty view or + // paint a stale hover neighbourhood. + if (state.focusId != null && !nodeIds.has(state.focusId)) state.focusId = null; + if (hilite != null && !nodeIds.has(hilite)) hilite = null; + hoverSet = hilite == null ? null : new Set([hilite].concat(adj[hilite] || [])); // Bridge *edges* are cheap (linear) and feed the stats readout, so they stay eager. // Betweenness is not: see ensureBetweenness. findBridges(raw.nodes, raw.links, adj); @@ -1694,9 +1924,20 @@ render() applies the reduced-motion exemption (`if(layout&&!prefersReducedMotion())`). */ const LAYOUT_KEYS = ['mode', 'repel', 'link', 'gravity', 'size']; api.setSettings = patch => { - if (LAYOUT_KEYS.some(k => patch && patch[k] !== undefined)) fullLayoutDirty = true; - Object.assign(state.settings, patch); - render(false, LAYOUT_KEYS.some(k => patch && patch[k] !== undefined)); + const next = patch && typeof patch === 'object' ? patch : {}; + const wasFrozen = state.settings.frozen === true; + const isUnfreezing = wasFrozen && next.frozen === false; + if (LAYOUT_KEYS.some(k => next[k] !== undefined)) fullLayoutDirty = true; + Object.assign(state.settings, next); + /* Classic synchronises the complete GSET object during a redraw. If the visible switch + was turned off by that sync after an earlier freeze, a plain render restores the + paint settings but leaves d3 at its old alpha/charge state. Route the transition + through the same release path as the visible control so both dashboards resume. */ + if (isUnfreezing) { + api.freeze(false); + return; + } + render(false, LAYOUT_KEYS.some(k => next[k] !== undefined)); }; api.setPreset = name => { const p = PRESETS[name] || PRESETS.compact; @@ -1723,38 +1964,66 @@ fullLayoutDirty = true; render(true, true); }; - api.setColorBy = name => { state.colorBy = name; clearMaterialCache(); refreshColors(); render(false, false); }; + api.setColorBy = name => { + state.colorBy = name; + clearMaterialCache(); + refreshColors(); + render(false, false); + }; api.setPalette = name => { - state.palette = name; - state.overrides = PALETTES[name] ? { ...PALETTES[name] } : {}; + state.palette = typeof name === 'string' ? name : 'theme'; + state.overrides = Object.create(null); + if (hasOwn(PALETTES, state.palette)) Object.assign(state.overrides, PALETTES[state.palette]); clearMaterialCache(); refreshColors(); }; api.setTypeColor = (type, color) => { - state.overrides[type] = color; + if (type == null || typeof color !== 'string') return; + state.overrides[String(type)] = color; state.palette = 'custom'; clearMaterialCache(); refreshColors(); }; /* Rehydrating saved overrides is not a user edit, so it must not flip the palette selector to "custom" behind the user's back the way setTypeColor deliberately does. */ - api.setTypeColors = map => { Object.assign(state.overrides, map || {}); clearMaterialCache(); refreshColors(); }; + api.setTypeColors = map => { + const next = map && typeof map === 'object' ? map : {}; + Object.keys(next).forEach(type => { + if (typeof next[type] === 'string') state.overrides[type] = next[type]; + }); + clearMaterialCache(); + refreshColors(); + }; /* The active theme's resolved `--entity-*` values. Replaced wholesale rather than merged: a theme switch must not leave the previous theme's colour for a type the new one omits. */ api.setThemeColors = map => { - state.themeColors = map && typeof map === 'object' ? { ...map } : {}; + const next = Object.create(null); + if (map && typeof map === 'object') { + Object.keys(map).forEach(key => { + if (typeof map[key] === 'string') next[key] = map[key]; + }); + } + state.themeColors = next; clearMaterialCache(); refreshColors(); }; /* One render for a whole batch of setters — see `batch`. */ - api.apply = (fn, fit, reheat) => { batch(fn, fit, reheat); }; + api.apply = (fn, fit, reheat) => { batch(typeof fn === 'function' ? fn : () => {}, fit, reheat); }; api.setHighlight = id => { hilite = id == null ? null : id; hoverSet = id == null ? null : new Set([id].concat(adj[id] || [])); invalidate(); }; - api.setScope = patch => { Object.assign(state, patch); render(false, true); }; - api.setLayers = layers => { state.layers = layers; render(false, false); }; + api.setScope = patch => { + if (!patch || typeof patch !== 'object') return; + Object.assign(state, patch); + if (!state.layers || typeof state.layers !== 'object') state.layers = {}; + render(false, true); + }; + api.setLayers = layers => { + state.layers = layers && typeof layers === 'object' ? { ...layers } : {}; + render(false, false); + }; /* `focus` remains the explicit neighbourhood-isolation action. It must not schedule a delayed zoom-to-fit: callers that also centre a node otherwise start two competing camera animations, and the late fit wins by dragging the selected entity away. */ @@ -1802,6 +2071,7 @@ if (on) { const charge = fg.d3Force('charge'); if (charge && charge.strength) charge.strength(0); + setSimulationBudget(true); fg.d3AlphaDecay(1); return; } @@ -1884,12 +2154,15 @@ /* The engine clusters its own copies of the nodes, so a caller that renders a cluster legend from the source data would otherwise report a single community. */ api.communityMap = () => { - const map = {}; + const map = Object.create(null); raw.nodes.forEach(n => { map[n.id] = n.community || 0; }); return map; }; - api.setGhosts = on => { state.ghost = on; render(false, false); }; - api.setRepoFilter = repo => { state.repo = (repo || '').trim().toLowerCase(); render(false, true); }; + api.setGhosts = on => { state.ghost = on === true; render(false, false); }; + api.setRepoFilter = repo => { + state.repo = typeof repo === 'string' ? repo.trim().toLowerCase() : ''; + render(false, true); + }; api.setAsOf = date => { state.asOf = asOfValue(date); render(false, true); }; api.setSizeBy = metric => { state.sizeBy = metric === 'betweenness' ? metric : 'degree'; @@ -1919,7 +2192,9 @@ api.setSuggestions = on => { state.suggestions = on; render(false, true); }; api.setCollapse = mode => { state.collapse = state.renderMode === 'full' ? false : mode; - const next = state.renderMode !== 'full' && (mode === true || (mode === 'auto' && zoom < 0.55)); + const collapseThreshold = state.settings.mode === 'communities' ? 0.22 : 0.42; + const canAutoCollapse = raw.nodes.length > 500; + const next = state.renderMode !== 'full' && (mode === true || (mode === 'auto' && canAutoCollapse && zoom < collapseThreshold)); collapsed = next; render(true, true); }; @@ -1942,8 +2217,12 @@ api.destroy = () => { if (destroyed) return; destroyed = true; + running = false; clearTimeout(fitTimer); + fitTimer = 0; cancelFrame(dragClickFrame); + dragClickFrame = 0; + pendingRender = null; try { if (detachManualDrag) { detachManualDrag(); detachManualDrag = null; } if (api._ro) { api._ro.disconnect(); api._ro = null; } @@ -1955,7 +2234,7 @@ el.innerHTML = ''; } catch (e) { /* teardown is best-effort: never let it block a view change */ } raw = { nodes: [], links: [], suggestions: [] }; - adj = {}; + adj = Object.create(null); seeded = null; hilite = null; hoverSet = null; diff --git a/engraphis/dashboard_assets/index.html b/engraphis/dashboard_assets/index.html index dd1a2099..34f6c078 100644 --- a/engraphis/dashboard_assets/index.html +++ b/engraphis/dashboard_assets/index.html @@ -24,11 +24,11 @@ -
+
+