diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index fb3ddd2f..4721c396 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "engraphis-memory", "source": "./", "description": "Discipline for giving agents durable, scoped, explainable memory across sessions and repos with the Engraphis MCP tools.", - "version": "1.5" + "version": "1.6" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c5147742..0bee82ea 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "engraphis-memory", - "version": "1.5", + "version": "1.6", "description": "Give agents durable, scoped, explainable memory across sessions and repos via the Engraphis MCP tools. Use when you learn something worth keeping, need prior context before acting, or ask why/how a fact changed. Covers remember/recall, why/timeline, forget/pin/correct, sessions, and code search.", "author": { "name": "The Engraphis Authors", diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256 index 32b840dc..cbd00041 100644 --- a/.claude-plugin/skill-assets.sha256 +++ b/.claude-plugin/skill-assets.sha256 @@ -1,5 +1,5 @@ -d30ad152dcc4c82ce10e7167fdfe67e709358e5f435293939125f2d6cffc5b7e .claude-plugin/marketplace.json -28dcd15a7a186f8cb8a15705f1bd7734086167991c4acc28ec2cfea59a2374ab .claude-plugin/plugin.json +4c18cdb509babb853ac7e5283ca9b309e2669b82ff098f3acf33238a9e4c1114 .claude-plugin/marketplace.json +94bfa06317a8fe6a6a7e204bb70c5abdc9e4bbc34d79dd6f8447a30140bc8b85 .claude-plugin/plugin.json 055655db84af07561d002f0c69744313d8413c39f3e873f941f0fa0b1e76dc66 skills/engraphis-memory/references/CONVENTIONS.md 62019760766ff472a76a0f81437898f39e3c1fe2631732b7b7733e50c1ad837f skills/engraphis-memory/references/SCOPING.md 4ce83a2768680ec84488a767fc3bd6cd62688d785010a0abd1d4b3edbf14d03a skills/engraphis-memory/references/TOOLS.md diff --git a/AGENTS.md b/AGENTS.md index e077b0a2..163a5231 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ most common mistake here. | Status | Primary scoped, bi-temporal, interface-driven implementation. | Compatibility/reference implementation with flat namespaces. | | Model | Scoped + bi-temporal + typed; interface-driven. | Single flat `namespace` string per memory. | | Code | `engraphis/core/`, `engraphis/backends/`, `eval/`, `tests/`, `scripts/migrate_to_v2.py` | `engraphis/app.py`, `config.py`, `models.py`, `routes/`, `stores/`, `engines/`, `llm/`, `static/` | -| Data | new v2 schema (`SCHEMA_VERSION = 15`) | `engraphis_v1.db` | +| Data | new v2 schema (`SCHEMA_VERSION = 16`) | `engraphis_v1.db` | | Entry | `engraphis.MemoryEngine.create()` / `engraphis.create_memory_engine()` → `engraphis/factory.py` → `core/engine.py` | Internal reference only; never a public launcher | **Rule:** build new capability on **v2** (`core/` + `backends/`) behind the interfaces. @@ -198,19 +198,25 @@ These are pure, unit-tested functions — change them only with a corresponding --- -## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 15`) +## 5. Data model cheat-sheet (`core/interfaces.py`, `core/schema.py` — `SCHEMA_VERSION = 16`) - **Scope hierarchy:** `workspace → repo → session → memory`. Scopes: `session|repo|workspace|user`. - **Bi-temporal validity on every record:** world-time `valid_from/valid_to` + system-time `ingested_at/expired_at`. Reads hide facts outside their validity window unless `include_invalid=True` or an `as_of` anchor is given. - **IDs:** ULID, time-sortable, **typed prefixes** (`ws_`, `repo_`, `ses_`, `mem_`, `ent_`, - `edg_`, `sym_`, `evt_`, `job_`, `aud_`, `dev_`, `rcpt_`) — `core/ids.py`. + `edg_`, `sym_`, `evt_`, `job_`, `aud_`, `dev_`, `rcpt_`, `vlt_`, `src_`) — `core/ids.py`. Lexicographic sort == chronological. - **Tables:** `workspaces`, `repos`, `sessions`, `memories`, `mem_vectors`, `embedding_state`, `mem_fts` (FTS5 + plain-table fallback), `entities`, `edges` (bi-temporal), `mem_links`, `memory_entities`, `symbols`, `code_edges`, `code_files`, `code_memory_links`, - `operation_receipts`, `events`, `audit`, `memory_tombstones`, `schema_migrations`. + `operation_receipts`, `events`, `audit`, `memory_tombstones`, `source_vaults`, + `source_imports`, `source_import_items`, `schema_migrations`. +- **Local document sources:** source collections are scope-bound, resumable manifests. Their + paths, digests, and import state are provenance; source folders never create implicit memory + scopes. `kind="documents"` is the source-neutral adapter, while `kind="obsidian"` retains + the rich Markdown adapter and its existing lineage. Import jobs persist the optional session + target, and schema checks keep source-job lineage and per-job items in that exact session. - **Erasure markers contain no memory content.** `memory_tombstones.export_class` is strictly `never_export|remote_erasure`; only `remote_erasure` may cross a sync boundary. - **Vectors are stored L2-normalized** so cosine similarity == dot product. @@ -250,6 +256,12 @@ These are pure, unit-tested functions — change them only with a corresponding with `engraphis/mcp_server.py`. - **`docs/SYNC.md`** — cloud sync (Pro): architecture, the convergent merge, CLI usage, and the untrusted-bundle security model. +- **`docs/DOCUMENT_IMPORT.md`** — universal local-document import, source safety, re-import, + conflicts, format adapters, and dashboard/CLI flows. Keep it synchronized with the parser, + importer, dashboard, and import-report schema. +- **`docs/OBSIDIAN_IMPORT.md`** — the rich Obsidian Markdown adapter (frontmatter, wikilinks, + aliases, attachments) and its compatibility command. It supplements, rather than replaces, + the universal document-import guide. - **`AGENTS.md`** (this file) + **`CLAUDE.md`** — how to work in the repo. - **`skills/engraphis-memory/`** — portable Agent Skill (SKILL.md + `references/`) that teaches any MCP-capable agent the *memory discipline* (when to remember/recall, scoping, tool selection). diff --git a/CHANGELOG.md b/CHANGELOG.md index 725d3382..f8549389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,35 @@ All notable changes to Engraphis are documented here. Format loosely follows ## [Unreleased] +## [1.6] - 2026-08-08 + +Minor release advancing the v2 engine through schema 16 with deterministic sync state, trusted +local document and Obsidian import, tighter trust boundaries, synchronized agent guidance, and +stronger release and evaluation evidence. + +### Added + +- A dependency-free, source-neutral local document importer for Markdown, plain text, + reStructuredText, HTML, JSON/JSONL, CSV/TSV, configuration/XML text, and stdlib-readable + source code, RTF, DOCX/ODT, XLSX/ODS, PPTX/ODP, and EPUB documents, with existing local + adapters for PDF text, image OCR, and explicitly local-model audio/video transcription. + `engraphis import documents` and the + dashboard’s **Import local documents** flow + provide strict previews, safe per-file reporting, resumable source manifests, temporal + re-import history, and explicit conflict choices. Obsidian remains the rich Markdown adapter. +- Offline, repeatable Obsidian-vault import with strict dry-run previews, source + safety exclusions, resumable per-note progress, temporal re-import history, and + a trusted-owner dashboard wizard that uploads only `.md` note bytes plus content-free + attachment manifests. It ships through + `engraphis import obsidian`, the `engraphis-import` console alias, and a deprecated + v1 seed-script wrapper that maps legacy namespaces to v2 workspaces. + ### Security - Fail closed on new `user`-scope memory writes until records carry an immutable owner identity; preserve historical reads and the existing promotion rejection instead of presenting workspace-bound rows as private personal memory. -- Load optional dotenv configuration only from the owner-private +- Parse bounded dotenv-style configuration without an optional runtime dependency, and load it only from the owner-private `~/.engraphis/config.env` or an absolute owner-private file selected by `ENGRAPHIS_ENV_FILE`; arbitrary working-directory `.env` files are not a trust boundary. - Clarify Cloud Sync credential-origin binding, secret-manager-only unattended credentials, @@ -19,11 +42,30 @@ All notable changes to Engraphis are documented here. Format loosely follows - Advance through schema 15: schema 12 classifies content-free erasure markers so local-only `never_export` markers remain private and only validated `remote_erasure` markers may cross sync boundaries; schema 13 adds per-memory hybrid logical clocks for deterministic - descriptive-state sync; schema 14-15 add the local source-import manifest for document - and note-collection tracking with scope-security triggers. + descriptive-state sync and durable, content-free proof that a memory crossed a sync boundary; + schema 14 adds Obsidian collection and import manifests; schema 15 generalizes them to + source-neutral `documents` and `obsidian` adapters, preserves temporal source lineage, enforces + adapter/job and target-scope integrity, and retains only bounded, content-free per-job + format/result metadata. Schema 16 persists the optional session target on import jobs and + enforces exact session equality for source lineage and job items. +- Bind each trusted-owner dashboard document or Obsidian run to an expiring, owner-session-bound, + one-time preview token over the exact note/document bytes, attachment manifest, target, source, + and conflict policy; invalidate changed client previews and keep job polling and cancellation + bound to the workspace where the job started. +- Make read-only Store inspection write-free for SQLite and injected/SQLCipher connectors: + require injected connectors to expose `open_read_only(path)`, open existing checkpointed files + with `mode=ro&immutable=1` plus `PRAGMA query_only=ON`, and reject missing paths or active + WAL/rollback journals before a connector can create or recover state. ### Fixed +- Publish separately backed vector-index changes for service memory-title edits only after the + canonical Store row, FTS mirror, portable vector, audit, and commit succeed; late Store failures + publish nothing, while post-commit provider failures preserve canonical state and record + content-free repair debt. +- Defer separately backed vector-index upserts and deletes during sync until each canonical apply + batch commits, coalesce repeated IDs, publish nothing on late Store failure, and record + content-free repair debt if the provider fails after commit. - Synchronize the portable memory skill with the live Smart nine-tool and Classic 34-tool surfaces, including the two intentionally narrower Smart overlap schemas, trust/origin fields, planner and response bounds, context-savings filters, receipt anchors, and expanded health @@ -47,6 +89,24 @@ All notable changes to Engraphis are documented here. Format loosely follows including clean-checkout completion receipts, exact source-question coverage, privacy-safe export binding, matched `context_k=2` comparators, and memory-type count evidence. +### Added + +- Dashboard Settings panel and startup banner now display the running Engraphis + version, fetched from the existing `/api/info` endpoint. + +### Fixed + +- Wrap `engraphis_get_memory` post-inspect body in error-redaction try/except + matching all other Smart gateway tools, preventing internal SQL errors and + file paths from leaking through FastMCP error responses. +- Fix malformed SQLite URI on Windows in `_keyword_search` and `/api/memories` + fallback paths: use `Path.resolve().as_uri()` instead of bare string + interpolation, matching the store's URI construction. +- Apply `_graph_csv()` limit enforcement to the `/graph` endpoint's `layers` + parameter, matching all other graph endpoints. +- Log a warning when `ENGRAPHIS_LLM_EXTRA_HEADERS` contains invalid JSON + instead of silently dropping the headers. + ## [1.5] - 2026-08-04 Minor release advancing the v2 engine to schema 11 with governed recall recovery, diff --git a/MANIFEST.in b/MANIFEST.in index 050ecbc3..1086ba48 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,6 +10,7 @@ recursive-include engraphis/dashboard_assets/vendor * include engraphis/commercial_manifest.json include LICENSE NOTICE README.md CHANGELOG.md BENCHMARKS.md include docs/RECALL_RECOVERY.md +include docs/DOCUMENT_IMPORT.md docs/OBSIDIAN_IMPORT.md include docs/images/context-efficiency.svg include pyproject.toml include .env.example requirements.txt diff --git a/README.md b/README.md index f114523e..b7b63d25 100644 --- a/README.md +++ b/README.md @@ -164,15 +164,19 @@ selection, set `ENGRAPHIS_UPDATE_EXTRAS` to a comma-separated list (for example > **Upgrading to 1.5:** schema 10 bounds legacy retention state and schema 11 backfills explicit > approval only for eligible pre-review local memories. Pending and quarantined evidence remains > gated. Existing 1.4.x databases migrate automatically when Engraphis 1.5 opens them; see the -> [1.5 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#150---2026-08-04). - -> **Current source:** schema 15 adds the local source-import manifest (vaults, imports, -> job items) for document and note-collection tracking. Schema 14 introduced the -> initial note-collection manifest; schema 15 generalized it to support multiple source -> kinds with content-free scope-security triggers. -> Schema 12 classifies content-free erasure markers before sync: existing markers migrate to -> local-only `never_export`; new secure erasures become `remote_erasure` only for non-secret -> `workspace`/`repo` records that were already eligible for sharing. +> [1.5 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#15---2026-08-04). + +> **Upgrading to 1.6:** existing 1.5 databases migrate automatically through schema 12, which +> classifies content-free erasure markers before sync: existing markers become local-only +> `never_export`, while new secure erasures become `remote_erasure` only for non-secret +> `workspace`/`repo` records already eligible for sharing. Schema 13 adds per-memory hybrid +> logical clocks for deterministic descriptive-state sync and durable, content-free proof that a +> memory crossed a sync boundary. Schema 14 adds the Obsidian collection and import manifests; +> schema 15 generalizes them to source-neutral local documents, preserves temporal source lineage +> across re-imports, binds adapters and target scopes, and retains only bounded, content-free +> per-job format/result metadata. The schema 16 migration persists each import job's optional session target +> and requires source lineage and job-item attachments to remain in that exact session. See the +> [1.6 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#16---2026-08-08). --- @@ -638,11 +642,36 @@ oversized key files rather than following an unexpected filesystem object. ## Import files and folders -Import supported documents and code through the dashboard, a local folder, or MCP. Optional -extractors add offline chunking, structured LLM extraction, document OCR, transcription, and -PostgreSQL schema ingestion. See the [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md), -[architecture guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md), and [security policy](https://github.com/Coding-Dev-Tools/engraphis/blob/main/SECURITY.md) for formats, -configuration, and local-resource safeguards. +The dependency-free universal core scans Markdown, plain text, RST, HTML, JSON/JSONL, CSV/TSV, +configuration/XML text, source code, RTF, DOCX/ODT, XLSX/ODS, PPTX/ODP, and EPUB into the normal +v2 memory path. Installed local resource adapters add PDF text, image OCR, and explicitly +local-model audio/video transcription. +Start with a zero-write +preview, then confirm the same source collection explicitly: + +```bash +engraphis import documents /path/to/collection --workspace acme --dry-run +engraphis import documents /path/to/collection --workspace acme --repo product --yes +``` + +The CLI never downloads an embedding model during import. Use a model that is already cached, +set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path`, or explicitly set +`ENGRAPHIS_EMBED_MODEL` to an empty value to use dependency-free deterministic hashing in +lexical degraded mode. + +The dashboard’s **Import local documents** flow offers the same preview, target scope, source +label, conflict policy, cancellation, and resumable progress. Re-imports are idempotent, +preserve temporal history, and report source removals without hard-deleting memories. Obsidian +remains the rich Markdown adapter for frontmatter, aliases, wikilinks, and attachment references: + +```bash +engraphis import obsidian /path/to/vault --workspace acme --dry-run +``` + +See the [document import guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCUMENT_IMPORT.md) +for supported formats, source safety, resume and conflict behavior, optional adapters, and +limitations; see the [Obsidian adapter guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/OBSIDIAN_IMPORT.md) +for Markdown-specific behavior. --- @@ -663,7 +692,7 @@ file. It never searches the working directory for `.env`, and explicit process v | Env Var | Default | Description | |---------|---------|-------------| -| `ENGRAPHIS_ENV_FILE` | `~/.engraphis/config.env` | Optional trusted config leaf selected before dotenv values load. An explicit value must be an absolute path to an owner-private regular file; arbitrary working-directory `.env` files are ignored. | +| `ENGRAPHIS_ENV_FILE` | `~/.engraphis/config.env` | Optional trusted config leaf selected before trusted values load. Its bounded dependency-free parser performs no interpolation. An explicit value must be an absolute path to an owner-private regular file; arbitrary working-directory `.env` files are ignored. | | `ENGRAPHIS_DB_PATH` | Source: `/engraphis.db`; installed: platform user-data directory | SQLite database file. Installed defaults are `%LOCALAPPDATA%\engraphis\engraphis.db` (Windows), `~/Library/Application Support/engraphis/engraphis.db` (macOS), and `$XDG_DATA_HOME/engraphis/engraphis.db` or `~/.local/share/engraphis/engraphis.db` (Linux). The environment variable overrides every default. | | `ENGRAPHIS_HOST` | `127.0.0.1` | Server bind address | | `ENGRAPHIS_PORT` | `8700` | Dashboard port | diff --git a/docs/DOCUMENT_IMPORT.md b/docs/DOCUMENT_IMPORT.md new file mode 100644 index 00000000..c3450215 --- /dev/null +++ b/docs/DOCUMENT_IMPORT.md @@ -0,0 +1,148 @@ +# Import local documents + +Engraphis imports local document collections into the v2 memory engine without uploading the +source. The dependency-free parser describes readable source material; the normal memory write +path then stores scoped, temporal memories and derives local search indexes. A source folder is +provenance, never an implicit `workspace`, `repo`, or `session` scope. + +Obsidian is a rich Markdown adapter within this architecture. Use +[the Obsidian adapter guide](OBSIDIAN_IMPORT.md) when frontmatter, aliases, wikilinks, block or +heading fragments, and attachment references matter. + +## CLI + +Preview is strict and creates no database records: + +```bash +engraphis import documents /path/to/collection --workspace acme --dry-run +``` + +After reviewing the preview, explicitly confirm the trusted-local write: + +```bash +engraphis import documents /path/to/collection --workspace acme --repo product --yes +``` + +Choose a session target when needed, or reuse an existing source collection identity: + +```bash +engraphis import documents /path/to/collection --workspace acme --repo product \ + --session ses_01EXAMPLE --scope session --source-id vlt_01EXAMPLE --yes +``` + +`engraphis-import documents ...` is the equivalent installed console entrypoint. Non-interactive +and JSON-mode writes require `--yes`; an interactive terminal asks for confirmation. Use `--db` +to select another v2 database, `--source-label` for a new collection’s display name, and +`--on-conflict error|replace|new` to choose the divergent-lineage policy. `--limit N` pauses a +write after N documents and leaves it resumable; a dry run always previews the whole collection. + +CLI imports never download an embedding model. The configured `ENGRAPHIS_EMBED_MODEL` must +already be cached or use a `local:/absolute/path` selector. To deliberately use Engraphis's +dependency-free deterministic hashing embedder instead, set `ENGRAPHIS_EMBED_MODEL` to an empty +value for the import process; recall will then expose lexical degraded mode rather than claiming +semantic-vector support. + +## Dashboard + +Open the dashboard and choose **Import local documents**. Select **Documents** mode, choose files +or a folder, then set the source label, workspace, optional repository/session, scope, memory +type, and conflict policy. **Preview import** performs zero writes and shows every candidate, +format, warning, skip, rejection, update, rename, conflict, and missing source. Check the local +document confirmation and choose **Import documents** only after reviewing that report. + +A new browser source requires a nonblank Source label; folder selection prefills its root folder +name. This label is part of the local source identity, so unrelated collections cannot silently +share a re-import lineage. Select a saved source when resuming or re-importing that collection. + +The browser processes selected bytes locally and does not retain a dashboard upload copy. The +trusted dashboard flow requires its local owner-browser/CSRF boundary and an +`ENGRAPHIS_API_TOKEN`; in zero-token loopback mode, use the CLI. + +## Supported formats + +The built-in parser is intentionally small and uses only the Python standard library: + +| Format | Extensions | Preserved/readable structure | +|---|---|---| +| Markdown | `.md`, `.markdown`, `.mdown` | Canonical Markdown; the Obsidian adapter adds frontmatter and note-link metadata. | +| Plain text | `.txt`, `.text`, `.log` | Canonical text. | +| reStructuredText | `.rst`, `.rest` | Canonical text and simple underline headings. | +| HTML | `.html`, `.htm`, `.xhtml` | Original HTML plus readable text; scripts, styles, templates, and noscript content are excluded from readable text. | +| JSON | `.json`, `.jsonl`, `.ndjson` | Structured, readable JSON/JSON Lines where valid; malformed JSON is preserved as text with a warning. | +| CSV/TSV tables | `.csv`, `.tsv`, `.tab` | Original table text with bounded header/row metadata. | +| Word-processing documents (DOCX, ODT, RTF) | `.docx`, `.odt`, `.rtf` | Readable paragraphs/text via bounded ZIP/XML or conservative RTF parsing; rich layout, comments, and tracked changes are not reproduced. | +| Spreadsheets (XLSX, ODS) | `.xlsx`, `.ods` | Bounded worksheet/cell values as readable rows; formulas are never executed and workbook layout is not reproduced. | +| Presentations (PPTX, ODP) | `.pptx`, `.odp` | Bounded slide text in slide order; animations, speaker media, and visual layout are not reproduced. | +| EPUB | `.epub` | Readable spine/chapter text via bounded ZIP/XML/HTML parsing. | +| Configuration/XML text | `.yaml`, `.yml`, `.toml`, `.ini`, `.cfg`, `.xml` | Readable source text; credentials and secret-like values are rejected. | +| Source code | `.py`, `.pyi`, `.js`, `.jsx`, `.mjs`, `.cjs`, `.ts`, `.tsx`, `.go`, `.rs`, `.java`, `.cs`, `.c`, `.h`, `.cc`, `.cpp`, `.cxx`, `.hpp`, `.hh`, `.hxx`, `.sql`, `.tf`, `.tfvars`, `.hcl`, `.sh`, `.ps1`, `.rb`, `.php`, `.swift`, `.kt`, `.kts`, `.scala`, `.lua`, `.r`, `.css` | Canonical source text. Link, tag, and heading discovery is disabled for source code so examples do not create graph relationships. | + +The outer v2 importer also reuses Engraphis's existing local resource adapters. With +`engraphis[documents]`, PDF text and image OCR are available; OCR additionally needs the local +Tesseract executable. Audio/video transcription needs `engraphis[transcription]` and +`ENGRAPHIS_WHISPER_MODEL` must point to an existing local model file or directory. The importer +refuses a model name or missing path so this workflow cannot trigger a model download. + +The parser retains original readable content or structure and source metadata. It masks fenced and +inline code before generic tag/link discovery, so examples and snippets cannot manufacture source +relationships. It never executes HTML, Markdown plugins, scripts, macros, or embedded content. + +## Safety and privacy + +Only regular files below the selected root are considered. The scanner rejects a symlinked root, +skips symlinks and hidden/configuration paths, rejects common credential/key filenames and +secret-like contents, bounds individual documents and the whole collection, and verifies a file +did not change while it was read. ZIP containers have member-count, decompressed-size, +compression-ratio, path, and DTD/entity protections. Unsafe, malformed, unreadable, oversized, +binary, or unsupported files are catalogued per file; one failure does not stop the rest of the +collection. Reports never echo secret-like source content. + +Default filename exclusions include `.env` variants, credentials, secrets, tokens, recovery +codes, SSH identity files, and `.pem`, `.key`, `.p12`, and `.pfx` material. A collection is +bounded to 10,000 encountered files and 250 MB of read bytes; an individual adapter input is +bounded to 100 MB, while canonical memory text is capped at 100,000 characters and is rejected +rather than silently split. Containers are additionally capped at 2,000 members and 20 MB of +declared decompressed content. Invalid UTF-8/UTF-16 in permitted text is replaced explicitly and +reported as a parsing warning. + +Markdown uses the Obsidian adapter's stricter 2 MB raw-note limit before decoding. + +Review a preview before importing sensitive material. The local importer makes no network request +and does not copy source folders or attachments into a hidden second collection. + +CLI dry runs inspect an existing plaintext or configured SQLCipher manifest through an immutable, +query-only connection. They tolerate a pre-importer database as an empty manifest and refuse an +active uncheckpointed WAL instead of creating or consulting writable sidecars. + +## Re-import, history, and conflicts + +Each collection has a stable local source identity plus per-document path and content identity. +The same collection and target scope re-import idempotently: unchanged documents are not +duplicated, changed documents follow the selected conflict policy, and unique exact-content moves +can be reported as renames. Source files missing on a later scan are reported; their memories are +not hard-deleted. + +`replace` creates a temporal successor under the normal v2 rules, `new` creates a distinct memory, +and the default `error` reports a divergent lineage without silently overwriting it. Progress is +recorded per document. Rerun an interrupted or `--limit`-paused command with the same root and +target to resume safely; the final report includes imported, updated, renamed, skipped, rejected, +conflict, missing, warning, and error counts. + +## Optional adapters and limits + +The universal core parser deliberately has no hard dependency on OCR, PDF decoding, +audio/video transcription, office applications, or a hosted model. The outer importer invokes +only installed local adapters for PDF/OCR/transcription and reports a per-file rejection when an +adapter or local executable/model is unavailable. Legacy OLE Office files (`.doc`, `.xls`, +`.ppt`), encrypted or DRM-protected documents, unknown containers, and arbitrary binary files +are never guessed or decoded as text. A PDF adapter extracts embedded PDF text; it does not OCR +scanned PDF pages. Import page images separately when local OCR is required. + +The parser does not render HTML/CSS, execute JavaScript or macros, interpret every RST directive, +evaluate spreadsheet formulas, reproduce presentation/EPUB layout, or preserve every +rich-document annotation. YAML and TOML are preserved safely but are not fully interpreted; RTF, +spreadsheets, presentations, and EPUB are readable-text imports rather than editing-fidelity +conversions. Unsupported files remain explicit per-file report entries and can be added later +through the bounded adapter interface; “universal” never means guessing arbitrary binary bytes. +For Obsidian-specific unsupported syntax and attachment behavior, see +[the Obsidian adapter guide](OBSIDIAN_IMPORT.md). diff --git a/docs/OBSIDIAN_IMPORT.md b/docs/OBSIDIAN_IMPORT.md new file mode 100644 index 00000000..8e398d84 --- /dev/null +++ b/docs/OBSIDIAN_IMPORT.md @@ -0,0 +1,123 @@ +# Import an Obsidian vault + +Obsidian is Engraphis’s rich Markdown adapter inside the universal local-document +import architecture. Use the source-neutral [document import guide](DOCUMENT_IMPORT.md) +for the shared safety, privacy, resume, history, conflict, dashboard, and supported-format +contract. This guide covers what the Obsidian adapter adds: frontmatter, aliases, tags, +wikilinks, block/heading fragments, and attachment references. + +The source Markdown remains your canonical data; Engraphis stores normal readable memory +records and derives embeddings, full-text search, and graph indexes from them. The importer +makes no network requests. + +## Quick start + +Preview an import before opening or changing the target database: + +```bash +engraphis import obsidian /path/to/vault --dry-run +``` + +After reviewing the preview, confirm an unattended import explicitly: + +```bash +engraphis import obsidian /path/to/vault --workspace acme --yes +``` + +Choose the existing target scope explicitly when it is not supplied by the +active session: + +```bash +engraphis import obsidian /path/to/vault --workspace acme --repo product \ + --session ses_01EXAMPLE --scope session --yes +``` + +`engraphis-import obsidian ...` is an equivalent installed console entrypoint. +Unattended and JSON-mode writes require `--yes`; an interactive terminal asks for +confirmation. An unattended preview must also name `--workspace`. Use `--db` to +select a database other than the configured v2 database. + +The dashboard offers the same flow through **Import local documents**: select **Obsidian vault** +as the import mode, select the workspace/repository/session target, review the preview, then +start the import. The preview is strict: it performs zero database writes. +It shows discovered Markdown files, folders, tags, aliases, links, attachments, +warnings, and the files classified as new, changed, unchanged, skipped, or +rejected. + +A new browser vault requires a nonblank source label; folder selection prefills its root folder +name. Select the saved source identity when resuming or re-importing that vault. + +The trusted dashboard wizard uses the existing owner browser-session and CSRF +confirmation boundary, so the local dashboard must have `ENGRAPHIS_API_TOKEN` +configured. In zero-token loopback mode, use the CLI importer instead. + +## What is imported + +Markdown is read recursively and retains its relative vault path, original +title, readable Markdown body, YAML frontmatter metadata, aliases, tags, +headings, common date fields, and source identity. Wikilinks such as +`[[Note]]`, `[[Note|label]]`, heading/block fragments, and `![[embed]]` are +discovered for graph linking. Referenced attachments are catalogued, not copied +into a hidden second vault. Fenced and inline code are retained in the note body +but ignored when discovering links, tags, and attachments. + +The import target follows Engraphis's existing scope hierarchy; vault folders +are source metadata, not invented scopes. Imported notes use the normal memory +write path and its normal indexing behavior. + +## Privacy and filesystem safety + +The shared source-neutral safety and privacy contract is maintained in the +[document import guide](DOCUMENT_IMPORT.md). The Obsidian-specific exclusions below are in +addition to that contract. + +Only files below the selected vault are considered. The importer does not follow +symlinks, skips hidden/VCS/configuration paths (including `.obsidian`), and +rejects common sensitive filenames and note contents that look like credentials +or private keys. It never prints secret-like contents in reports. Unsupported, +malformed, unreadable, or oversized files are reported per file and do not stop +a vault import. Invalid UTF-8 is replaced explicitly and reported as a warning. + +Review the dry-run report before importing a vault that contains personal or +work-sensitive material. The importer does not upload vault data or make API +calls. + +## Re-import, history, and recovery + +Each source file has a stable identity based on the local vault identity and its +relative path, plus recorded content hashes and importer version. A later run +skips unchanged notes, imports new notes, identifies changed and renamed notes, +and reports deleted source files. Deleted files never trigger automatic hard +deletion of Engraphis memories. + +Changed notes preserve Engraphis's temporal history rather than silently +overwriting it. When an existing target has a conflict, choose an explicit +conflict option in the CLI or dashboard to replace according to temporal rules +or create a distinct memory. The default is non-destructive reporting. + +Progress is recorded per note. If an import is interrupted, run the same command +again to resume safely; completed, unchanged source records are not duplicated. +The final report includes counts and paths for imported, updated, skipped, +rejected, conflicts, and warnings. + +The deprecated `python -m scripts.seed_from_obsidian` command remains available +for old local automation. Its `--namespace NAME` option maps directly to +`--workspace NAME`, and invoking the historical write command counts as explicit +local confirmation. Its `--limit N` option processes at most N notes and leaves a +resumable partial run; rerun without `--limit` to finish link reconciliation and +the missing-source report. The primary importer accepts the same compatibility +option, but a dry run always previews the entire vault because it processes no +notes. + +Exit status is `0` for a completed import or clean preview, `2` for invalid input, +`3` for a partial/conflicted/limit-paused run, and `130` for an operator cancellation. + +## Current limitations + +The frontmatter reader intentionally supports common top-level scalar and list +forms, not every YAML feature. Obsidian plugin-specific syntax (Dataview, +Canvas, queries, templates, and executable/plugin content) is not interpreted. +Attachments are referenced but not copied or OCRed; unresolved links remain +unresolved until a matching note is present. The importer preserves Markdown, +but does not attempt to reproduce Obsidian's rendered/transclusion behavior. +Automatic rename detection is limited to unique exact-content matches. diff --git a/engraphis/__init__.py b/engraphis/__init__.py index d712281c..771ccf9d 100644 --- a/engraphis/__init__.py +++ b/engraphis/__init__.py @@ -2,7 +2,7 @@ from importlib.metadata import PackageNotFoundError, version as _dist_version -_SOURCE_VERSION = "1.5" +_SOURCE_VERSION = "1.6" try: __version__ = _dist_version("engraphis") @@ -11,10 +11,10 @@ # the prior MCP contract merely because metadata has not been refreshed yet. if __version__ != _SOURCE_VERSION: __version__ = _SOURCE_VERSION -except PackageNotFoundError: # source tree without an installed distribution - # Keep in step with [project] version in pyproject.toml — tests/test_packaging.py - # pins the two together so a release cannot ship them out of sync. - __version__ = "1.5" +except PackageNotFoundError: # source tree without an installed distribution + # Keep in step with [project] version in pyproject.toml — tests/test_packaging.py + # pins the two together so a release cannot ship them out of sync. + __version__ = "1.6" def _default_memory_engine_factory(**kwargs): diff --git a/engraphis/backends/encrypted_db.py b/engraphis/backends/encrypted_db.py index c7b111ab..b34984fb 100644 --- a/engraphis/backends/encrypted_db.py +++ b/engraphis/backends/encrypted_db.py @@ -20,7 +20,7 @@ import re import sqlite3 from pathlib import Path -from typing import Callable, Optional +from typing import Optional from engraphis.private_state import read_private_text @@ -185,10 +185,83 @@ def __exit__(self, *exc): return _guard(self._raw.__exit__, *exc) -def make_connector(key: str) -> Callable[[str], object]: - """Return a ``connect(path) -> connection`` factory that opens *path* as an encrypted - SQLCipher database keyed with *key*. Raises :class:`EncryptionError` with an actionable - message if the driver is missing or the key does not unlock an existing file.""" +class _EncryptedConnector: + """SQLCipher connector with distinct writable and immutable-open entry points. + + ``__call__`` preserves the historical writable connector behavior. Store's + explicit read-only connector contract uses ``open_read_only``; that path never + creates parent directories and requires SQLCipher's SQLite URI support for + ``mode=ro&immutable=1`` rather than opening writable and setting query-only late. + """ + + def __init__(self, driver, pragma: str) -> None: + self._driver = driver + self._pragma = pragma + + def __call__(self, path: str): + if path != ":memory:": + Path(path).parent.mkdir(parents=True, exist_ok=True) + return self._open(path, uri=False, read_only=False) + + def open_read_only(self, path: str): + try: + target = Path(path).resolve(strict=True).as_uri() + "?mode=ro&immutable=1" + except OSError: + raise EncryptionError( + "could not initialize the encrypted database connection" + ) from None + return self._open(target, uri=True, read_only=True) + + def _open(self, target: str, *, uri: bool, read_only: bool): + options = {"timeout": 30, "check_same_thread": False} + if uri: + options["uri"] = True + try: + raw = self._driver.connect(target, **options) + except Exception: # noqa: BLE001 + raise EncryptionError( + "could not initialize the encrypted database connection" + ) from None + try: + raw.execute(self._pragma) # MUST be the first statement + except Exception: # noqa: BLE001 + try: + raw.close() + except Exception: # noqa: BLE001 + pass + # Suppress the driver message (`from None`): a PRAGMA syntax error can echo the + # statement text, which contains the key. Never surface key material. + raise EncryptionError( + "failed to apply the database key — check the ENGRAPHIS_DB_KEY format" + ) from None + try: + if read_only: + # Defense in depth after the immutable URI has already constrained + # the open itself. This PRAGMA is connection-local and non-persistent. + raw.execute("PRAGMA query_only=ON") + # Touch the header so a wrong key / plaintext-vs-encrypted mismatch fails now, + # with a clear message, instead of deep inside an unrelated query later. + raw.execute("SELECT count(*) FROM sqlite_master").fetchone() + except Exception: # noqa: BLE001 + try: + raw.close() + except Exception: # noqa: BLE001 + pass + raise EncryptionError( + "could not open the encrypted database — wrong ENGRAPHIS_DB_KEY, or " + "the file is not SQLCipher-encrypted (an existing plaintext DB cannot be " + "opened with a key; migrate it first)." + ) from None + raw.row_factory = self._driver.Row + return _TranslatingConnection(raw) + + +def make_connector(key: str) -> _EncryptedConnector: + """Return a SQLCipher connector with writable and explicit read-only opens. + + Raises :class:`EncryptionError` with an actionable message if the driver is + missing or the key does not unlock an existing file. + """ try: sqlcipher3 = importlib.import_module("sqlcipher3") except Exception: # noqa: BLE001 @@ -239,14 +312,16 @@ def _connect(path: str): raw.row_factory = sqlcipher3.Row return _TranslatingConnection(raw) - return _connect + return _EncryptedConnector(sqlcipher3, pragma) -def connector_from_env() -> Optional[Callable[[str], object]]: - """The connection factory for the current environment, or None when encryption is off. +def connector_from_env() -> Optional[_EncryptedConnector]: + """The dual-mode connector for this environment, or None when encryption is off. Callers pass the result to ``Store(path, connect=...)`` / ``MemoryEngine.create`` / - ``MemoryService.create``. None means "use the stdlib sqlite3 default" (plaintext).""" + ``MemoryService.create``. Writable calls use ``connector(path)``; read-only Store + construction uses its explicit ``connector.open_read_only(path)`` contract. None + means "use the stdlib sqlite3 default" (plaintext).""" key = _resolve_key() if key is None: return None diff --git a/engraphis/backends/extractor.py b/engraphis/backends/extractor.py index 2bf85927..04921410 100644 --- a/engraphis/backends/extractor.py +++ b/engraphis/backends/extractor.py @@ -765,7 +765,7 @@ def _load_chunk_token_counter( require_immutable_models: Optional[bool] = None, ) -> tuple[Callable[[str], int], str]: """Load an explicitly configured Hugging Face tokenizer at the backend edge.""" - from engraphis.backends.model_source import validate_model_source + from engraphis.backends.model_source import is_local_model_source, validate_model_source validate_model_source( model, @@ -779,15 +779,23 @@ def _load_chunk_token_counter( raise RuntimeError( "ENGRAPHIS_CHUNK_TOKENIZER_MODEL requires the optional transformers package" ) from exc + raw_model = str(model or "").strip() + has_local_prefix = raw_model.startswith("local:") + local_files_only = is_local_model_source(raw_model) + resolved_model = raw_model[len("local:"):].strip() if has_local_prefix else raw_model + if not resolved_model: + raise ValueError("local chunk tokenizer selector requires a path or cached model name") kwargs: dict[str, Any] = {"trust_remote_code": False} if revision: kwargs["revision"] = revision - tokenizer = AutoTokenizer.from_pretrained(model, **kwargs) + if local_files_only: + kwargs["local_files_only"] = True + tokenizer = AutoTokenizer.from_pretrained(resolved_model, **kwargs) def count(text: str) -> int: return len(tokenizer.encode(text or "", add_special_tokens=False)) - identity = f"hf:{model}@{revision or 'unversioned'}" + identity = f"hf:{resolved_model}@{revision or 'unversioned'}" count.identity = identity # type: ignore[attr-defined] return count, identity diff --git a/engraphis/backends/sync_relay.py b/engraphis/backends/sync_relay.py index 45940b42..8080b3e2 100644 --- a/engraphis/backends/sync_relay.py +++ b/engraphis/backends/sync_relay.py @@ -153,6 +153,19 @@ def _saved_sync_token(relay_origin: str) -> str: "configured relay credential is malformed; replace or unset it", status=409, ) from None + configured_origin = os.environ.get("ENGRAPHIS_SYNC_TOKEN_ORIGIN", "") + try: + configured_origin = _validated_base_url(configured_origin) + except ValueError: + raise RelayError( + "configured relay credential has no valid relay binding", + status=409, + ) from None + if configured_origin != relay_origin: + raise RelayError( + "configured relay credential belongs to another relay", + status=409, + ) return configured try: raw = read_private_text( diff --git a/engraphis/classic_assets/dashboard.css b/engraphis/classic_assets/dashboard.css index 5b472f4d..6e85bc52 100644 --- a/engraphis/classic_assets/dashboard.css +++ b/engraphis/classic_assets/dashboard.css @@ -757,3 +757,7 @@ progress.graph-degree[data-graph-node-type="person_or_concept"]::-webkit-progres #graph-net.engraphis-graph-node-hover{cursor:pointer} #graph-net:not(.engraphis-graph-node-hover){cursor:grab} .savings-hero{display:flex;align-items:flex-end;justify-content:space-between;gap:16px;margin:8px 0}.savings-number{margin:0;font-variant-numeric:tabular-nums}.savings-unit{color:var(--text-dim);font-size:12px}.savings-rate{display:flex;flex-direction:column;align-items:flex-end;gap:2px;text-align:right}.savings-rate strong{color:var(--green);font-size:20px;line-height:1;font-variant-numeric:tabular-nums}.savings-rate span{color:var(--text-dim);font-size:11px}.savings-progress{display:block;width:100%;height:7px;margin:0 0 8px;appearance:none;border:0;border-radius:999px;background:var(--surface2)}.savings-progress::-webkit-progress-bar{border-radius:999px;background:var(--surface2)}.savings-progress::-webkit-progress-value{border-radius:999px;background:var(--green)}.savings-progress::-moz-progress-bar{border-radius:999px;background:var(--green)}.savings-summary{margin:0;color:var(--text-muted);font-size:12px} +.skip-link{position:fixed;top:8px;left:8px;z-index:1000;padding:8px 12px;border-radius:4px;background:var(--accent);color:var(--bg);transform:translateY(-150%)}.skip-link:focus{transform:translateY(0)} +#ov-savings .savings-hero{flex-wrap:wrap} +#ov-savings .cfg-row>span:last-child{display:flex;flex-wrap:wrap;justify-content:flex-end;gap:6px} +@media(max-width:520px){#ov-savings .savings-hero{align-items:flex-start;flex-direction:column;gap:8px}#ov-savings .savings-rate{align-items:flex-start;text-align:left}#ov-savings .cfg-row{align-items:flex-start;flex-direction:column}#ov-savings .cfg-row>span:last-child{justify-content:flex-start}} diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js index d1333fff..76dd4ad0 100644 --- a/engraphis/classic_assets/dashboard.js +++ b/engraphis/classic_assets/dashboard.js @@ -427,7 +427,7 @@ async function loadReceipts(){const el=document.getElementById('audit-body');el. async function downloadReceipts(){try{const d=await api('/receipts/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-receipts-'+(WS||'workspace')+'.json';a.click();URL.revokeObjectURL(a.href);toast('Privacy-safe receipts exported','ok')}catch(e){toast(e.message,'err')}} let SAVINGS_PRESET='all'; -function savingsPresetQuery(){const p=new URLSearchParams({workspace:WS||''});if(SAVINGS_PRESET==='current')p.set('release_version','1.5.0');if(SAVINGS_PRESET==='7d')p.set('from_ts',String(Date.now()/1000-604800));return p.toString()} +function savingsPresetQuery(){const p=new URLSearchParams({workspace:WS||''});if(SAVINGS_PRESET==='current')p.set('release_version','1.6');if(SAVINGS_PRESET==='7d')p.set('from_ts',String(Date.now()/1000-604800));return p.toString()} function renderSavingsDetail(s){const e=(s&&s.estimated)||{},eligible=Number(e.eligible_receipt_count)||0,excluded=(Number(e.excluded_receipt_count)||0)+(Number(e.unclassified_receipt_count)||0)+(Number(e.invalid_estimate_count)||0),basisRows=(e.by_basis||[]).map(x=>'
'+esc((x.basis||'unclassified').replaceAll('_',' '))+' · '+esc(x.confidence||'unknown')+''+formatTokenCount(x.baseline_tokens)+' → '+formatTokenCount(x.emitted_tokens)+' · '+formatTokenCount(x.saved_tokens)+' saved ('+(x.receipt_count||0)+' delivery)
').join(''),counterRows=(e.by_token_counter||[]).map(x=>'
'+esc(x.token_counter||'unknown')+''+formatTokenCount(x.saved_tokens)+' saved · '+(x.receipt_count||0)+' eligible delivery
').join(''),preset=SAVINGS_PRESET==='current'?'Current release':SAVINGS_PRESET==='7d'?'Last 7 days':SAVINGS_PRESET==='since'?'Since tracking started':'All time';const buttons=['since','current','7d','all'].map(x=>'').join('');return '
Estimated context saved
View'+buttons+'
'+(eligible?'
'+formatTokenCount(e.saved_tokens)+' tokens
Baseline '+formatTokenCount(e.baseline_tokens)+' → emitted '+formatTokenCount(e.emitted_tokens)+' · '+(Number(e.savings_ratio||0)*100).toFixed(1)+'% estimated reduction
'+eligible+' eligible deliveries · confidence: '+esc(e.confidence||'unknown')+' · range: '+preset+'
'+(basisRows||'
No basis breakdown available.
')+(counterRows?'
Token counters
'+counterRows:''):'
No eligible estimates in this range.
')+'
'+excluded+' excluded or unclassified delivery(s). Measures estimated prompt-context reduction; it does not measure provider billing.
'} async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{if(!window.__savingsPresetBound){window.__savingsPresetBound=true;document.addEventListener('click',function(ev){const button=ev.target.closest('[data-savings-preset]');if(!button)return;SAVINGS_PRESET=button.getAttribute('data-savings-preset')||'all';loadReceipts()})}const q='workspace='+encodeURIComponent(WS||''),sq=savingsPresetQuery();const [d,v,s]=await Promise.all([api('/receipts?'+q+'&limit=500'),api('/receipts/verify?'+q),api('/context-savings?'+sq)]);const rows=d.entries||[],packed=(s.by_token_counter||[]).map(x=>'
'+esc(x.token_counter||'unknown')+''+formatTokenCount(x.context_tokens)+' packed / '+formatTokenCount(x.source_tokens)+' source · '+formatTokenCount(x.saved_tokens)+' legacy saved
').join('');const packedCard='
Packed context accounting
Packing savings compare retrieved source tokens with emitted context. They are not added again to adaptive history savings.
'+(packed||'
No complete context-usage receipts yet.
')+'
';el.innerHTML=renderSavingsDetail(s)+packedCard+'
Receipt chain '+(v.valid?'verified':'invalid')+'
'+(v.count||0)+' receipts · head '+esc((v.head||'').slice(0,24))+'
'+(rows.length?'
'+rows.map(r=>'
'+esc(r.operation||'operation')+''+esc((r.hash||'').slice(0,20))+' · '+esc(r.status||'ok')+' · '+(r.target_count||0)+' target(s)'+(r.ts_ms?fmtRel(r.ts_ms/1000):'')+'
').join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} diff --git a/engraphis/classic_assets/index.html b/engraphis/classic_assets/index.html index 421b2892..d38302fd 100644 --- a/engraphis/classic_assets/index.html +++ b/engraphis/classic_assets/index.html @@ -9,6 +9,7 @@ +
-
+
diff --git a/engraphis/cloud_features.py b/engraphis/cloud_features.py index 0c16f4c3..54d58b3f 100644 --- a/engraphis/cloud_features.py +++ b/engraphis/cloud_features.py @@ -356,7 +356,7 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, ) count = service.store.conn.execute( "SELECT COUNT(*) AS n FROM memories WHERE workspace_id=? " - "AND COALESCE(scope, 'workspace')!='session'", + "AND COALESCE(scope, 'workspace') NOT IN ('session', 'user')", (workspace_id,), ).fetchone()["n"] if count > MAX_MEMORIES: @@ -366,7 +366,8 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, "SELECT id, title, content, mtype, scope, ingested_at, last_access, valid_from, " "valid_to, valid_to_recorded_at, expired_at, subject_key, claim_kind, " "stability, importance, pinned, sensitivity, metadata, provenance " - "FROM memories WHERE workspace_id=? AND COALESCE(scope, 'workspace')!='session' " + "FROM memories WHERE workspace_id=? " + "AND COALESCE(scope, 'workspace') NOT IN ('session', 'user') " "ORDER BY ingested_at, id", (workspace_id,), ) diff --git a/engraphis/commercial_manifest.json b/engraphis/commercial_manifest.json index 898326fc..f581f5e4 100644 --- a/engraphis/commercial_manifest.json +++ b/engraphis/commercial_manifest.json @@ -1,6 +1,6 @@ { "schema": "engraphis-commercial/v2", - "version": "1.5", + "version": "1.6", "control_plane": "https://api.engraphis.com", "account_portal": "https://api.engraphis.com/account", "billing": { diff --git a/engraphis/config.py b/engraphis/config.py index 71a5218a..d0c92531 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -12,7 +12,6 @@ import sys import time from contextlib import contextmanager -from io import StringIO import uuid from dataclasses import dataclass, field from pathlib import Path, PurePosixPath, PureWindowsPath @@ -27,6 +26,9 @@ ) _MAX_CONFIG_ENV_BYTES = 1024 * 1024 +_CONFIG_ENV_ASSIGNMENT = re.compile( + r"(?:export[ \t]+)?([A-Z][A-Z0-9_]*)[ \t]*=(.*)" +) def _resolve_config_env_path( @@ -54,6 +56,88 @@ def trusted_env_path() -> Path: return _CONFIG_ENV_PATH +def _trusted_env_syntax_error(line_number: int) -> ValueError: + """Return a value-free parse error so configuration secrets are never echoed.""" + return ValueError(f"trusted config contains invalid syntax on line {line_number}") + + +def _parse_trusted_env_value(value: str, line_number: int) -> str: + """Parse one bounded dotenv-style value without expansion or shell evaluation.""" + text = value.strip(" \t") + if not text: + return "" + + # A quote at the beginning delimits the entire value. Backslashes only escape + # that quote or another backslash; every other sequence stays literal. + if text[0] in {"'", '"'}: + delimiter = text[0] + parsed: list[str] = [] + index = 1 + while index < len(text): + char = text[index] + if char == delimiter: + suffix = text[index + 1 :] + if suffix and re.fullmatch(r"[ \t]+#.*", suffix) is None: + raise _trusted_env_syntax_error(line_number) + return "".join(parsed) + if char == "\\" and index + 1 < len(text): + following = text[index + 1] + if following in {delimiter, "\\"}: + parsed.append(following) + index += 2 + continue + parsed.append(char) + index += 1 + raise _trusted_env_syntax_error(line_number) + + # Unquoted JSON and policies may contain balanced quotes. Scan them so a # + # inside JSON/CSP remains data while a whitespace-delimited trailing comment + # is ignored. Dollar expressions are deliberately left untouched. + delimiter = "" + escaped = False + comment_at: Optional[int] = None + for index, char in enumerate(text): + if escaped: + escaped = False + continue + if char == "\\": + escaped = True + continue + if delimiter: + if char == delimiter: + delimiter = "" + continue + if char in {"'", '"'} and ( + index == 0 or text[index - 1] in " \t{[(:,=" + ): + delimiter = char + elif char == "#" and index > 0 and text[index - 1] in " \t": + comment_at = index + break + if delimiter: + raise _trusted_env_syntax_error(line_number) + if comment_at is not None: + text = text[:comment_at].rstrip(" \t") + return text + + +def _parse_trusted_env(raw: str) -> dict[str, str]: + """Parse the bounded, deterministic environment-file subset we support.""" + parsed: dict[str, str] = {} + for line_number, raw_line in enumerate(raw.splitlines(), start=1): + if "\x00" in raw_line: + raise _trusted_env_syntax_error(line_number) + line = raw_line.lstrip(" \t") + if not line or line.startswith("#"): + continue + match = _CONFIG_ENV_ASSIGNMENT.fullmatch(line) + if match is None: + raise _trusted_env_syntax_error(line_number) + key, value = match.groups() + parsed[key] = _parse_trusted_env_value(value, line_number) + return parsed + + def _load_trusted_dotenv() -> None: raw = read_private_text( _CONFIG_ENV_PATH, @@ -63,35 +147,10 @@ def _load_trusted_dotenv() -> None: ) if raw is None: return - try: - from dotenv import dotenv_values - except ImportError as exc: - if _CONFIG_ENV_EXPLICIT: - raise RuntimeError( - "python-dotenv is required to load ENGRAPHIS_ENV_FILE" - ) from exc - # Core-floor (numpy-only) installs do not include python-dotenv. - # If the implicit owner-private config exists but dotenv is unavailable, - # warn instead of crashing at import time so the core remains usable - # without optional extras. Explicit ENGRAPHIS_ENV_FILE stays strict so - # user-selected configuration is not silently ignored. - import warnings - warnings.warn( - "A trusted Engraphis config file exists but python-dotenv is not installed; " - "the trusted config file will not be loaded. Install " - "'python-dotenv' or 'engraphis[all]' to enable env-file support.", - RuntimeWarning, - stacklevel=2, - ) - return - parsed = dotenv_values(stream=StringIO(raw)) + parsed = _parse_trusted_env(raw) for key, value in parsed.items(): - if key == "ENGRAPHIS_ENV_FILE" or value is None: + if key == "ENGRAPHIS_ENV_FILE": continue - if re.fullmatch(r"[A-Z][A-Z0-9_]*", key) is None: - raise ValueError("trusted config contains an invalid environment setting") - if "\x00" in value: - raise ValueError("trusted config contains an invalid environment value") os.environ.setdefault(key, value) @@ -829,11 +888,19 @@ def _parse_headers(raw: str) -> dict: try: parsed = json.loads(raw) except Exception: + # The decoder error text can echo a fragment of a header value that may + # contain secret-like content; emit a value-free diagnostic instead. + print("[engraphis] ENGRAPHIS_LLM_EXTRA_HEADERS contains invalid JSON", + file=sys.stderr) return {} if not isinstance(parsed, dict): + print("[engraphis] ENGRAPHIS_LLM_EXTRA_HEADERS must be a JSON object", + file=sys.stderr) return {} if not all(isinstance(key, str) and isinstance(value, str) for key, value in parsed.items()): + print("[engraphis] ENGRAPHIS_LLM_EXTRA_HEADERS keys and values must be strings", + file=sys.stderr) return {} return parsed diff --git a/engraphis/core/documents.py b/engraphis/core/documents.py new file mode 100644 index 00000000..b30692b3 --- /dev/null +++ b/engraphis/core/documents.py @@ -0,0 +1,1760 @@ +"""Safe, dependency-free parsing and discovery for local documents. + +This module describes source material only. Persistence deliberately belongs to +the caller so previews, imports, and future source adapters can share one bounded +parser without coupling :mod:`engraphis.core` to the service or a database. + +The parser is intentionally conservative: recognised formats are parsed through +the standard library; unknown and binary files are catalogued by a tree scan +instead of being decoded optimistically. Markdown uses the existing Obsidian +parser as an adapter, preserving its frontmatter, wikilink, tag, and attachment +semantics. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +import codecs +import csv +import configparser +import hashlib +from html.parser import HTMLParser +import io +import json +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import re +import stat +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union +import unicodedata +from urllib.parse import unquote, urlsplit +import zipfile +from xml.etree import ElementTree + +from engraphis.core.obsidian import parse_obsidian_note +from engraphis.core.secrets import secret_kind + + +IMPORTER_VERSION = "1" +MAX_DOCUMENT_BYTES = 100_000_000 +MAX_DOCUMENT_CHARS = 100_000 +MAX_DOCUMENT_WARNINGS = 100 +MAX_DOCUMENT_FILES = 10_000 +MAX_DOCUMENT_TREE_BYTES = 250_000_000 +MAX_CONTAINER_MEMBERS = 2_000 +MAX_CONTAINER_XML_BYTES = 20_000_000 +MAX_XML_ATTRIBUTE_METADATA_CHARS = 8_000 +MAX_SOURCE_PATH_CHARS = 4_096 +MAX_CONTAINER_TEXT_CHARS = MAX_DOCUMENT_CHARS +MAX_JSON_NESTING = 128 +SENSITIVE_FILENAMES = { + ".env", "credentials", "credentials.json", "id_rsa", "id_dsa", + "id_ecdsa", "id_ed25519", "authorized_keys", "known_hosts", + "recovery-codes", "recovery_codes", "tokens", "tokens.json", + "secrets", "secrets.json", "secret", "secret.json", +} + + +@dataclass(frozen=True) +class DocumentFormat: + """One intentionally small, stdlib-readable document format declaration.""" + + name: str + extensions: Tuple[str, ...] + media_type: str + container: bool = False + requires_adapter: bool = False + + +DOCUMENT_FORMATS: Dict[str, DocumentFormat] = { + "markdown": DocumentFormat("markdown", (".md", ".markdown", ".mdown"), "text/markdown"), + "text": DocumentFormat("text", (".txt", ".text", ".log"), "text/plain"), + "rst": DocumentFormat("rst", (".rst", ".rest"), "text/x-rst"), + "html": DocumentFormat("html", (".html", ".htm", ".xhtml"), "text/html"), + "json": DocumentFormat("json", (".json", ".jsonl", ".ndjson"), "application/json"), + "csv": DocumentFormat("csv", (".csv",), "text/csv"), + "tsv": DocumentFormat("tsv", (".tsv", ".tab"), "text/tab-separated-values"), + "docx": DocumentFormat("docx", (".docx",), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", True), + "odt": DocumentFormat("odt", (".odt",), "application/vnd.oasis.opendocument.text", True), + "epub": DocumentFormat("epub", (".epub",), "application/epub+zip", True), + "xlsx": DocumentFormat("xlsx", (".xlsx",), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", True), + "pptx": DocumentFormat("pptx", (".pptx",), "application/vnd.openxmlformats-officedocument.presentationml.presentation", True), + "ods": DocumentFormat("ods", (".ods",), "application/vnd.oasis.opendocument.spreadsheet", True), + "odp": DocumentFormat("odp", (".odp",), "application/vnd.oasis.opendocument.presentation", True), + "rtf": DocumentFormat("rtf", (".rtf",), "application/rtf"), + "yaml": DocumentFormat("yaml", (".yaml", ".yml"), "application/yaml"), + "toml": DocumentFormat("toml", (".toml",), "application/toml"), + "ini": DocumentFormat("ini", (".ini", ".cfg"), "text/plain"), + "xml": DocumentFormat("xml", (".xml",), "application/xml"), + "source": DocumentFormat( + "source", + ( + ".py", ".pyi", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", + ".go", ".rs", ".java", ".cs", ".c", ".h", ".cc", ".cpp", + ".cxx", ".hpp", ".hh", ".hxx", ".sql", ".tf", ".tfvars", + ".hcl", ".sh", ".ps1", ".rb", ".php", ".swift", ".kt", ".kts", + ".scala", ".lua", ".r", ".css", + ), + "text/plain", + ), + "pdf": DocumentFormat("pdf", (".pdf",), "application/pdf", requires_adapter=True), + "image": DocumentFormat( + "image", (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tif", ".tiff", ".webp"), + "image/*", requires_adapter=True, + ), + "audio": DocumentFormat( + "audio", (".mp3", ".wav", ".m4a", ".flac", ".ogg", ".opus", ".aac"), + "audio/*", requires_adapter=True, + ), + "video": DocumentFormat( + "video", (".mp4", ".mov", ".mkv", ".avi", ".webm", ".m4v"), + "video/*", requires_adapter=True, + ), +} + +DocumentAdapter = Callable[[bytes, str, Optional[int]], "DocumentRecord"] + +_HEADING_RE = re.compile(r"(?m)^\s*(#{1,6})\s+(.+?)\s*#*\s*$") +_RST_OVERLINE_RE = re.compile(r"(?m)^(.+?)\n([=\-~^`:#*+]{3,})\s*$") +_TAG_RE = re.compile(r"(?()\[\]{}]+", re.I) +_MARKDOWN_LINK_RE = re.compile(r"(? Dict[str, Any]: + return { + "relative_path": self.relative_path, "format": self.format, + "media_type": self.media_type, "title": self.title, + "content": self.content, "body": self.body, + "raw_sha256": self.raw_sha256, "canonical_sha256": self.canonical_sha256, + "source_size": self.source_size, "source_mtime_ns": self.source_mtime_ns, + "title_source": self.title_source, "metadata": self.metadata, + "frontmatter": self.frontmatter, "aliases": self.aliases, + "tags": self.tags, "dates": self.dates, "headings": self.headings, + "links": [link.__dict__ for link in self.links], + "attachments": [item.__dict__ for item in self.attachments], + "warnings": self.warnings, + } + + +@dataclass(frozen=True) +class DocumentFileIssue: + relative_path: str + reason: str + + +@dataclass +class DocumentScan: + root_path: str + source_id: str + documents: List[DocumentRecord] = field(default_factory=list) + rejected: List[DocumentFileIssue] = field(default_factory=list) + skipped: List[DocumentFileIssue] = field(default_factory=list) + complete: bool = True + + @property + def vault_path(self) -> str: + """Compatibility-friendly synonym for source-oriented callers.""" + return self.root_path + + @property + def notes(self) -> List[DocumentRecord]: + """Read-only Obsidian-shaped view used by source import planners.""" + return self.documents + + @property + def vault_id(self) -> str: + """Read-only Obsidian-shaped identity alias for generic import planners.""" + return self.source_id + + +class DocumentParseError(ValueError): + """Content-free parsing failure safe to surface in per-file reports.""" + + +def supported_document_extensions() -> set[str]: + return {extension for spec in DOCUMENT_FORMATS.values() for extension in spec.extensions} + + +def document_format_for_path(relative_path: str) -> Optional[DocumentFormat]: + suffix = Path(relative_path).suffix.casefold() + return next((spec for spec in DOCUMENT_FORMATS.values() if suffix in spec.extensions), None) + + +def canonical_source_id(root_path: Union[os.PathLike[str], str]) -> str: + root = Path(root_path).resolve() + return hashlib.sha256(str(root).encode("utf-8", "surrogatepass")).hexdigest() + + +def normalize_document_path(relative_path: str) -> str: + """Return one safe POSIX source-relative path or reject traversal/absolute paths.""" + source = unicodedata.normalize("NFC", str(relative_path or "")) + raw = source.replace("\\", "/") + candidate = PurePosixPath(raw) + windows = PureWindowsPath(raw) + if ( + not raw or source != source.strip() or candidate.is_absolute() or windows.is_absolute() + or bool(windows.drive) or any(ord(char) < 32 for char in raw) + or any(":" in part for part in candidate.parts) + or any(part in {"", ".", ".."} for part in candidate.parts) + ): + raise DocumentParseError("source path must be a safe source-relative path") + normalized = candidate.as_posix() + if len(normalized) > MAX_SOURCE_PATH_CHARS: + raise DocumentParseError("source path exceeds 4096 character safety limit") + return normalized + + +def parse_document( + raw: bytes, relative_path: str, *, source_mtime_ns: Optional[int] = None, + adapter: Optional[DocumentAdapter] = None, +) -> DocumentRecord: + """Parse one recognised document without executing its active content. + + ``content`` retains the canonical readable source representation. ``body`` is + the readable text for formats such as HTML and XML containers. Discovery uses + a code-masked representation so snippets cannot fabricate links or tags. + """ + relative_path = normalize_document_path(relative_path) + if not isinstance(raw, bytes): + raise DocumentParseError("document data must be bytes") + if len(raw) > MAX_DOCUMENT_BYTES: + raise DocumentParseError("document exceeds 100000000 byte safety limit") + spec = document_format_for_path(relative_path) + if spec is None: + raise DocumentParseError("unsupported document format") + fallback = Path(relative_path).stem or "document" + if spec.requires_adapter: + if adapter is None: + raise DocumentParseError("document format requires an optional local adapter") + try: + record = adapter(raw, relative_path, source_mtime_ns) + except (KeyboardInterrupt, SystemExit, DocumentParseError): + raise + except ValueError as exc: + raise DocumentParseError(_safe_reason(exc)) from None + except Exception: + raise DocumentParseError("optional document adapter failed") from None + if not isinstance(record, DocumentRecord): + raise DocumentParseError("document adapter returned an invalid record") + # Validate text before deriving its canonical hash. Adapters are an + # extension boundary; a malformed third-party adapter must produce the + # same content-free per-file error as every other parser failure rather + # than leaking an AttributeError/UnicodeEncodeError to a caller. + if ( + not isinstance(record.content, str) + or not isinstance(record.body, str) + or not isinstance(record.title, str) + or not isinstance(record.metadata, dict) + ): + raise DocumentParseError("document adapter returned invalid text") + if ( + not isinstance(record.warnings, list) + or len(record.warnings) > MAX_DOCUMENT_WARNINGS + or any(not isinstance(warning, str) for warning in record.warnings) + or any(len(warning) > MAX_DOCUMENT_CHARS for warning in record.warnings) + or any(secret_kind(warning) is not None for warning in record.warnings) + ): + raise DocumentParseError("document adapter returned invalid warnings") + try: + canonical_sha256 = hashlib.sha256(record.content.encode("utf-8")).hexdigest() + # Validate both writable strings, not only the canonical content. + # A lone surrogate in ``body`` otherwise escapes this boundary and + # can fail later while serialising preview or memory metadata. + record.body.encode("utf-8") + record.title.encode("utf-8") + except UnicodeEncodeError: + raise DocumentParseError("document adapter returned invalid text") from None + if ( + record.relative_path != relative_path or record.format != spec.name + or record.source_size != len(raw) + or record.raw_sha256 != hashlib.sha256(raw).hexdigest() + or record.canonical_sha256 != canonical_sha256 + or record.source_mtime_ns != source_mtime_ns + ): + raise DocumentParseError("document adapter returned an invalid source identity") + if not record.body.strip(): + raise DocumentParseError("document produced no readable text") + if len(record.content) > MAX_DOCUMENT_CHARS or len(record.body) > MAX_DOCUMENT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + if ( + secret_kind(record.content) is not None + or secret_kind(record.body) is not None + or secret_kind(record.title) is not None + or secret_kind(record.metadata) is not None + ): + raise DocumentParseError("source appears to contain a secret") + return record + if not spec.container and _looks_binary(raw): + raise DocumentParseError("binary content is not a readable document") + if spec.name == "markdown": + markdown_text, decode_warnings = _decode_text(raw) + if len(markdown_text) > MAX_DOCUMENT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + record = _markdown_record( + markdown_text, relative_path, source_mtime_ns, + raw=raw, decode_warnings=decode_warnings, + ) + if not record.body.strip(): + raise DocumentParseError("document produced no readable text") + return record + if spec.name == "xml": + body, title, metadata, warnings = _xml_body(raw, fallback) + content = body + elif spec.container: + content, body, title, metadata, warnings = _parse_container(spec.name, raw) + else: + if spec.name == "rtf": + content, decode_warnings = _decode_rtf(raw) + elif spec.name == "html": + content, decode_warnings = _decode_html(raw) + else: + content, decode_warnings = _decode_text(raw) + content = _canonical(content) + if len(content) > MAX_DOCUMENT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + body, title, metadata, warnings = _parse_text_format(spec.name, content, relative_path) + warnings = decode_warnings + warnings + if len(content) > MAX_DOCUMENT_CHARS or len(body) > MAX_DOCUMENT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + if not body.strip(): + raise DocumentParseError("document produced no readable text") + if ( + secret_kind(content) is not None + or secret_kind(body) is not None + or secret_kind(title) is not None + or secret_kind(metadata) is not None + ): + raise DocumentParseError("source appears to contain a secret") + return _record( + relative_path, spec, raw, content, body, title, metadata, warnings, + source_mtime_ns=source_mtime_ns, + ) + + +def scan_document_tree( + root_path: Union[os.PathLike[str], str], *, adapter: Optional[DocumentAdapter] = None, +) -> DocumentScan: + """Discover recognised documents below a safe directory, continuing per-file errors.""" + selected = Path(root_path) + try: + selected_info = os.lstat(selected) + if selected.is_symlink() or _is_reparse_point(selected_info): + raise DocumentParseError("source root cannot be a symlink") + root = selected.resolve(strict=True) + except OSError as exc: + raise DocumentParseError("source path must be an existing directory") from exc + if not root.is_dir(): + raise DocumentParseError("source path must be an existing directory") + result = DocumentScan(str(root), canonical_source_id(root)) + scanned_files = scanned_bytes = 0 + normalized_paths = set() + portable_paths = set() + for path, issue in _walk_tree(root, root): + raw_relative = path.relative_to(root).as_posix() + if issue: + # The root itself is represented by "." when its directory listing fails. + # Keep that sentinel as a valid issue path so the incomplete flag is set + # before normal path validation can reject it. + if raw_relative == ".": + relative = "." + else: + try: + relative = normalize_document_path(raw_relative) + except DocumentParseError as exc: + result.rejected.append(DocumentFileIssue(raw_relative, _safe_reason(exc))) + continue + result.skipped.append(DocumentFileIssue(relative, issue)) + if issue in {"unreadable directory", "unreadable path", "directory exceeds safety limit"}: + result.complete = False + continue + try: + relative = normalize_document_path(raw_relative) + except DocumentParseError as exc: + result.rejected.append(DocumentFileIssue(raw_relative, _safe_reason(exc))) + continue + if relative in normalized_paths or relative.casefold() in portable_paths: + result.rejected.append(DocumentFileIssue(relative, "duplicate normalized source path")) + continue + normalized_paths.add(relative) + portable_paths.add(relative.casefold()) + scanned_files += 1 + if scanned_files > MAX_DOCUMENT_FILES: + result.rejected.append(DocumentFileIssue(relative, "source exceeds 10000 file safety limit")) + result.complete = False + break + if _sensitive_filename(path.name): + result.rejected.append(DocumentFileIssue(relative, "sensitive filename")) + continue + if document_format_for_path(relative) is None: + result.skipped.append(DocumentFileIssue(relative, "unsupported document format")) + continue + try: + raw, mtime_ns = _read_tree_file(root, path) + scanned_bytes += len(raw) + if scanned_bytes > MAX_DOCUMENT_TREE_BYTES: + result.rejected.append(DocumentFileIssue(relative, "source exceeds 250000000 byte safety limit")) + result.complete = False + break + result.documents.append(parse_document( + raw, relative, source_mtime_ns=mtime_ns, adapter=adapter, + )) + except (OSError, DocumentParseError, ValueError) as exc: + result.rejected.append(DocumentFileIssue(relative, _safe_reason(exc))) + return result + + +def _markdown_record( + text: str, relative_path: str, mtime_ns: Optional[int], *, + raw: bytes, decode_warnings: List[str], +) -> DocumentRecord: + try: + # The adapter accepts UTF-8 bytes, so pass the already detected text through + # UTF-8 while retaining the original bytes for identity and size metadata. + note = parse_obsidian_note(text.encode("utf-8"), relative_path, source_mtime_ns=mtime_ns) + except ValueError as exc: + if "no readable text" in str(exc): + raise DocumentParseError("document produced no readable text") from None + raise DocumentParseError(_safe_reason(exc)) from None + # The compatibility parser masks ordinary fences. Reparse a locally masked copy + # for discovery so unclosed or variable-length backtick fences cannot mint links, + # tags, attachments, headings, or a title from code. The original body is retained. + masked_body = _mask_code(note.body) + discovered = note + if masked_body != note.body: + prefix = note.content[:-len(note.body)] if note.body else note.content + try: + discovered = parse_obsidian_note( + (prefix + masked_body).encode("utf-8"), relative_path, + source_mtime_ns=mtime_ns, + ) + except ValueError as exc: + raise DocumentParseError(_safe_reason(exc)) from None + spec = DOCUMENT_FORMATS["markdown"] + links = [ + DocumentLink( + link.target, link.display_text, link.heading, link.block_id, link.embedded, + ) + for link in discovered.links + ] + links = _dedupe_links([*links, *_links(masked_body)]) + attachments = [ + AttachmentReference(item.path, item.embedded) for item in discovered.attachments + ] + attachment_keys = {(item.path, item.embedded) for item in attachments} + for item in _attachments(masked_body): + key = (item.path, item.embedded) + if key not in attachment_keys: + attachment_keys.add(key) + attachments.append(item) + return DocumentRecord( + relative_path=note.relative_path, format=spec.name, media_type=spec.media_type, + title=discovered.title, content=note.content, body=note.body, + raw_sha256=hashlib.sha256(raw).hexdigest(), canonical_sha256=note.canonical_sha256, + source_size=len(raw), source_mtime_ns=mtime_ns, + title_source=discovered.title_source, metadata={"adapter": "obsidian-markdown"}, + frontmatter=discovered.frontmatter, aliases=discovered.aliases, tags=discovered.tags, + dates=discovered.dates, headings=discovered.headings, + links=links, attachments=attachments, + warnings=decode_warnings + note.warnings + [ + item for item in discovered.warnings + if item not in note.warnings and item not in decode_warnings + ], + ) + + +def _record( + relative_path: str, spec: DocumentFormat, raw: bytes, content: str, body: str, + title: str, metadata: Dict[str, Any], warnings: List[str], *, + source_mtime_ns: Optional[int], +) -> DocumentRecord: + bounded_title = title[:MAX_DOCUMENT_CHARS] + bounded_metadata = dict(metadata) + metadata_title = bounded_metadata.get("title") + if isinstance(metadata_title, str): + bounded_metadata["title"] = metadata_title[:MAX_DOCUMENT_CHARS] + visible = "" if spec.name == "source" else _mask_code(body) + headings = _headings(spec.name, visible) + for heading in bounded_metadata.get("document_headings", []): + if isinstance(heading, str) and heading.strip(): + headings.append(heading.strip()[:300]) + headings = _dedupe(headings) + links = _links(visible) + for target in bounded_metadata.get("document_links", []): + if isinstance(target, str) and target: + links.append(DocumentLink(target)) + links = _dedupe_links(links) + attachments = _attachments(visible) + tags = _dedupe(_TAG_RE.findall(visible)) + fallback = Path(relative_path).stem or "document" + return DocumentRecord( + relative_path=relative_path, format=spec.name, media_type=spec.media_type, + title=bounded_title or (headings[0] if headings else fallback), content=content, body=body, + raw_sha256=hashlib.sha256(raw).hexdigest(), + canonical_sha256=hashlib.sha256(content.encode("utf-8")).hexdigest(), + source_size=len(raw), source_mtime_ns=source_mtime_ns, + title_source="metadata" if bounded_title else "heading" if headings else "filename", + metadata=bounded_metadata, tags=tags, headings=headings, links=links, + attachments=attachments, warnings=warnings, + ) + + +def _decode_text(raw: bytes) -> Tuple[str, List[str]]: + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + try: + return raw.decode("utf-16"), [] + except UnicodeDecodeError: + return raw.decode("utf-16", errors="replace"), ["invalid UTF-16 was replaced with U+FFFD"] + try: + return raw.decode("utf-8-sig"), [] + except UnicodeDecodeError: + return raw.decode("utf-8-sig", errors="replace"), ["invalid UTF-8 was replaced with U+FFFD"] + + +class _HTMLCharsetParser(HTMLParser): + """Find the first real HTML ``meta`` charset declaration.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=False) + self.encoding = "" + + def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None: + if self.encoding or tag.casefold() != "meta": + return + values = { + key.casefold(): value or "" + for key, value in attrs + if key + } + candidate = values.get("charset", "").strip() + if not candidate and values.get("http-equiv", "").casefold() == "content-type": + match = re.search( + r"\bcharset\s*=\s*([A-Za-z0-9._:-]+)", + values.get("content", ""), + re.IGNORECASE, + ) + candidate = match.group(1) if match else "" + if candidate: + self.encoding = candidate + + +def _decode_xhtml_prolog(raw: bytes) -> Optional[str]: + """Return the encoding declared in an XML prolog, if raw looks like XML. + + XHTML may declare its encoding in the ```` prolog + before any HTML ```` element, so check it before the meta + path. ``None`` means no usable prolog encoding was found. + """ + if not raw.lstrip().startswith(b"<"): + return None + head = raw.lstrip()[1:].lstrip() + if not head.startswith(b"?xml"): + return None + match = _EPUB_XML_ENCODING_RE.search(raw[:4096]) + if not match: + return None + try: + return codecs.lookup(match.group(2).decode("ascii")).name + except (LookupError, UnicodeError): + return None + + +def _decode_html(raw: bytes) -> Tuple[str, List[str]]: + """Decode HTML using an early in-document charset declaration when present.""" + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + return _decode_text(raw) + prolog_encoding = _decode_xhtml_prolog(raw) + if prolog_encoding: + try: + return raw.decode(prolog_encoding), [] + except UnicodeDecodeError: + return raw.decode(prolog_encoding, errors="replace"), [ + "invalid %s was replaced with U+FFFD" % prolog_encoding.upper(), + ] + parser = _HTMLCharsetParser() + try: + # Charset declarations are ASCII by definition. Parsing a latin-1 view + # lets HTMLParser ignore comments, script/style data, and other non-meta + # content without needing to decode the document using a guessed charset. + parser.feed(raw[:65536].decode("latin-1")) + parser.close() + except Exception: + return _decode_text(raw) + if not parser.encoding: + return _decode_text(raw) + try: + encoding = codecs.lookup(parser.encoding).name + except LookupError: + return _decode_text(raw) + try: + return raw.decode(encoding), [] + except UnicodeDecodeError: + return raw.decode(encoding, errors="replace"), [ + "invalid %s was replaced with U+FFFD" % encoding.upper(), + ] + + +_RTF_ANSI_CODE_PAGE_RE = re.compile(rb"\\ansicpg([0-9]+)") +def _decode_rtf(raw: bytes) -> Tuple[str, List[str]]: + """Decode literal RTF bytes using the document's declared ANSI code page.""" + match = _RTF_ANSI_CODE_PAGE_RE.search(raw[:4096]) + encoding = "cp1252" + if match: + encoding = "cp%s" % match.group(1).decode("ascii") + try: + codecs.lookup(encoding) + except LookupError: + # _rtf_body() reports the malformed control word without exposing + # the code-page value in the error surface. + encoding = "cp1252" + try: + return raw.decode(encoding), [] + except UnicodeDecodeError: + return raw.decode(encoding, errors="replace"), [ + "invalid %s was replaced with U+FFFD" % encoding.upper(), + ] + + +def _looks_binary(raw: bytes) -> bool: + """Reject binary payloads even when an extension claims to be text.""" + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + return False + sample = raw[:8192] + if not sample: + return False + if b"\x00" in sample: + return True + controls = sum(byte < 32 and byte not in (9, 10, 12, 13) for byte in sample) + return controls / len(sample) > 0.02 + + +def _is_reparse_point(info: os.stat_result) -> bool: + marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(getattr(info, "st_file_attributes", 0) & marker) + + +def _parse_text_format( + name: str, content: str, relative_path: str, +) -> Tuple[str, str, Dict[str, Any], List[str]]: + fallback = Path(relative_path).stem or "document" + if name == "html": + parser = _ReadableHTMLParser() + try: + parser.feed(content) + parser.close() + except DocumentParseError: + raise + except Exception: + raise DocumentParseError("invalid HTML document") from None + metadata: Dict[str, Any] = { + "html_title": parser.title, + "document_headings": _dedupe(parser.headings), + **parser.metadata, + } + return parser.text(), parser.title or fallback, metadata, [] + if name == "json": + return _json_body( + content, fallback, + line_delimited=Path(relative_path).suffix.casefold() in {".jsonl", ".ndjson"}, + ) + if name in {"csv", "tsv"}: + return _tabular_body(content, fallback, "\t" if name == "tsv" else None) + if name in {"yaml", "toml"}: + match = _CONFIG_TITLE_RE.search(content) + title = match.group(1).strip()[:300] if match else fallback + return content, title, {"config_kind": name}, [] + if name == "ini": + return _ini_body(content, fallback) + if name == "rtf": + return _rtf_body(content, fallback) + return content, fallback, {}, [] + + +def _ini_body(content: str, fallback: str) -> Tuple[str, str, Dict[str, Any], List[str]]: + parser = configparser.ConfigParser(interpolation=None) + try: + parser.read_string(content) + except (configparser.Error, UnicodeError): + return content, fallback, {"config_kind": "ini"}, ["INI could not be parsed; preserved as text"] + title = parser.defaults().get("title") or parser.defaults().get("name") or fallback + return content, str(title)[:300], {"config_kind": "ini", "sections": parser.sections()[:100]}, [] + + +def _xml_body(raw: bytes, fallback: str) -> Tuple[str, str, Dict[str, Any], List[str]]: + root = _xml_root(raw, "XML document") + body = "\n".join(part.strip() for part in root.itertext() if part.strip()) + if not body: + raise DocumentParseError("document produced no readable text") + title = str(root.attrib.get("title") or root.attrib.get("name") or fallback)[:300] + links = [] + attributes: List[Dict[str, str]] = [] + attribute_chars = 0 + omitted_attributes = 0 + for element in root.iter(): + for name, raw_value in element.attrib.items(): + value = str(raw_value) + # XML text extraction intentionally excludes attributes, so inspect the + # complete attribute map before applying the bounded metadata projection. + if secret_kind({str(name): value}) is not None: + raise DocumentParseError("source appears to contain a secret") + name_text = str(name)[:200] + value_text = value[:1000] + entry_size = len(name_text) + len(value_text) + 4 + if attribute_chars + entry_size <= MAX_XML_ATTRIBUTE_METADATA_CHARS: + attributes.append({"name": name_text, "value": value_text}) + attribute_chars += entry_size + else: + omitted_attributes += 1 + for name in ("href", "src"): + value = element.attrib.get(name) + if value: + links.append(value) + metadata: Dict[str, Any] = { + "xml_root": root.tag, "document_links": _dedupe(links), + "xml_attributes": attributes, + } + if omitted_attributes: + metadata["xml_attributes_omitted"] = omitted_attributes + return body, title, metadata, [] + + +def _rtf_body(content: str, fallback: str) -> Tuple[str, str, Dict[str, Any], List[str]]: + if not content.lstrip().startswith("{\\rtf"): + raise DocumentParseError("invalid RTF document") + output: List[str] = [] + suppressed = [False] + unicode_fallback = [1] + ansi_code_page = ["cp1252"] + pending_high_surrogate: Optional[int] = None + destinations = {"colortbl", "datastore", "fonttbl", "info", "object", "pict", "stylesheet"} + + def append_text(value: str) -> None: + nonlocal pending_high_surrogate + if pending_high_surrogate is not None: + output.append("\ufffd") + pending_high_surrogate = None + output.append(value) + + def append_unicode_unit(unit: int) -> None: + nonlocal pending_high_surrogate + if pending_high_surrogate is not None: + if 0xDC00 <= unit <= 0xDFFF: + output.append(chr( + 0x10000 + + ((pending_high_surrogate - 0xD800) << 10) + + (unit - 0xDC00) + )) + pending_high_surrogate = None + return + output.append("\ufffd") + pending_high_surrogate = None + if 0xD800 <= unit <= 0xDBFF: + pending_high_surrogate = unit + elif 0xDC00 <= unit <= 0xDFFF: + output.append("\ufffd") + else: + output.append(chr(unit)) + + def skip_unicode_fallback(start: int) -> int: + """Consume one RTF fallback character without exposing its syntax.""" + if start >= len(content) or content[start] in "{}": + return start + if content[start] != "\\": + return start + 1 + end = start + 1 + if end >= len(content): + return end + marker = content[end] + if marker == "'": + return min(len(content), end + 3) + if marker.isalpha(): + end += 1 + while end < len(content) and content[end].isalpha(): + end += 1 + while end < len(content) and content[end] in "-0123456789": + end += 1 + if end < len(content) and content[end] == " ": + end += 1 + return end + return end + 1 + + index = 0 + while index < len(content): + char = content[index] + if char == "{": + if len(suppressed) >= 256: + raise DocumentParseError("RTF nesting exceeds safety limit") + suppressed.append(suppressed[-1]) + unicode_fallback.append(unicode_fallback[-1]) + ansi_code_page.append(ansi_code_page[-1]) + index += 1 + elif char == "}": + if len(suppressed) == 1: + raise DocumentParseError("invalid RTF document") + suppressed.pop() + unicode_fallback.pop() + ansi_code_page.pop() + index += 1 + elif char != "\\": + if not suppressed[-1]: + append_text(char) + index += 1 + elif index + 1 >= len(content): + raise DocumentParseError("invalid RTF document") + else: + marker = content[index + 1] + if marker == "*": + suppressed[-1] = True + index += 2 + elif marker == "'" and index + 3 < len(content): + try: + decoded = bytes.fromhex(content[index + 2:index + 4]).decode( + ansi_code_page[-1], + ) + except (LookupError, UnicodeDecodeError, ValueError): + raise DocumentParseError("invalid RTF document") from None + if not suppressed[-1]: + append_text(decoded) + index += 4 + elif marker.isalpha(): + end = index + 1 + while end < len(content) and content[end].isalpha(): + end += 1 + word = content[index + 1:end].casefold() + number_start = end + if word in destinations: + suppressed[-1] = True + while end < len(content) and content[end] in "-0123456789": + end += 1 + number_text = content[number_start:end] + try: + number = int(number_text) if number_text else None + except ValueError: + number = None + if end < len(content) and content[end] == " ": + end += 1 + if word == "uc" and number is not None: + unicode_fallback[-1] = max(0, min(number, MAX_DOCUMENT_CHARS)) + elif word == "ansicpg" and number is not None: + try: + code_page = f"cp{number}" + codecs.lookup(code_page) + except LookupError: + raise DocumentParseError("invalid RTF document") from None + ansi_code_page[-1] = code_page + elif word == "u" and number is not None: + codepoint = number if number >= 0 else number + 0x10000 + if 0 <= codepoint <= 0x10FFFF and not suppressed[-1]: + append_unicode_unit(codepoint) + remaining = unicode_fallback[-1] + while remaining and end < len(content): + # A fallback control symbol/word represents one character; + # never consume a group delimiter while skipping it. + next_end = skip_unicode_fallback(end) + if next_end == end: + break + end = next_end + remaining -= 1 + index = end + continue + elif word == "bin" and number is not None: + # \binN is followed by N raw binary bytes that may contain + # braces or backslashes; skip them without emitting, parsing, + # or counting them toward suppression so the group stack + # cannot be unbalanced by a binary payload. + if not 0 <= number <= MAX_DOCUMENT_CHARS: + raise DocumentParseError("invalid RTF document") + index = min(len(content), end + number) + continue + if not suppressed[-1] and word in {"line", "par"}: + append_text("\n") + elif not suppressed[-1] and word == "tab": + append_text("\t") + index = end + else: + if not suppressed[-1] and marker in {"~", "-", "_"}: + append_text(" " if marker != "_" else "-") + elif not suppressed[-1] and marker in {"\\", "{", "}"}: + append_text(marker) + index += 2 + if len(suppressed) != 1: + raise DocumentParseError("invalid RTF document") + if pending_high_surrogate is not None: + output.append("\ufffd") + body = _canonical("".join(output)).strip() + if not body: + raise DocumentParseError("document produced no readable text") + return body, fallback, {"rtf": True}, [] + + +def _json_body( + content: str, fallback: str, *, line_delimited: bool = False, +) -> Tuple[str, str, Dict[str, Any], List[str]]: + warnings: List[str] = [] + if _json_nesting_exceeds(content, MAX_JSON_NESTING): + return content, fallback, {}, [ + "JSON nesting exceeds safety limit; preserved as text", + ] + try: + if not line_delimited: + value = json.loads(content) + structured = _bounded_json_dump(value) + if structured is None: + return content, fallback, {}, [ + "JSON output exceeds safety limit; preserved as text", + ] + title = str(value.get("title") or value.get("name") or fallback) if isinstance(value, dict) else fallback + meta: Dict[str, Any] = {"json_kind": type(value).__name__} + if isinstance(value, dict): + meta["keys"] = sorted(str(key) for key in value)[:100] + return structured, title, meta, warnings + rows = [json.loads(line) for line in content.splitlines() if line.strip()] + structured = _bounded_json_dump(rows) + if structured is None: + return content, fallback, {}, [ + "JSON output exceeds safety limit; preserved as text", + ] + return structured, fallback, {"json_kind": "jsonl", "records": len(rows)}, warnings + except (json.JSONDecodeError, RecursionError, TypeError, ValueError): + warnings.append("JSON could not be parsed; preserved as text") + return content, fallback, {}, warnings + + +def _json_nesting_exceeds(content: str, limit: int) -> bool: + """Bound JSON structure depth without parsing attacker-controlled objects.""" + depth = 0 + in_string = False + escaped = False + for char in content: + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "[{": + depth += 1 + if depth > limit: + return True + elif char in "]}": + depth = max(0, depth - 1) + return False + + +def _bounded_json_dump(value: Any) -> Optional[str]: + """Pretty-print JSON without materializing output beyond the text budget.""" + encoder = json.JSONEncoder( + ensure_ascii=False, indent=2, sort_keys=True, + ) + chunks: List[str] = [] + length = 0 + try: + for chunk in encoder.iterencode(value): + length += len(chunk) + if length > MAX_DOCUMENT_CHARS: + return None + chunks.append(chunk) + except (RecursionError, TypeError, ValueError): + return None + return "".join(chunks) + + +def _tabular_body(content: str, fallback: str, delimiter: Optional[str]) -> Tuple[str, str, Dict[str, Any], List[str]]: + try: + dialect = csv.excel_tab if delimiter == "\t" else csv.Sniffer().sniff(content[:8192], delimiters=",;\t|") + reader = csv.reader(io.StringIO(content), dialect=dialect) + rows = list(reader) + except (csv.Error, UnicodeError): + return content, fallback, {}, ["table could not be parsed; preserved as text"] + if not rows: + return content, fallback, {"rows": 0, "columns": []}, [] + columns = rows[0][:200] + return content, str(columns[0]).strip() or fallback, {"rows": len(rows) - 1, "columns": columns, "delimiter": dialect.delimiter}, [] + + +def _parse_container(name: str, raw: bytes) -> Tuple[str, str, str, Dict[str, Any], List[str]]: + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + _validate_archive(archive) + if name == "docx": + body, meta = _office_body(archive, "word/document.xml", "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}", ("p",), "t") + elif name == "ods": + body, meta = _ods_body(archive) + elif name in {"odt", "odp"}: + body, meta = _office_body(archive, "content.xml", "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}", ("h", "p"), None) + elif name == "xlsx": + body, meta = _xlsx_body(archive) + elif name == "pptx": + body, meta = _pptx_body(archive) + else: + body, meta = _epub_body(archive) + except DocumentParseError: + raise + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile, KeyError): + raise DocumentParseError("invalid %s archive" % name) from None + content = body + if not body.strip(): + raise DocumentParseError("document produced no readable text") + return content, body, str(meta.get("title") or ""), meta, [] + + +def _validate_archive(archive: zipfile.ZipFile) -> None: + members = archive.infolist() + if len(members) > MAX_CONTAINER_MEMBERS: + raise DocumentParseError("container has too many members") + total = 0 + names = set() + for info in members: + path = PurePosixPath(info.filename) + if ( + path.is_absolute() or ".." in path.parts or not info.filename + or "\\" in info.filename or "\x00" in info.filename + ): + raise DocumentParseError("container has unsafe member path") + if info.filename in names: + raise DocumentParseError("container has duplicate member paths") + names.add(info.filename) + if info.flag_bits & 0x1: + raise DocumentParseError("encrypted containers are not supported") + if info.is_dir(): + continue + if info.file_size > MAX_CONTAINER_XML_BYTES: + raise DocumentParseError("container member is too large after decompression") + total += info.file_size + if total > MAX_CONTAINER_XML_BYTES: + raise DocumentParseError("container is too large after decompression") + if info.compress_size and info.file_size > info.compress_size * 200: + raise DocumentParseError("container compression ratio is unsafe") + + +def _xml_root(raw: bytes, label: str) -> ElementTree.Element: + if re.search(br" Tuple[str, Dict[str, Any]]: + raw = archive.read(member) + root = _xml_root(raw, "office document") + values: List[str] = [] + total = 0 + block_tags = {namespace + block for block in blocks} + for element in root.iter(): + if element.tag not in block_tags: + continue + nodes = element.itertext() if text_node is None else (node.text or "" for node in element.iter(namespace + text_node)) + text = _bounded_join(nodes, limit=MAX_CONTAINER_TEXT_CHARS - total).strip() + if text: + separator = 2 if values else 0 + if total + separator + len(text) > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + values.append(text) + total += separator + len(text) + return "\n\n".join(values), {"paragraphs": len(values)} + + +def _ods_body(archive: zipfile.ZipFile) -> Tuple[str, Dict[str, Any]]: + """Extract displayed and attribute-backed values from an ODS worksheet.""" + raw = archive.read("content.xml") + root = _xml_root(raw, "ODS document") + rows: List[str] = [] + cell_count = 0 + total = 0 + for row in (item for item in root.iter() if item.tag.endswith("table-row")): + repeated_row_raw = next( + ( + value for key, value in row.attrib.items() + if str(key).endswith("number-rows-repeated") + ), + "1", + ) + try: + repeated_row = max(1, min(int(repeated_row_raw), 10_000)) + except (TypeError, ValueError): + repeated_row = 1 + cells: List[str] = [] + row_cell_count = 0 + row_size = 0 + row_separator = 1 if rows else 0 + for cell in (item for item in row if item.tag.endswith("table-cell")): + repeated_raw = next( + ( + value for key, value in cell.attrib.items() + if str(key).endswith("number-columns-repeated") + ), + "1", + ) + try: + repeated = max(1, min(int(repeated_raw), 10_000)) + except (TypeError, ValueError): + repeated = 1 + value = _bounded_join( + cell.itertext(), limit=MAX_CONTAINER_TEXT_CHARS - total, + ).strip() + if not value: + value = next( + ( + str(raw_value) for key, raw_value in cell.attrib.items() + if ( + str(key).endswith("value") + or str(key).endswith("date-value") + or str(key).endswith("time-value") + or str(key).endswith("boolean-value") + or str(key).endswith("string-value") + ) + ), + "", + ).strip() + if not value: + continue + addition = len(value) * repeated + max(0, repeated - 1) + (1 if cells else 0) + if total + row_separator + row_size + addition > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + cells.extend([value] * repeated) + row_size += addition + row_cell_count += repeated + text = "\t".join(cells).strip() + if text: + addition = ( + row_separator + len(text) * repeated_row + + max(0, repeated_row - 1) + ) + if total + addition > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + rows.extend([text] * repeated_row) + total += addition + cell_count += row_cell_count * repeated_row + if not rows: + body, metadata = _office_body( + archive, "content.xml", + "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}", + ("h", "p"), None, + ) + metadata.update({"rows": 0, "cells": cell_count}) + return body, metadata + return "\n".join(rows), {"rows": len(rows), "cells": cell_count} + + +def _bounded_join(values: Iterable[str], *, limit: int) -> str: + parts: List[str] = [] + total = 0 + for value in values: + if not value: + continue + if total + len(value) > limit: + raise DocumentParseError("document exceeds 100000 character safety limit") + parts.append(value) + total += len(value) + return "".join(parts) + + +def _xlsx_body(archive: zipfile.ZipFile) -> Tuple[str, Dict[str, Any]]: + shared: List[str] = [] + if "xl/sharedStrings.xml" in archive.namelist(): + root = _xml_root(archive.read("xl/sharedStrings.xml"), "XLSX shared strings") + shared = [_bounded_join(item.itertext(), limit=MAX_CONTAINER_TEXT_CHARS) for item in root if item.tag.endswith("si")] + sheets = sorted( + (name for name in archive.namelist() if re.fullmatch(r"xl/worksheets/sheet\d+\.xml", name)), + key=lambda value: _archive_member_number(value), + ) + sheets = _relationship_ordered( + archive, sheets, + rels_member="xl/_rels/workbook.xml.rels", + part_member="xl/workbook.xml", + part_label="XLSX workbook", + entry_suffix="sheet", + base="xl", + ) + rows: List[str] = [] + total = 0 + for name in sheets: + root = _xml_root(archive.read(name), "XLSX worksheet") + for row in (item for item in root.iter() if item.tag.endswith("row")): + values: List[str] = [] + row_size = 0 + for cell in (item for item in row if item.tag.endswith("c")): + cell_type = cell.attrib.get("t") + if cell_type == "inlineStr": + inline = next( + ( + node for node in cell + if node.tag == "is" or node.tag.endswith("}is") + ), + None, + ) + value = ( + _bounded_join( + inline.itertext(), limit=MAX_CONTAINER_TEXT_CHARS, + ) + if inline is not None else "" + ) + else: + value = next( + ( + node.text or "" for node in cell.iter() + if node.tag == "v" or node.tag.endswith("}v") + ), + "", + ) + if cell_type == "s" and value.isdigit() and int(value) < len(shared): + value = shared[int(value)] + separator = 1 if values else 0 + if ( + total + (1 if rows else 0) + row_size + + separator + len(value) > MAX_CONTAINER_TEXT_CHARS + ): + raise DocumentParseError( + "document exceeds 100000 character safety limit" + ) + values.append(value) + row_size += separator + len(value) + text = "\t".join(values).strip() + if text: + if total + len(text) + (1 if rows else 0) > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + rows.append(text) + total += len(text) + (1 if rows else 0) + return "\n".join(rows), {"sheets": len(sheets), "rows": len(rows)} + + +def _pptx_body(archive: zipfile.ZipFile) -> Tuple[str, Dict[str, Any]]: + slides = sorted( + (name for name in archive.namelist() if re.fullmatch(r"ppt/slides/slide\d+\.xml", name)), + key=lambda value: _archive_member_number(value), + ) + slides = _relationship_ordered( + archive, slides, + rels_member="ppt/_rels/presentation.xml.rels", + part_member="ppt/presentation.xml", + part_label="PPTX presentation", + entry_suffix="sldId", + base="ppt", + ) + parts: List[str] = [] + total = 0 + for name in slides: + root = _xml_root(archive.read(name), "PPTX slide") + text = _bounded_join((item.text or "" for item in root.iter() if item.tag.endswith("}t")), limit=MAX_CONTAINER_TEXT_CHARS - total).strip() + if text: + if total + len(text) + (2 if parts else 0) > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + parts.append(text) + total += len(text) + (2 if parts else 0) + return "\n\n".join(parts), {"slides": len(slides)} + + +def _archive_member_number(value: str) -> int: + match = re.search(r"\d+", Path(value).stem) + return int(match.group(0)) if match else 0 + + +def _relationship_targets( + raw_rels: bytes, rels_label: str, base: str, +) -> Dict[str, str]: + """Map relationship Ids to resolved part targets from one OOXML rels part. + + Returns ``{relationship_id: resolved_member}`` for relationships whose + target resolves to a member inside the package. Relative targets are + joined to the owning part's folder, ``..`` escapes are rejected and + package-absolute (leading ``/``) targets are normalized; the caller falls + back to numeric ordering when the rels part is missing or malformed. + """ + root = _xml_root(raw_rels, rels_label) + targets: Dict[str, str] = {} + for relationship in root.iter(): + if not relationship.tag.endswith("Relationship"): + continue + rid = next( + ( + str(value) for key, value in relationship.attrib.items() + if key == "Id" or key.endswith("}Id") + ), + "", + ) + target = next( + ( + str(value) for key, value in relationship.attrib.items() + if key == "Target" or key.endswith("}Target") + ), + "", + ) + if not rid or not target: + continue + resolved = _resolve_relationship_target(target, base) + if resolved is not None: + targets[rid] = resolved + return targets + + +def _resolve_relationship_target(target: str, base: str) -> Optional[str]: + """Resolve an OOXML relationship target to a package member path. + + Relative targets are joined to the owning part's folder (``base``); a + leading ``/`` marks a package-absolute target and is kept as the full + member. ``..`` references escape the package root and are rejected, as + are empty and directory-style targets. ``None`` means the target cannot + be used. + """ + normalized = target.replace("\\", "/") + if ".." in normalized.split("/"): + return None + if not normalized or normalized.endswith("/"): + return None + if normalized.startswith("/"): + return normalized.lstrip("/") + return base + "/" + normalized if base else normalized + + +def _relationship_ordered( + archive: zipfile.ZipFile, + members: List[str], + *, + rels_member: str, + part_member: str, + part_label: str, + entry_suffix: str, + base: str, +) -> List[str]: + """Reorder container members by the part-declared relationship order. + + The workbook/presentation part lists its sheets/slides as ``r:id`` + references whose rels part maps each id to a target member; that order is + the one users see, and it need not match the numeric member order. When + the rels part, the listing part, or either XML is missing or malformed the + members keep their (numeric) input order. + """ + try: + raw_rels = archive.read(rels_member) + raw_part = archive.read(part_member) + except KeyError: + return members + try: + targets = _relationship_targets(raw_rels, rels_member, base) + if not targets: + return members + root = _xml_root(raw_part, part_label) + except DocumentParseError: + return members + members_set = set(members) + ordered: List[str] = [] + for element in root.iter(): + if not element.tag.endswith(entry_suffix): + continue + rid = next( + ( + str(value) for key, value in element.attrib.items() + if key.endswith("}id") + ), + "", + ) + target = targets.get(rid) + if target is None or target not in members_set or target in ordered: + continue + ordered.append(target) + remaining = [member for member in members if member not in ordered] + return ordered + remaining + + +_EPUB_XML_ENCODING_RE = re.compile( + br"<\?xml\b[^>]*\bencoding\s*=\s*(['\"])([^'\"]+)\1", + flags=re.I | re.S, +) + + +def _decode_epub_chapter(raw: bytes) -> str: + """Decode one EPUB spine member before feeding it to the HTML parser.""" + if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): + encoding = "utf-16" + elif raw.startswith(b"<\x00"): + # XML permits UTF-16 without a BOM; the first code unit identifies LE. + encoding = "utf-16-le" + elif raw.startswith(b"\x00<"): + encoding = "utf-16-be" + else: + match = _EPUB_XML_ENCODING_RE.search(raw[:4096]) + if match: + try: + encoding = codecs.lookup(match.group(2).decode("ascii")).name + except (LookupError, UnicodeError): + raise DocumentParseError("invalid EPUB chapter encoding") from None + else: + encoding = "utf-8" + return raw.decode(encoding, errors="replace") + + +def _epub_body(archive: zipfile.ZipFile) -> Tuple[str, Dict[str, Any]]: + container = _xml_root(archive.read("META-INF/container.xml"), "EPUB container") + rootfile = next((item.attrib.get("full-path", "") for item in container.iter() if item.tag.endswith("rootfile")), "") + if not rootfile or rootfile.startswith("/") or ".." in PurePosixPath(rootfile).parts: + raise DocumentParseError("invalid EPUB package path") + package = _xml_root(archive.read(rootfile), "EPUB package") + manifest = {item.attrib.get("id", ""): item.attrib.get("href", "") for item in package.iter() if item.tag.endswith("item")} + spine = [item.attrib.get("idref", "") for item in package.iter() if item.tag.endswith("itemref")] + base = PurePosixPath(rootfile).parent + parts: List[str] = [] + for item_id in spine: + href = manifest.get(item_id, "") + parsed_href = urlsplit(href) + href_path = unquote(parsed_href.path) + candidate = base / href_path + if ( + not href_path + or parsed_href.scheme + or parsed_href.netloc + or candidate.is_absolute() + or ".." in candidate.parts + ): + raise DocumentParseError("invalid EPUB content path") + raw = archive.read(candidate.as_posix()) + text = _html_text(_decode_epub_chapter(raw), limit=MAX_CONTAINER_TEXT_CHARS - sum(len(part) for part in parts)) + if text: + if sum(len(part) for part in parts) + len(text) + (2 if parts else 0) > MAX_CONTAINER_TEXT_CHARS: + raise DocumentParseError("document exceeds 100000 character safety limit") + parts.append(text) + title = next(("".join(item.itertext()).strip() for item in package.iter() if item.tag.endswith("title") and "".join(item.itertext()).strip()), "") + return "\n\n".join(parts), {"chapters": len(parts), "title": title} + + +class _ReadableHTMLParser(HTMLParser): + def __init__(self, *, limit: Optional[int] = None) -> None: + super().__init__(convert_charrefs=True) + self.parts: List[str] = [] + self.limit = limit + self.size = 0 + self._ignored_tags: List[str] = [] + self._code_depth = 0 + self._title = False + self._heading_depth = 0 + self._heading_parts: List[str] = [] + self.headings: List[str] = [] + self.title = "" + self.metadata: Dict[str, Any] = {} + + def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None: + lower = tag.casefold() + if self._ignored_tags: + if lower in {"script", "style", "noscript", "template"}: + self._ignored_tags.append(lower) + return + if lower in {"script", "style", "noscript", "template"}: + self._ignored_tags.append(lower) + elif lower == "title": + self._title = True + elif lower in {"pre", "code"}: + if self._code_depth == 0: + self._add("\n```\n") + self._code_depth += 1 + elif lower in {"h1", "h2", "h3", "h4", "h5", "h6"}: + if self._heading_depth == 0: + self._heading_parts = [] + self._heading_depth += 1 + elif lower == "meta": + values = {str(key).casefold(): value for key, value in attrs} + key, value = values.get("name") or values.get("property"), values.get("content") + if key and value and key.casefold() in {"description", "author", "keywords"}: + self.metadata[key.casefold()] = value[:1000] + elif lower in {"p", "br", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "article", "section"}: + self._add("\n") + if lower == "a": + values = {str(key).casefold(): value for key, value in attrs} + href = values.get("href") + if href: + self.metadata.setdefault("document_links", []).append(href) + + def handle_endtag(self, tag: str) -> None: + lower = tag.casefold() + if self._ignored_tags: + if lower == self._ignored_tags[-1]: + self._ignored_tags.pop() + return + if lower in {"h1", "h2", "h3", "h4", "h5", "h6"} and self._heading_depth: + self._heading_depth -= 1 + if self._heading_depth == 0: + heading = "".join(self._heading_parts).strip() + if heading: + self.headings.append(heading[:300]) + if lower == "title": + self._title = False + elif lower in {"pre", "code"} and self._code_depth: + self._code_depth -= 1 + if self._code_depth == 0: + self._add("\n```\n") + elif lower in {"p", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "article", "section"}: + self._add("\n") + + def handle_data(self, data: str) -> None: + if self._title: + self.title = (self.title + data).strip()[:300] + if self._heading_depth: + self._heading_parts.append(data) + if not self._ignored_tags and not self._title: + self._add(data) + + def _add(self, value: str) -> None: + if self.limit is not None and self.size + len(value) > self.limit: + raise DocumentParseError("document exceeds 100000 character safety limit") + self.parts.append(value) + self.size += len(value) + + def text(self) -> str: + return "\n".join(line.strip() for line in "".join(self.parts).splitlines() if line.strip()) + + +def _html_text(content: str, *, limit: Optional[int] = None) -> str: + parser = _ReadableHTMLParser(limit=limit) + parser.feed(content) + parser.close() + return parser.text() + + +def _headings(format_name: str, visible: str) -> List[str]: + if format_name == "rst": + return _dedupe(match.group(1).strip() for match in _RST_OVERLINE_RE.finditer(visible)) + return [match.group(2).strip() for match in _HEADING_RE.finditer(visible)] + + +def _mask_code(text: str) -> str: + lines = text.splitlines(keepends=True) + masked: List[str] = [] + fence_char = "" + fence_size = 0 + for line in lines: + stripped = line.rstrip("\r\n") + start = _FENCE_START_RE.match(stripped) + if not fence_char and start: + token = start.group(1) + fence_char, fence_size = token[0], len(token) + masked.append(_blank_code(line)) + continue + if fence_char: + if re.match(r"^[ \t]*%s{%d,}[ \t]*$" % (re.escape(fence_char), fence_size), stripped): + fence_char, fence_size = "", 0 + masked.append(_blank_code(line)) + continue + masked.append(_mask_inline_code(line)) + return "".join(masked) + + +def _blank_code(value: str) -> str: + return "".join("\n" if char == "\n" else "\r" if char == "\r" else " " for char in value) + + +def _mask_inline_code(line: str) -> str: + result = list(line) + index = 0 + while index < len(line): + if line[index] != "`": + index += 1 + continue + end = index + while end < len(line) and line[end] == "`": + end += 1 + token = line[index:end] + close = line.find(token, end) + stop = close + len(token) if close >= 0 else len(line.rstrip("\r\n")) + for cursor in range(index, stop): + if result[cursor] not in "\r\n": + result[cursor] = " " + index = stop if stop > index else end + return "".join(result) + + +def _links(text: str) -> List[DocumentLink]: + links = [DocumentLink(match.group(1)) for match in _MARKDOWN_LINK_RE.finditer(text)] + links.extend(DocumentLink(match.group(0).rstrip(".,;:")) for match in _URL_RE.finditer(text)) + return _dedupe_links(links) + + +def _attachments(text: str) -> List[AttachmentReference]: + return [AttachmentReference(path) for path in _dedupe(match.group(1) for match in _MARKDOWN_ATTACHMENT_RE.finditer(text))] + + +def _dedupe_links(values: Iterable[DocumentLink]) -> List[DocumentLink]: + result: List[DocumentLink] = [] + seen = set() + for value in values: + if value.target not in seen: + seen.add(value.target) + result.append(value) + return result + + +def _dedupe(values: Iterable[str]) -> List[str]: + result: List[str] = [] + seen = set() + for value in values: + if value and value not in seen: + seen.add(value) + result.append(value) + return result + + +def _canonical(value: str) -> str: + return value.replace("\r\n", "\n").replace("\r", "\n") + + +def _sensitive_filename(name: str) -> bool: + lowered = name.casefold() + return (lowered in SENSITIVE_FILENAMES or lowered.startswith(".env.") + or lowered.endswith((".pem", ".key", ".p12", ".pfx")) + or bool(re.search(r"(?:credential|recovery[-_ ]?code|secret|token)", lowered))) + + +def _safe_reason(exc: BaseException) -> str: + text = str(exc) + if "optional local document extractor" in text: + return "optional local extractor unavailable; install engraphis[documents]" + if "local model path" in text: + return "transcription requires an existing local model path" + allowed = ( + "secret", "safety limit", "unsupported", "invalid", "unsafe", "unreadable", "binary", + "too large", "no readable text", "changed during scan", "document data must be bytes", + ) + return next((label for label in allowed if label in text), "document parsing failed") + + +def _read_tree_file(root: Path, path: Path) -> Tuple[bytes, int]: + before = os.lstat(path) + if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_reparse_point(before): + raise DocumentParseError("unsafe file type") + if not _is_within(root, path.resolve(strict=True)): + raise DocumentParseError("path escapes source root") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + try: + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_identity(before, opened): + raise DocumentParseError("file changed during scan") + if opened.st_size > MAX_DOCUMENT_BYTES: + raise DocumentParseError("document exceeds 100000000 byte safety limit") + chunks: List[bytes] = [] + total = 0 + while True: + chunk = os.read(fd, min(64 * 1024, MAX_DOCUMENT_BYTES + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > MAX_DOCUMENT_BYTES: + raise DocumentParseError("document exceeds 100000000 byte safety limit") + finished, after = os.fstat(fd), os.lstat(path) + if (not _same_identity(opened, finished) or opened.st_size != finished.st_size + or opened.st_mtime_ns != finished.st_mtime_ns or stat.S_ISLNK(after.st_mode) + or _is_reparse_point(after) + or not _same_identity(finished, after) or not _is_within(root, path.resolve(strict=True))): + raise DocumentParseError("file changed during scan") + return b"".join(chunks), int(finished.st_mtime_ns) + finally: + os.close(fd) + + +def _same_identity(left: os.stat_result, right: os.stat_result) -> bool: + if left.st_dev or left.st_ino or right.st_dev or right.st_ino: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + return True + + +def _walk_tree(root: Path, directory: Path) -> Iterable[Tuple[Path, Optional[str]]]: + try: + entries = list(directory.iterdir()) + except OSError: + yield directory, "unreadable directory" + return + # Never sort an unbounded listing: once a single directory exceeds the scan + # budget, deterministic ordering no longer matters and sorting thousands of + # entries just wastes memory. Emit the directory as an issue and stop so + # scan_document_tree marks the result incomplete. + if len(entries) > MAX_DOCUMENT_FILES: + yield directory, "directory exceeds safety limit" + return + entries.sort( + key=lambda item: ( + unicodedata.normalize("NFC", item.name).casefold(), + unicodedata.normalize("NFC", item.name), + ), + ) + for entry in entries: + try: + relative = entry.relative_to(root) + info = entry.lstat() + if entry.is_symlink() or _is_reparse_point(info): + yield entry, "symlink skipped" + elif not _is_within(root, entry.resolve()): + yield entry, "path escapes source root" + elif any(part.startswith(".") for part in relative.parts): + yield entry, "hidden/configuration path skipped" + elif entry.is_dir(): + yield from _walk_tree(root, entry) + elif entry.is_file(): + yield entry, None + except OSError: + yield entry, "unreadable path" + + +def _is_within(root: Path, candidate: Path) -> bool: + try: + candidate.relative_to(root) + return True + except ValueError: + return False + + +__all__ = [ + "AttachmentReference", "DOCUMENT_FORMATS", "DocumentFileIssue", "DocumentFormat", + "DocumentLink", "DocumentParseError", "DocumentRecord", "DocumentScan", + "MAX_DOCUMENT_BYTES", "MAX_DOCUMENT_CHARS", "MAX_DOCUMENT_FILES", "MAX_DOCUMENT_TREE_BYTES", + "MAX_DOCUMENT_WARNINGS", + "canonical_source_id", "document_format_for_path", "normalize_document_path", + "parse_document", "scan_document_tree", "supported_document_extensions", +] diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index 3b9362a4..55234721 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -84,7 +84,13 @@ def _safe_upsert(index, ids, vecs, meta=None, *, commit=True): try: index.upsert(ids, vecs, meta, commit=commit) except TypeError: - index.upsert(ids, vecs, commit=commit) + try: + index.upsert(ids, vecs, meta) + except TypeError: + try: + index.upsert(ids, vecs, commit=commit) + except TypeError: + index.upsert(ids, vecs) logger = logging.getLogger("engraphis.core.engine") @@ -1040,8 +1046,10 @@ def _upsert_external_vector(self, memory_id: str, vec: np.ndarray) -> None: if not vector_index_requires_sync(self.index, self.store): return try: - self.index.upsert( - [memory_id], vec.reshape(1, -1), + _safe_upsert( + self.index, + [memory_id], + vec.reshape(1, -1), [{"model": self.embedding_space}], ) except Exception as exc: # noqa: BLE001 — a failed index write must not lose the memory @@ -1366,14 +1374,16 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra # SearchFilter as lexical/graph retrieval, so it is hidden from current recall # but remains available for historical ``as_of`` queries. Deleting it made # time travel silently lose the semantic arm. - linked = self._evolve(mid, neighbors, exclude={decision.target_id}) if trusted_write else [] + linked = self._evolve( + mid, neighbors, exclude={decision.target_id}, valid_from=rec.valid_from, + ) if trusted_write else [] out = {"id": mid, "op": "invalidate", "superseded": [decision.target_id], "reason": decision.reason} if linked: out["linked"] = linked return out - linked = self._evolve(mid, neighbors) if trusted_write else [] + linked = self._evolve(mid, neighbors, valid_from=rec.valid_from) if trusted_write else [] if conflicted_with: # Deterministic conflict repair: persist the ``conflicts_with`` relation # (with the real new-memory id), the audit row, and a bounded confidence @@ -1408,7 +1418,10 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra if decision is not None and decision.op == ResolutionOp.RELATE: related_to = decision.target_id if related_to and not self.store.has_link(mid, related_to): - self.store.add_link(mid, related_to, "related", reason=decision.reason) + self.store.add_link( + mid, related_to, "related", reason=decision.reason, + valid_from=rec.valid_from, + ) out = { "id": mid, "op": "relate", "related_to": related_to, "reason": decision.reason, @@ -1529,7 +1542,13 @@ def _link_memory_entities(self, memory_id: str, content: str, *, memory_id=memory_id, entity_id=entity.id, workspace_id=workspace_id, repo_id=repo_id, source_kind="text_mention", confidence=0.8, - valid_from=valid_from, commit=False, + valid_from=valid_from, + provenance={ + "memory_id": memory_id, + "source": "text_mention", + "source_kind": "text_mention", + }, + commit=False, ) if owns_transaction: self.store.conn.commit() @@ -1542,7 +1561,10 @@ def _link_memory_entities(self, memory_id: str, content: str, *, self.store.conn.rollback() self._warn_redacted_failure("memory-entity linking", exc) - def _evolve(self, new_id: str, neighbors: list, *, exclude: Optional[set] = None) -> list[str]: + def _evolve( + self, new_id: str, neighbors: list, *, exclude: Optional[set] = None, + valid_from: Optional[float] = None, + ) -> list[str]: """A-MEM-style memory evolution on write: a new memory auto-links to its closest still-live neighbors and gives them a small reinforcement touch, so old notes gain connectivity (and resist decay a little @@ -1568,7 +1590,15 @@ def _evolve(self, new_id: str, neighbors: list, *, exclude: Optional[set] = None continue if self.store.has_link(new_id, nrec.id): continue - self.store.add_link(new_id, nrec.id, "related") + link_valid_from = valid_from + if nrec.valid_from is not None: + link_valid_from = max( + nrec.valid_from, + link_valid_from if link_valid_from is not None else nrec.valid_from, + ) + self.store.add_link( + new_id, nrec.id, "related", valid_from=link_valid_from, + ) self.store.reinforce(nrec.id, boost=scoring.INTERACTION_BOOST["view"]) linked.append(nrec.id) if linked: @@ -1699,16 +1729,43 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id ) current_fallback = True neighbors = [] - for nid, sim in hits: - nrec = self.store.get_memory(nid) - if (nrec and nrec.workspace_id == workspace_id and nrec.repo_id == repo_id - and nrec.scope == scope and nrec.mtype == mtype - and nrec.session_id == session_id - and prompt_eligible(nrec.provenance, nrec.metadata) - and (memory_matches_filter(nrec, flt) - or (current_fallback and nrec.expired_at is None - and nrec.valid_to is None))): - neighbors.append((sim, nrec)) + + def append_visible_neighbors( + candidates: list[tuple[str, float]], + *, + fallback: bool, + ) -> None: + for nid, sim in candidates: + nrec = self.store.get_memory(nid) + if (nrec and nrec.workspace_id == workspace_id + and nrec.repo_id == repo_id and nrec.scope == scope + and nrec.mtype == mtype + and (scope != Scope.SESSION or nrec.session_id == session_id) + and prompt_eligible(nrec.provenance, nrec.metadata) + and (memory_matches_filter(nrec, flt) + or (fallback and nrec.expired_at is None + and nrec.valid_to is None))): + neighbors.append((sim, nrec)) + + append_visible_neighbors(hits, fallback=current_fallback) + if not neighbors and valid_at is not None and not current_fallback: + # A stale or overly broad injected index can return candidates that are + # all outside the requested historical view. Retry against the current + # index/store mirror instead of silently turning a duplicate/correction + # into a new ADD. + current_filter = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id if scope == Scope.SESSION else None, + scopes=[scope], mtypes=[mtype], + ) + hits, canonical_fallback = self._search_resolution_vectors( + vec, + candidate_k, + current_filter, + canonical_only=canonical_fallback, + ) + current_fallback = True + append_visible_neighbors(hits, fallback=current_fallback) if subject_key: # A claim identity is authoritative, while vector retrieval is only a # bounded candidate-discovery aid. Always add its visible predecessor(s): a diff --git a/engraphis/core/graphrank.py b/engraphis/core/graphrank.py index 38a1e744..0689c3e3 100644 --- a/engraphis/core/graphrank.py +++ b/engraphis/core/graphrank.py @@ -37,8 +37,41 @@ def personalized_pagerank( inputs return ``{}`` deterministically instead of attempting an unbounded local computation. """ + if not isinstance(adjacency, dict) or not isinstance(seeds, list): + return {} if not adjacency or not seeds: return {} + try: + damping = float(damping) + tol = float(tol) + iterations = int(iterations) + except (TypeError, ValueError, OverflowError): + return {} + if not math.isfinite(damping) or not 0.0 <= damping <= 1.0: + return {} + if not math.isfinite(tol) or tol < 0.0: + return {} + sanitized: dict[str, list[tuple[str, float]]] = {} + for source, neighbors in adjacency.items(): + if not isinstance(source, str) or not isinstance(neighbors, (list, tuple)): + continue + clean_neighbors: list[tuple[str, float]] = [] + for item in neighbors: + if not isinstance(item, (list, tuple)) or len(item) != 2: + continue + destination, weight = item + if not isinstance(destination, str): + continue + try: + weight = float(weight) + except (TypeError, ValueError, OverflowError): + continue + if math.isfinite(weight) and weight > 0.0: + clean_neighbors.append((destination, weight)) + sanitized[source] = clean_neighbors + adjacency = sanitized + if not any(adjacency.values()): + return {} nodes = set(adjacency) edge_count = 0 for neighbors in adjacency.values(): diff --git a/engraphis/core/ids.py b/engraphis/core/ids.py index ddeff1ae..d4280dcb 100644 --- a/engraphis/core/ids.py +++ b/engraphis/core/ids.py @@ -31,6 +31,8 @@ "audit": "aud", "device": "dev", "receipt": "rcpt", + "vault": "vlt", + "source": "src", } diff --git a/engraphis/core/obsidian.py b/engraphis/core/obsidian.py new file mode 100644 index 00000000..b23d0770 --- /dev/null +++ b/engraphis/core/obsidian.py @@ -0,0 +1,570 @@ +"""Offline, dependency-free parsing and safe discovery for Obsidian vaults. + +This module intentionally only describes source material. Persisting notes and +creating graph records belongs to the engine-level importer so that this core +utility remains usable for previews without opening a database. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +import hashlib +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import re +import stat +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +import unicodedata + +from engraphis.core.secrets import secret_kind + + +IMPORTER_VERSION = "1" +MAX_NOTE_CHARS = 100_000 +MAX_NOTE_BYTES = 2_000_000 +MAX_VAULT_FILES = 10_000 +MAX_VAULT_BYTES = 250_000_000 +MAX_SOURCE_PATH_CHARS = 4_096 +ATTACHMENT_SUFFIXES = { + ".aac", ".avif", ".bmp", ".csv", ".epub", ".gif", ".jpeg", ".jpg", + ".m4a", ".md", ".mkv", ".mov", ".mp3", ".mp4", ".ogg", ".pdf", + ".png", ".svg", ".tif", ".tiff", ".wav", ".webm", ".webp", +} +SENSITIVE_FILENAMES = { + ".env", "credentials", "credentials.json", "id_rsa", "id_dsa", + "id_ecdsa", "id_ed25519", "authorized_keys", "known_hosts", + "recovery-codes", "recovery_codes", "tokens", "tokens.json", + "secrets", "secrets.json", "secret", "secret.json", +} +_FENCE_START_RE = re.compile(r"^[ \t]*(`{3,}|~{3,})[^\n]*$") +_WIKILINK_RE = re.compile(r"(!?)\[\[([^\]\r\n]+)\]\]") +_MARKDOWN_ATTACHMENT_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)(?:\s+[^)]*)?\)") +_H1_RE = re.compile(r"(?m)^\s*#\s+(.+?)\s*#*\s*$") +_HEADING_RE = re.compile(r"(?m)^\s*(#{1,6})\s+(.+?)\s*#*\s*$") +_TAG_RE = re.compile(r"(? Dict[str, Any]: + """Return a JSON-serialisable preview record.""" + return { + "relative_path": self.relative_path, "title": self.title, + "content": self.content, "body": self.body, + "raw_sha256": self.raw_sha256, "canonical_sha256": self.canonical_sha256, + "source_size": self.source_size, "source_mtime_ns": self.source_mtime_ns, + "title_source": self.title_source, + "frontmatter": self.frontmatter, "aliases": self.aliases, "tags": self.tags, + "dates": self.dates, "headings": self.headings, + "links": [link.__dict__ for link in self.links], + "attachments": [attachment.__dict__ for attachment in self.attachments], + "warnings": self.warnings, + } + + +@dataclass(frozen=True) +class ObsidianFileIssue: + relative_path: str + reason: str + + +@dataclass +class ObsidianVaultScan: + vault_path: str + vault_id: str + notes: List[ObsidianNote] = field(default_factory=list) + rejected: List[ObsidianFileIssue] = field(default_factory=list) + skipped: List[ObsidianFileIssue] = field(default_factory=list) + complete: bool = True + + +def canonical_vault_id(vault_path: Union[os.PathLike[str], str]) -> str: + """Return a stable local identity without reading vault contents.""" + root = Path(vault_path).resolve() + return hashlib.sha256(str(root).encode("utf-8", "surrogatepass")).hexdigest() + + +def normalize_obsidian_path(relative_path: str) -> str: + """Return one safe POSIX vault-relative path or reject traversal/absolute input.""" + source = unicodedata.normalize("NFC", str(relative_path or "")) + raw = source.replace("\\", "/") + candidate = PurePosixPath(raw) + windows = PureWindowsPath(raw) + if ( + not raw + or source != source.strip() + or candidate.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or any(ord(character) < 32 for character in raw) + or any(":" in part for part in candidate.parts) + or any(part in {"", ".", ".."} for part in candidate.parts) + ): + raise ValueError("source path must be a safe vault-relative path") + normalized = candidate.as_posix() + if len(normalized) > MAX_SOURCE_PATH_CHARS: + raise ValueError("source path exceeds 4096 character safety limit") + return normalized + + +def parse_obsidian_note( + raw: bytes, + relative_path: str, + *, + source_mtime_ns: Optional[int] = None, +) -> ObsidianNote: + """Parse one Markdown file without evaluating any Obsidian/plugin content.""" + relative_path = normalize_obsidian_path(relative_path) + if not isinstance(raw, bytes): + raise ValueError("note data must be bytes") + # Keep the direct parser as bounded as the filesystem and browser scanners. + # Otherwise a caller that uses this public core API directly can force a very + # large UTF-8 replacement decode before the character limit is reached. + if len(raw) > MAX_NOTE_BYTES: + raise ValueError("note exceeds 2000000 byte safety limit") + warnings: List[str] = [] + raw_sha256 = hashlib.sha256(raw).hexdigest() + try: + content = raw.decode("utf-8-sig") + except UnicodeDecodeError: + content = raw.decode("utf-8-sig", errors="replace") + warnings.append("invalid UTF-8 was replaced with U+FFFD") + if len(content) > MAX_NOTE_CHARS: + raise ValueError("note exceeds 100000 character safety limit") + if _contains_secret(content): + raise ValueError("source appears to contain a secret") + canonical = content.replace("\r\n", "\n").replace("\r", "\n") + frontmatter, body, fm_warnings = _split_frontmatter(canonical) + warnings.extend(fm_warnings) + if not body.strip(): + raise ValueError("note produced no readable text") + visible = _mask_code(body) + stem = Path(relative_path).stem + h1 = _first_h1(visible) + frontmatter_title = _text_value(frontmatter.get("title")) + title = frontmatter_title or h1 or stem + title_source = "frontmatter" if frontmatter_title else "heading" if h1 else "filename" + aliases = _string_values(frontmatter.get("aliases")) + tags = _dedupe(_normalise_tags(_string_values(frontmatter.get("tags"))) + + _normalise_tags(_TAG_RE.findall(visible))) + dates = _date_values(frontmatter) + headings = [match.group(2).strip() for match in _HEADING_RE.finditer(visible)] + links = _links(visible) + attachments = _attachments(visible, links) + return ObsidianNote( + relative_path=relative_path, title=title, + content=canonical, body=body, + raw_sha256=raw_sha256, + canonical_sha256=hashlib.sha256(body.encode("utf-8")).hexdigest(), + source_size=len(raw), source_mtime_ns=source_mtime_ns, + title_source=title_source, + frontmatter=frontmatter, aliases=aliases, tags=tags, dates=dates, + headings=headings, links=links, attachments=attachments, warnings=warnings, + ) + + +def scan_obsidian_vault(vault_path: Union[os.PathLike[str], str]) -> ObsidianVaultScan: + """Recursively discover safe Markdown notes; symlinks and hidden trees are skipped.""" + selected_root = Path(vault_path) + try: + selected_info = os.lstat(selected_root) + if selected_root.is_symlink() or _is_reparse_point(selected_info): + raise ValueError("vault root cannot be a symlink") + root = selected_root.resolve(strict=True) + except OSError as exc: + raise ValueError("vault path must be an existing directory") from exc + if not root.is_dir(): + raise ValueError("vault path must be an existing directory") + result = ObsidianVaultScan(vault_path=str(root), vault_id=canonical_vault_id(root)) + scanned_files = 0 + scanned_bytes = 0 + normalized_paths = set() + portable_paths = set() + for path, issue in _walk_vault(root, root): + raw_relative = path.relative_to(root).as_posix() + if issue: + # The root itself is represented by "." when its directory listing + # fails. Keep that sentinel as a valid issue path before normal path + # validation can reject it, and defer reconciliation for all + # unreadable paths so transient filesystem failures are not treated + # as deletions. + if raw_relative == ".": + relative = "." + else: + try: + relative = normalize_obsidian_path(raw_relative) + except ValueError as exc: + result.rejected.append(ObsidianFileIssue(raw_relative, str(exc))) + continue + result.skipped.append(ObsidianFileIssue(relative, issue)) + if issue in {"unreadable directory", "unreadable path"}: + result.complete = False + continue + try: + relative = normalize_obsidian_path(raw_relative) + except ValueError as exc: + result.rejected.append(ObsidianFileIssue(raw_relative, str(exc))) + continue + if relative in normalized_paths or relative.casefold() in portable_paths: + result.rejected.append(ObsidianFileIssue(relative, "duplicate normalized source path")) + continue + normalized_paths.add(relative) + portable_paths.add(relative.casefold()) + if path.suffix.lower() != ".md": + continue + scanned_files += 1 + if scanned_files > MAX_VAULT_FILES: + result.rejected.append( + ObsidianFileIssue(relative, "vault exceeds 10000 Markdown file safety limit") + ) + result.complete = False + break + if _sensitive_filename(path.name): + result.rejected.append(ObsidianFileIssue(relative, "sensitive filename")) + continue + try: + raw, source_mtime_ns = _read_vault_note(root, path) + scanned_bytes += len(raw) + if scanned_bytes > MAX_VAULT_BYTES: + result.rejected.append( + ObsidianFileIssue(relative, "vault exceeds 250000000 byte safety limit") + ) + result.complete = False + break + result.notes.append( + parse_obsidian_note( + raw, relative, source_mtime_ns=source_mtime_ns, + ) + ) + except OSError: + result.rejected.append(ObsidianFileIssue(relative, "unreadable file")) + except ValueError as exc: + result.rejected.append(ObsidianFileIssue(relative, str(exc))) + return result + + +def _read_vault_note(root: Path, path: Path) -> Tuple[bytes, int]: + """Read one regular file without following a swapped symlink. + + ``O_NOFOLLOW`` closes the POSIX check/open race. The lstat/fstat identity and + post-read stability checks also fail closed on platforms where that flag is + unavailable (notably Windows) and discard bytes if the directory entry changed. + """ + before = os.lstat(path) + if stat.S_ISLNK(before.st_mode) or _is_reparse_point(before) or not stat.S_ISREG(before.st_mode): + raise ValueError("unsafe file type") + if not _is_within(root, path.resolve(strict=True)): + raise ValueError("path escapes vault") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + nofollow = getattr(os, "O_NOFOLLOW", 0) + if nofollow: + flags |= nofollow + fd = os.open(path, flags) + try: + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_file_identity(before, opened): + raise ValueError("file changed during scan") + if opened.st_size > MAX_NOTE_BYTES: + raise ValueError("note exceeds 2000000 byte safety limit") + chunks: List[bytes] = [] + total = 0 + while True: + chunk = os.read(fd, min(64 * 1024, MAX_NOTE_BYTES + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > MAX_NOTE_BYTES: + raise ValueError("note exceeds 2000000 byte safety limit") + finished = os.fstat(fd) + after = os.lstat(path) + if ( + not _same_file_identity(opened, finished) + or opened.st_size != finished.st_size + or opened.st_mtime_ns != finished.st_mtime_ns + or stat.S_ISLNK(after.st_mode) + or _is_reparse_point(after) + or not _same_file_identity(finished, after) + or not _is_within(root, path.resolve(strict=True)) + ): + raise ValueError("file changed during scan") + return b"".join(chunks), int(finished.st_mtime_ns) + finally: + os.close(fd) + + +def _same_file_identity(left: os.stat_result, right: os.stat_result) -> bool: + """Compare stable filesystem identity when the platform exposes it.""" + if left.st_dev or left.st_ino or right.st_dev or right.st_ino: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + return True + + +def _is_reparse_point(info: os.stat_result) -> bool: + marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(getattr(info, "st_file_attributes", 0) & marker) + + +def _walk_vault(root: Path, directory: Path) -> Iterable[Tuple[Path, Optional[str]]]: + try: + entries = sorted( + directory.iterdir(), + key=lambda item: ( + unicodedata.normalize("NFC", item.name).casefold(), + unicodedata.normalize("NFC", item.name), + ), + ) + except OSError: + yield directory, "unreadable directory" + return + for entry in entries: + try: + relative = entry.relative_to(root) + info = entry.lstat() + if entry.is_symlink() or _is_reparse_point(info): + yield entry, "symlink skipped" + continue + if not _is_within(root, entry.resolve()): + yield entry, "path escapes vault" + continue + if any(part.startswith(".") for part in relative.parts): + yield entry, "hidden/configuration path skipped" + continue + if entry.is_dir(): + yield from _walk_vault(root, entry) + elif entry.is_file(): + yield entry, None + except OSError: + yield entry, "unreadable path" + + +def _is_within(root: Path, candidate: Path) -> bool: + try: + candidate.relative_to(root) + return True + except ValueError: + return False + + +def _split_frontmatter(content: str) -> Tuple[Dict[str, Any], str, List[str]]: + if not (content.startswith("---\n") or content == "---"): + return {}, content, [] + lines = content.splitlines(keepends=True) + closing = next((index for index in range(1, len(lines)) if lines[index].strip() in {"---", "..."}), None) + if closing is None: + return {}, content, ["unclosed YAML frontmatter treated as Markdown"] + frontmatter, warnings = _parse_simple_yaml("".join(lines[1:closing])) + return frontmatter, "".join(lines[closing + 1:]), warnings + + +def _parse_simple_yaml(source: str) -> Tuple[Dict[str, Any], List[str]]: + values: Dict[str, Any] = {} + warnings: List[str] = [] + active: Optional[str] = None + for number, raw in enumerate(source.splitlines(), 1): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + list_match = re.match(r"^\s+-\s+(.+?)\s*$", raw) + if list_match and active: + old = values.setdefault(active, []) + if not isinstance(old, list): + warnings.append("frontmatter line %d: mixed scalar/list ignored" % number) + else: + old.append(_yaml_scalar(list_match.group(1))) + continue + match = re.match(r"^([A-Za-z0-9_.-]+)\s*:\s*(.*?)\s*$", raw) + if not match: + warnings.append("frontmatter line %d: unsupported YAML construct" % number) + active = None + continue + key, value = match.groups() + active = key + values[key] = ([] if not value else + _yaml_list(value) if value.startswith("[") and value.endswith("]") else _yaml_scalar(value)) + return values, warnings + + +def _yaml_list(value: str) -> List[str]: + inside = value[1:-1].strip() + if not inside: + return [] + return [_yaml_scalar(part.strip()) for part in inside.split(",")] + + +def _yaml_scalar(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value + + +def _mask_code(text: str) -> str: + lines = text.splitlines(keepends=True) + masked: List[str] = [] + fence_char = "" + fence_size = 0 + for line in lines: + stripped = line.rstrip("\r\n") + start = _FENCE_START_RE.match(stripped) + if not fence_char and start: + token = start.group(1) + fence_char, fence_size = token[0], len(token) + masked.append(_blank_code(line)) + continue + if fence_char: + if re.match( + r"^[ \t]*%s{%d,}[ \t]*$" + % (re.escape(fence_char), fence_size), + stripped, + ): + fence_char, fence_size = "", 0 + masked.append(_blank_code(line)) + continue + masked.append(_mask_inline_code(line)) + return "".join(masked) + + +def _blank_code(value: str) -> str: + return "".join( + "\n" if char == "\n" else "\r" if char == "\r" else " " + for char in value + ) + + +def _mask_inline_code(line: str) -> str: + result = list(line) + index = 0 + while index < len(line): + if line[index] != "`": + index += 1 + continue + end = index + while end < len(line) and line[end] == "`": + end += 1 + token = line[index:end] + close = line.find(token, end) + stop = close + len(token) if close >= 0 else len(line.rstrip("\r\n")) + for cursor in range(index, stop): + if result[cursor] not in "\r\n": + result[cursor] = " " + index = stop if stop > index else end + return "".join(result) + + +def _links(text: str) -> List[ObsidianLink]: + found: List[ObsidianLink] = [] + for match in _WIKILINK_RE.finditer(text): + source = match.group(2).strip() + target_part, separator, display = source.partition("|") + target, heading, block_id = _split_link_target(target_part.strip()) + if target or heading or block_id: + found.append(ObsidianLink(target, display.strip() or None if separator else None, heading, block_id, bool(match.group(1)))) + return found + + +def _split_link_target(value: str) -> Tuple[str, Optional[str], Optional[str]]: + target, heading, block_id = value, None, None + if "#" in target: + target, heading = target.split("#", 1) + heading = heading.strip() or None + if "^" in target: + target, block_id = target.split("^", 1) + block_id = block_id.strip() or None + elif heading and "^" in heading: + heading, block_id = heading.split("^", 1) + heading, block_id = heading.strip() or None, block_id.strip() or None + return target.strip(), heading, block_id + + +def _attachments(text: str, links: List[ObsidianLink]) -> List[AttachmentReference]: + paths = [link.target for link in links if link.embedded and _is_attachment(link.target)] + paths.extend(match.group(1) for match in _MARKDOWN_ATTACHMENT_RE.finditer(text) if _is_attachment(match.group(1))) + return [AttachmentReference(path) for path in _dedupe(paths)] + + +def _is_attachment(value: str) -> bool: + return Path(value.split("#", 1)[0]).suffix.lower() in ATTACHMENT_SUFFIXES - {".md"} + + +def _first_h1(text: str) -> Optional[str]: + match = _H1_RE.search(text) + return match.group(1).strip() if match else None + + +def _string_values(value: Any) -> List[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if isinstance(value, str) and value.strip(): + return [value.strip()] + return [] + + +def _normalise_tags(values: Iterable[str]) -> List[str]: + return [value.strip().lstrip("#") for value in values if value.strip().lstrip("#")] + + +def _date_values(frontmatter: Dict[str, Any]) -> Dict[str, str]: + result: Dict[str, str] = {} + for key, value in frontmatter.items(): + if key.lower() in {"date", "created", "modified", "updated", "published"}: + text = _text_value(value) + if text: + result[key] = text + return result + + +def _text_value(value: Any) -> Optional[str]: + return value.strip() if isinstance(value, str) and value.strip() else None + + +def _dedupe(values: Iterable[str]) -> List[str]: + result: List[str] = [] + seen = set() + for value in values: + if value not in seen: + seen.add(value) + result.append(value) + return result + + +def _sensitive_filename(name: str) -> bool: + lowered = name.lower() + return (lowered in SENSITIVE_FILENAMES or lowered.startswith(".env.") or + lowered.endswith((".pem", ".key", ".p12", ".pfx")) or + bool(re.search(r"(?:credential|recovery[-_ ]?code|secret|token)", lowered))) + + +def _contains_secret(content: str) -> bool: + return secret_kind(content) is not None diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py index d3209bf2..4dc032fa 100644 --- a/engraphis/core/recall.py +++ b/engraphis/core/recall.py @@ -1117,15 +1117,23 @@ def _graph_arm( prompt_only=prompt_only, ) - def _prompt_eligible_memory_ids(self, memory_ids: set[str]) -> set[str]: - """Return only approved, non-quarantined memory nodes for prompt PPR.""" + def _prompt_eligible_memory_ids( + self, memory_ids: set[str], flt: Optional[SearchFilter] = None, + ) -> set[str]: + """Return prompt-safe memory nodes visible to the active read filter. + + Edge provenance is untrusted input. Its support ids must obey the same + hierarchy and bi-temporal visibility rules as ordinary recall, otherwise a + foreign or expired support can authorize an otherwise in-scope edge. + """ if not memory_ids: return set() records = self.store.get_memories(sorted(memory_ids)) return { memory_id for memory_id, record in records.items() - if prompt_eligible(record.provenance, record.metadata) + if (flt is None or memory_matches_filter(record, flt)) + and prompt_eligible(record.provenance, record.metadata) } @staticmethod @@ -1137,13 +1145,15 @@ def _edge_source_memory_ids(edge) -> set[str]: values.extend(many) return {str(value) for value in values if value} - def _prompt_eligible_edges(self, edges: list) -> list: + def _prompt_eligible_edges( + self, edges: list, flt: Optional[SearchFilter] = None, + ) -> list: """Keep trusted direct edges and memory-supported prompt-eligible edges.""" source_ids = ( set().union(*(self._edge_source_memory_ids(edge) for edge in edges)) if edges else set() ) - eligible_ids = self._prompt_eligible_memory_ids(source_ids) + eligible_ids = self._prompt_eligible_memory_ids(source_ids, flt) return [ edge for edge in edges if edge_provenance_prompt_eligible(edge.provenance) @@ -1220,7 +1230,7 @@ def connect(a: str, b: str, w: object, layer: GraphLayer) -> None: limit=edge_cap - len(edges_by_id), prompt_only=prompt_only, ) if prompt_only: - edges = self._prompt_eligible_edges(edges) + edges = self._prompt_eligible_edges(edges, flt) for edge in edges: if _positive_graph_weight(edge.weight) is None: continue @@ -1284,7 +1294,7 @@ def connect(a: str, b: str, w: object, layer: GraphLayer) -> None: ) } if prompt_only: - memory_ids = self._prompt_eligible_memory_ids(memory_ids) + memory_ids = self._prompt_eligible_memory_ids(memory_ids, flt) incidence = [ row for row in incidence if str(row.get("memory_id") or "") in memory_ids @@ -1360,7 +1370,7 @@ def _graph_arm_1hop( seed_ids, at=now, layers=flt.graph_layers, flt=flt, prompt_only=prompt_only, ) if prompt_only: - edges = self._prompt_eligible_edges(edges) + edges = self._prompt_eligible_edges(edges, flt) for edge in edges: if _positive_graph_weight(edge.weight) is not None: related_ids.add(edge.src) @@ -1372,7 +1382,7 @@ def _graph_arm_1hop( self._prompt_eligible_memory_ids({ str(row.get("memory_id") or "") for row in rows if row.get("memory_id") - }) + }, flt) if prompt_only else None ) out: dict[str, float] = {} diff --git a/engraphis/core/schema.py b/engraphis/core/schema.py index 4777b29a..c8832978 100644 --- a/engraphis/core/schema.py +++ b/engraphis/core/schema.py @@ -8,7 +8,7 @@ """ from __future__ import annotations -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -225,6 +225,7 @@ id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, repo_id TEXT, + session_id TEXT REFERENCES sessions(id) ON DELETE SET NULL, kind TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'queued', dry_run INTEGER NOT NULL DEFAULT 1, @@ -243,6 +244,25 @@ CREATE INDEX IF NOT EXISTS idx_jobs_scope_state ON jobs(workspace_id, kind, state, created_at); +-- A job may be session-targeted, but generic jobs remain backward-compatible +-- with a NULL session. SQLite FKs alone cannot assert that its optional repo +-- belongs to the same session, so retain the v2 scope relation at the boundary. +CREATE TRIGGER IF NOT EXISTS trg_job_session_scope_insert +BEFORE INSERT ON jobs WHEN NEW.session_id IS NOT NULL BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM sessions s WHERE s.id=NEW.session_id + AND s.workspace_id=NEW.workspace_id AND s.repo_id IS NEW.repo_id + ) THEN RAISE(ABORT, 'job session scope mismatch') END; +END; +CREATE TRIGGER IF NOT EXISTS trg_job_session_scope_update +BEFORE UPDATE OF workspace_id, repo_id, session_id ON jobs +WHEN NEW.session_id IS NOT NULL BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM sessions s WHERE s.id=NEW.session_id + AND s.workspace_id=NEW.workspace_id AND s.repo_id IS NEW.repo_id + ) THEN RAISE(ABORT, 'job session scope mismatch') END; +END; + CREATE TABLE IF NOT EXISTS graph_index_state ( workspace_id TEXT PRIMARY KEY, generation INTEGER NOT NULL DEFAULT 1, @@ -740,7 +760,10 @@ SELECT 1 FROM jobs j JOIN source_vaults v ON v.id=NEW.vault_id WHERE j.id=NEW.last_seen_job_id AND j.kind IN ('document_import','obsidian_import') + AND ((v.kind='documents' AND j.kind='document_import') + OR (v.kind='obsidian' AND j.kind='obsidian_import')) AND j.workspace_id=v.workspace_id AND j.repo_id IS v.repo_id + AND j.session_id IS v.session_id ) THEN RAISE(ABORT, 'source import seen-job scope mismatch') END; END; CREATE TRIGGER IF NOT EXISTS trg_source_import_seen_job_update @@ -750,7 +773,10 @@ SELECT 1 FROM jobs j JOIN source_vaults v ON v.id=NEW.vault_id WHERE j.id=NEW.last_seen_job_id AND j.kind IN ('document_import','obsidian_import') + AND ((v.kind='documents' AND j.kind='document_import') + OR (v.kind='obsidian' AND j.kind='obsidian_import')) AND j.workspace_id=v.workspace_id AND j.repo_id IS v.repo_id + AND j.session_id IS v.session_id ) THEN RAISE(ABORT, 'source import seen-job scope mismatch') END; END; @@ -766,7 +792,10 @@ JOIN jobs j ON j.id=NEW.job_id WHERE i.id=NEW.source_id AND j.kind IN ('document_import','obsidian_import') + AND ((v.kind='documents' AND j.kind='document_import') + OR (v.kind='obsidian' AND j.kind='obsidian_import')) AND j.workspace_id=v.workspace_id AND j.repo_id IS v.repo_id + AND j.session_id IS v.session_id ) AND NEW.source_id IS NOT NULL THEN RAISE(ABORT, 'source import job scope mismatch') END; END; @@ -782,7 +811,10 @@ JOIN jobs j ON j.id=NEW.job_id WHERE i.id=NEW.source_id AND j.kind IN ('document_import','obsidian_import') + AND ((v.kind='documents' AND j.kind='document_import') + OR (v.kind='obsidian' AND j.kind='obsidian_import')) AND j.workspace_id=v.workspace_id AND j.repo_id IS v.repo_id + AND j.session_id IS v.session_id ) AND NEW.source_id IS NOT NULL THEN RAISE(ABORT, 'source import job scope mismatch') END; END; diff --git a/engraphis/core/store.py b/engraphis/core/store.py index 90a5cc2b..47112313 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -17,16 +17,18 @@ import re import sqlite3 import stat +import tempfile import threading import time import unicodedata import weakref from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable, Iterable, Optional +from typing import Any, Callable, Iterable, Optional, Protocol import numpy as np +from engraphis.private_state import ensure_owner_private_dir from engraphis.core import ids from engraphis.core.graph_layers import infer_graph_layer, normalize_graph_layer from engraphis.core.interfaces import ( @@ -50,6 +52,7 @@ pending_llm_consolidation_envelope, pending_llm_extraction_envelope, ) +from engraphis.core.documents import normalize_document_path from engraphis.core.retention_policy import ( DEFAULT_STABILITY_DAYS, MAX_ACCESS_COUNT, @@ -88,6 +91,13 @@ TOMBSTONE_NEVER_EXPORT, TOMBSTONE_REMOTE_ERASURE, }) +_SOURCE_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_IMPORT_RECEIPT_COUNT_KEYS = frozenset({ + "files_scanned", "files_imported", "files_updated", "files_skipped", + "files_renamed", "files_rejected", "files_missing", "files_errored", + "conflicts", "warnings", "attachments", "wikilinks", "aliases", "tags", +}) +_MAX_RECEIPT_COUNT = 1_000_000_000 USER_SCOPE_UNSUPPORTED = ( "user scope is not supported until owner-aware memories are implemented; " "use workspace, repo, or session" @@ -100,6 +110,17 @@ def now_ts() -> float: return time.time() +def _content_free_source_error(value: Any) -> str: + """Persist an error code or one-way digest, never a source-derived message.""" + raw = str(value or "") + normalized = raw.strip().casefold().replace(" ", "_") + if not normalized: + return "" + if re.fullmatch(r"[a-z0-9_.:-]{1,100}", normalized): + return normalized + return "sha256:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() + + def _escape_like(value: str) -> str: """Escape LIKE wildcards so ``%``/``_``/``\\`` in user input match literally. @@ -133,6 +154,22 @@ def _close_connection_quietly(conn: Any) -> None: pass +class ReadOnlyConnector(Protocol): + """Explicit contract for injected connectors used by a read-only ``Store``. + + Writable compatibility remains the ordinary ``connector(path)`` call. A + connector that also supports inspection must expose ``open_read_only(path)`` + and open the already-existing regular file without creating, recovering, or + mutating the database or any sidecar. Implementations should use their + driver's equivalent of SQLite ``mode=ro&immutable=1``. Store rejects a bare + callable in read-only mode rather than guessing that it is safe. + """ + + def __call__(self, path: str) -> Any: ... + + def open_read_only(self, path: str) -> Any: ... + + def _row_is_prompt_eligible(provenance: Any, metadata: Any) -> bool: """Use the one trust predicate before exposing a derived bridge. @@ -356,7 +393,10 @@ def _receipt_metadata(metadata: dict) -> dict: allowed = { "mtype", "scope", "resolution", "retention", "extracted", "intent", "k", "result_count", "grounded", "citations", "relation", "layer", "graph_layers", - "files_scanned", "files_indexed", "files_removed", "symbols", "edges", + "files_scanned", "files_indexed", "files_removed", "files_imported", + "files_updated", "files_renamed", "files_skipped", "files_rejected", + "files_missing", "files_errored", "conflicts", "warnings", + "attachments", "wikilinks", "aliases", "tags", "symbols", "edges", "entities", "relations", "tables", "dry_run", "error_count", "entities_added", "relations_added", "retrieval_profile", "candidate_depth", "candidate_k_requested", @@ -375,7 +415,13 @@ def content_free_label(key: str, value: str) -> str: if safe_key not in allowed: continue value = metadata[key] - if safe_key == "token_usage": + if safe_key in _IMPORT_RECEIPT_COUNT_KEYS: + # Import summaries are counts, never arbitrary floats/labels. Reject + # booleans and clamp adversarially large values so the durable public + # receipt stays predictable and bounded. + if type(value) is int: + out[safe_key] = max(0, min(_MAX_RECEIPT_COUNT, value)) + elif safe_key == "token_usage": if not isinstance(value, dict): continue numeric = { @@ -429,6 +475,9 @@ def content_free_label(key: str, value: str) -> str: "mtype", "scope", "resolution", "retention", "extracted", "intent", "k", "result_count", "grounded", "citations", "relation", "layer", "graph_layers", "files_scanned", "files_indexed", "files_removed", "symbols", "edges", + "files_imported", "files_updated", "files_renamed", "files_skipped", + "files_rejected", "files_missing", "files_errored", "conflicts", + "warnings", "attachments", "wikilinks", "aliases", "tags", "entities", "relations", "tables", "dry_run", "error_count", "entities_added", "relations_added", "retrieval_profile", "candidate_depth", "candidate_k_requested", "candidate_k_used", "response_mode", "historical", @@ -437,12 +486,12 @@ def content_free_label(key: str, value: str) -> str: _PUBLIC_RECEIPT_OPERATIONS = { "remember", "recall", "promote", "link", "index_repo", "graph_index", "grounded_recall", "adaptive_context", "proactive_context", "smart_gateway", - "consolidate", "sync", + "consolidate", "sync", "document_import", "obsidian_import", } _PUBLIC_RECEIPT_STATUSES = { "ok", "add", "noop", "invalidate", "relate", "ingested", "postgres_schema", "grounded", "abstained", "promoted", - "indexed", "skipped", "error", "failed", "cancelled", "partial", + "indexed", "skipped", "error", "failed", "completed", "cancelled", "partial", } @@ -546,7 +595,10 @@ def safe_hash(value: Any, *, allow_empty: bool = False) -> str: ): return invalid for key, value in metadata.items(): - if key == "token_usage": + if key in _IMPORT_RECEIPT_COUNT_KEYS: + if type(value) is not int or not 0 <= value <= _MAX_RECEIPT_COUNT: + return invalid + elif key == "token_usage": if not isinstance(value, dict): return invalid allowed_usage = { @@ -1017,23 +1069,30 @@ def __init__(self, path: str = ":memory:", *, writer: it opens a checkpointed SQLite file with ``mode=ro&immutable=1`` and skips schema setup, migrations, backups, and the persistent WAL-mode pragma. It is for inspection tools (notably security dry-runs) whose safety contract - includes leaving a database and its sidecar files untouched. A non-empty WAL - is rejected rather than silently scanning an incomplete immutable snapshot. + includes leaving a database and its sidecar files untouched. Non-empty WAL and + rollback-journal sidecars are rejected rather than silently scanning an + incomplete immutable snapshot. An injected connector must implement the + :class:`ReadOnlyConnector` ``open_read_only(path)`` contract; a bare writable + callable is rejected before it can be invoked. """ self.path = path self._connect = connect self.read_only = bool(read_only) if self.read_only and path == ":memory:": raise ValueError("read-only Store requires an existing database file") - if self.read_only and self._connect is None: - wal_path = Path(f"{path}-wal") - if wal_path.is_file() and wal_path.stat().st_size: - raise RuntimeError( - "read-only Store requires a checkpointed database; active WAL found" + read_only_path: Optional[str] = None + if self.read_only: + if self._connect is not None and not callable( + getattr(self._connect, "open_read_only", None) + ): + raise TypeError( + "read-only Store requires an injected connector with an " + "open_read_only(path) method" ) + read_only_path = self._preflight_read_only_path(path) if path != ":memory:" and not self.read_only: Path(path).parent.mkdir(parents=True, exist_ok=True) - raw_conn = self._open_connection(path) + raw_conn = self._open_connection(read_only_path or path) # Serialize the shared connection so concurrent threadpool handlers can't interleave # transactions on it (see _SerializedConnection). All Store/service/backend access # goes through self.conn, so wrapping here covers every writer. @@ -1050,9 +1109,9 @@ def __init__(self, path: str = ":memory:", *, try: self.conn.execute("PRAGMA foreign_keys=ON") if self.read_only: - # ``query_only`` also protects injected connectors whose implementation - # cannot express SQLite's URI ``mode=ro`` option. Do not probe FTS5 by - # creating a temporary table here: a dry-run must not write anything. + # Defense in depth after the stdlib immutable URI or the injected + # connector's explicit immutable-open contract. Do not probe FTS5 by + # creating a temporary table here: an inspection must not write anything. self.conn.execute("PRAGMA query_only=ON") self._validate_read_only_ready() row = self.conn.execute( @@ -1087,6 +1146,8 @@ def _open_connection(self, path: str): if self._connect is not None: # Injected factories own opening, keying, row_factory, and exception # translation (notably the SQLCipher backend). + if self.read_only: + return self._connect.open_read_only(path) # type: ignore[attr-defined] return self._connect(path) if self.read_only: uri = Path(path).resolve().as_uri() + "?mode=ro&immutable=1" @@ -1096,6 +1157,54 @@ def _open_connection(self, path: str): conn.row_factory = sqlite3.Row return conn + @staticmethod + def _preflight_read_only_path(path: str) -> str: + """Validate one immutable snapshot path without creating or opening it. + + The resolved regular file is returned so a connector cannot reinterpret a + relative path after validation. Active WAL/rollback-journal state is + refused because an immutable connection would skip recovery and silently + expose an incomplete snapshot. + """ + candidate = Path(path) + try: + info = os.lstat(candidate) + except OSError: + raise RuntimeError( + "read-only Store requires an existing regular database file" + ) from None + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + file_attributes = int(getattr(info, "st_file_attributes", 0)) + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISREG(info.st_mode) + or bool(reparse_flag and file_attributes & reparse_flag) + ): + raise RuntimeError( + "read-only Store requires an existing regular database file" + ) + for suffix, label in (("-wal", "WAL"), ("-journal", "rollback journal")): + sidecar = Path(f"{candidate}{suffix}") + try: + sidecar_info = os.lstat(sidecar) + except FileNotFoundError: + continue + except OSError: + raise RuntimeError( + f"read-only Store could not validate the {label} sidecar" + ) from None + if sidecar_info.st_size: + raise RuntimeError( + "read-only Store requires a checkpointed database; " + f"active {label} found" + ) + try: + return str(candidate.resolve(strict=True)) + except OSError: + raise RuntimeError( + "read-only Store requires an existing regular database file" + ) from None + def _validate_read_only_ready(self) -> None: """Fail closed unless the immutable snapshot can serve the current schema.""" required = { @@ -1125,6 +1234,55 @@ def _validate_read_only_ready(self) -> None: "read-only Store requires a complete current schema; missing " + ", ".join(missing) ) + source_security_objects = { + str(row["name"]) + for row in self.conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('index','trigger')" + ).fetchall() + } + required_source_security_objects = { + "trg_job_session_scope_insert", + "trg_job_session_scope_update", + "idx_source_vaults_identity", + "trg_source_vault_scope_insert", + "trg_source_vault_scope_update", + "trg_source_import_scope_insert", + "trg_source_import_scope_update", + "trg_source_import_seen_job_insert", + "trg_source_import_seen_job_update", + "trg_source_import_job_insert", + "trg_source_import_job_update", + } + missing_source_security = sorted( + required_source_security_objects - source_security_objects + ) + if missing_source_security: + raise RuntimeError( + "read-only Store requires a complete current schema; missing " + + ", ".join(missing_source_security) + ) + job_columns = { + str(item["name"]) + for item in self.conn.execute("PRAGMA table_info(jobs)").fetchall() + } + if "session_id" not in job_columns: + raise RuntimeError( + "read-only Store requires jobs.session_id for source import scope isolation" + ) + source_vault_foreign_keys = { + (str(row["from"]), str(row["table"]), str(row["to"])) + for row in self.conn.execute( + "PRAGMA foreign_key_list(source_vaults)" + ).fetchall() + } + if not { + ("workspace_id", "workspaces", "id"), + ("repo_id", "repos", "id"), + ("session_id", "sessions", "id"), + }.issubset(source_vault_foreign_keys): + raise RuntimeError( + "read-only Store requires source_vaults scope foreign keys" + ) session_columns = { str(item["name"]) for item in self.conn.execute( "PRAGMA table_info(sessions)" @@ -1175,6 +1333,45 @@ def _validate_read_only_ready(self) -> None: "read-only Store requires a complete current schema; missing " "memory_tombstones.export_class" ) + source_import_columns = { + str(item["name"]) for item in self.conn.execute( + "PRAGMA table_info(source_imports)" + ).fetchall() + } + required_source_import_columns = { + "vault_id", "source_key", "relative_path", "memory_id", + "subject_key", "content_sha256", "canonical_sha256", + "last_seen_job_id", "state", "last_seen_at", + } + missing_source_import_columns = sorted( + required_source_import_columns - source_import_columns + ) + if missing_source_import_columns: + raise RuntimeError( + "read-only Store requires a complete current schema; missing " + + ", ".join( + f"source_imports.{name}" for name in missing_source_import_columns + ) + ) + source_item_columns = { + str(item["name"]) for item in self.conn.execute( + "PRAGMA table_info(source_import_items)" + ).fetchall() + } + required_source_item_columns = { + "job_id", "source_id", "relative_path", "planned_action", + "source_format", "result_state", "warning_count", "error_code", + } + missing_source_item_columns = sorted( + required_source_item_columns - source_item_columns + ) + if missing_source_item_columns: + raise RuntimeError( + "read-only Store requires a complete current schema; missing " + + ", ".join( + f"source_import_items.{name}" for name in missing_source_item_columns + ) + ) row = self.conn.execute( "SELECT MAX(version) AS version FROM schema_migrations" ).fetchone() @@ -1187,6 +1384,233 @@ def _validate_read_only_ready(self) -> None: if not self._quick_check(self.conn): raise sqlite3.DatabaseError("read-only Store integrity check failed") + @classmethod + def snapshot_source_import_manifest( + cls, path: str, *, connect: Optional[ReadOnlyConnector] = None, + ) -> dict: + """Read an importer manifest without migrating or changing a database. + + This is deliberately usable by previews against an older v13 database (and + against a not-yet-created database). Active WAL and rollback-journal state is + refused: immutable reads would otherwise silently miss recovered manifest + state. Injected connectors must opt in through ``open_read_only(path)``; + ordinary writable callables are never invoked here. + """ + empty = {"schema_version": 0, "vaults": [], "items": []} + if path in (":memory:", ""): + return empty + db_path = Path(path) + try: + os.lstat(db_path) + except FileNotFoundError: + return empty + except OSError: + raise RuntimeError( + "import manifest database could not be inspected" + ) from None + if connect is not None and not callable( + getattr(connect, "open_read_only", None) + ): + raise TypeError( + "import manifest snapshot requires an injected connector with an " + "open_read_only(path) method" + ) + resolved_path = cls._preflight_read_only_path(path) + db_path = Path(resolved_path) + + def sidecar_state() -> dict[str, Optional[tuple[int, int, int, int]]]: + state: dict[str, Optional[tuple[int, int, int, int]]] = {} + for suffix in ("-wal", "-journal", "-shm"): + sidecar = Path(f"{db_path}{suffix}") + try: + info = os.lstat(sidecar) + except FileNotFoundError: + state[suffix] = None + except OSError: + raise RuntimeError( + "import manifest database sidecars could not be inspected" + ) from None + else: + state[suffix] = ( + int(info.st_dev), int(info.st_ino), int(info.st_size), + int(info.st_mtime_ns), + ) + return state + + version_fields = ( + ("st_size", "st_mtime_ns") + if os.name == "nt" + else ("st_size", "st_mtime_ns", "st_ctime_ns") + ) + + def same_version(left, right) -> bool: + return cls._same_file(left, right) and all( + getattr(left, name, None) == getattr(right, name, None) + for name in version_fields + ) + + try: + before = os.lstat(db_path) + before_sidecars = sidecar_state() + except OSError: + raise RuntimeError( + "import manifest database could not be inspected" + ) from None + + source_flags = ( + os.O_RDONLY | getattr(os, "O_BINARY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + try: + source_descriptor = os.open(str(db_path), source_flags) + except OSError: + raise RuntimeError( + "import manifest database could not be opened safely" + ) from None + try: + opened = os.fstat(source_descriptor) + attributes = int(getattr(opened, "st_file_attributes", 0)) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + if ( + not stat.S_ISREG(opened.st_mode) + or bool(reparse_flag and attributes & reparse_flag) + or not same_version(before, opened) + ): + raise RuntimeError( + "import manifest database changed while it was opened" + ) + + with tempfile.TemporaryDirectory( + prefix="engraphis-manifest-snapshot-", + ) as temp_root: + temp_directory = Path(temp_root) + ensure_owner_private_dir(temp_directory) + temp_path = temp_directory / "manifest.db" + output_flags = ( + os.O_WRONLY | os.O_CREAT | os.O_EXCL + | getattr(os, "O_BINARY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + output_descriptor = os.open(str(temp_path), output_flags, 0o600) + try: + fchmod = getattr(os, "fchmod", None) + if fchmod is not None: + fchmod(output_descriptor, 0o600) + while True: + chunk = os.read(source_descriptor, 1024 * 1024) + if not chunk: + break + view = memoryview(chunk) + while view: + written = os.write(output_descriptor, view) + if written <= 0: + raise OSError("private snapshot copy made no progress") + view = view[written:] + finally: + os.close(output_descriptor) + + try: + after_copy = os.fstat(source_descriptor) + current = os.lstat(db_path) + current_sidecars = sidecar_state() + except OSError: + raise RuntimeError( + "import manifest database changed during snapshot" + ) from None + if ( + not same_version(opened, after_copy) + or not same_version(after_copy, current) + or before_sidecars != current_sidecars + ): + raise RuntimeError( + "import manifest database changed during snapshot" + ) + + if connect is not None: + conn = connect.open_read_only(str(temp_path)) + else: + uri = temp_path.as_uri() + "?mode=ro&immutable=1" + conn = sqlite3.connect(uri, uri=True) + conn.row_factory = sqlite3.Row + try: + conn.execute("PRAGMA query_only=ON") + tables = {str(row[0]) for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall()} + version = 0 + if "schema_migrations" in tables: + row = conn.execute( + "SELECT MAX(version) FROM schema_migrations" + ).fetchone() + version = int(row[0]) if row and row[0] is not None else 0 + if not {"source_vaults", "source_imports"}.issubset(tables): + result = { + "schema_version": version, "vaults": [], "items": [], + } + else: + vaults = [dict(row) for row in conn.execute( + "SELECT v.id, v.kind, v.root_digest, v.display_name, " + "v.workspace_id, v.repo_id, v.session_id, v.scope, " + "v.memory_type, v.importer_version, v.created_at, " + "v.updated_at, w.name AS workspace_name, " + "r.name AS repo_name FROM source_vaults v " + "JOIN workspaces w ON w.id=v.workspace_id " + "LEFT JOIN repos r ON r.id=v.repo_id ORDER BY v.id" + ).fetchall()] + items = [dict(row) for row in conn.execute( + "SELECT id, vault_id, source_key, relative_path, " + "memory_id, subject_key, content_sha256, " + "canonical_sha256, file_mtime_ns, file_size, " + "importer_version, last_seen_job_id, state, " + "first_imported_at, last_imported_at, last_seen_at, " + "missing_at, last_error FROM source_imports " + "ORDER BY vault_id, relative_path" + ).fetchall()] + repos = [] + sessions = [] + if {"repos", "workspaces"}.issubset(tables): + repos = [dict(row) for row in conn.execute( + "SELECT r.id, r.workspace_id, r.name, " + "w.name AS workspace_name FROM repos r " + "JOIN workspaces w ON w.id=r.workspace_id " + "ORDER BY r.id" + ).fetchall()] + if "sessions" in tables: + sessions = [dict(row) for row in conn.execute( + "SELECT s.id, s.workspace_id, s.repo_id, " + "w.name AS workspace_name, " + "r.name AS repo_name FROM sessions s " + "JOIN workspaces w ON w.id=s.workspace_id " + "LEFT JOIN repos r ON r.id=s.repo_id " + "ORDER BY s.id" + ).fetchall()] + result = { + "schema_version": version, "vaults": vaults, + "items": items, "repos": repos, "sessions": sessions, + } + finally: + conn.close() + + try: + after = os.lstat(db_path) + after_handle = os.fstat(source_descriptor) + after_sidecars = sidecar_state() + except OSError: + raise RuntimeError( + "import manifest database changed during snapshot" + ) from None + if ( + not same_version(opened, after_handle) + or not same_version(after_handle, after) + or before_sidecars != after_sidecars + ): + raise RuntimeError( + "import manifest database changed during snapshot" + ) + return result + finally: + os.close(source_descriptor) + @staticmethod def _raw_connection(conn): """Unwrap core/backend adapters for sqlite3's type-checked backup API.""" @@ -1399,6 +1823,202 @@ def _execute_script_transactional(self, script: str) -> None: if statement.strip(): raise sqlite3.OperationalError("incomplete schema statement") + def _prepare_source_manifest_v15(self, previous_version: int) -> bool: + """Stage the v14 Obsidian-only manifest for a source-neutral rebuild. + + SQLite cannot widen a table ``CHECK`` constraint in place. Keep the + complete content-free manifest in temporary tables, drop children before + parents, let :data:`SCHEMA_SQL` create the v15 shape, then restore it in + the same migration transaction. A failure rolls the primary database + back to v14; the durable pre-migration backup remains the final fallback. + """ + if previous_version >= 15: + return False + row = self.conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='source_vaults'" + ).fetchone() + if row is None: + return False + definition = str(row["sql"] or "") + if "'documents'" in definition: + return False + for name in ( + "_source_vaults_v15", "_source_imports_v15", "_source_import_items_v15", + ): + self.conn.execute(f"DROP TABLE IF EXISTS temp.{name}") + self.conn.execute( + "CREATE TEMP TABLE _source_vaults_v15 AS SELECT * FROM source_vaults" + ) + self.conn.execute( + "CREATE TEMP TABLE _source_imports_v15 AS SELECT * FROM source_imports" + ) + self.conn.execute( + "CREATE TEMP TABLE _source_import_items_v15 AS SELECT * FROM source_import_items" + ) + self.conn.execute("DROP TABLE source_import_items") + self.conn.execute("DROP TABLE source_imports") + self.conn.execute("DROP TABLE source_vaults") + return True + + def _prepare_job_session_scope_v16(self) -> None: + """Install the nullable job session target before v16 triggers compile. + + ``CREATE TABLE IF NOT EXISTS`` cannot add the column to an existing v15 + jobs table, while the current source-manifest triggers reference it. Do + the additive, idempotent repair inside the surrounding migration + transaction before executing :data:`SCHEMA_SQL`. When v14 source + manifests are staged, also recover the session target for their jobs: + those legacy jobs predate ``jobs.session_id`` but their source vaults + already carry the authoritative session scope. + """ + row = self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs'" + ).fetchone() + if row is None: + return + columns = { + str(item["name"]) + for item in self.conn.execute("PRAGMA table_info(jobs)").fetchall() + } + if "session_id" not in columns: + self.conn.execute( + "ALTER TABLE jobs ADD COLUMN session_id TEXT " + "REFERENCES sessions(id) ON DELETE SET NULL" + ) + staged_tables = { + str(item["name"]) + for item in self.conn.execute( + "SELECT name FROM temp.sqlite_master WHERE type='table'" + ).fetchall() + } + required_staged_tables = { + "_source_vaults_v15", "_source_imports_v15", "_source_import_items_v15", + } + if not required_staged_tables.issubset(staged_tables): + # v15 (and later) databases carry the live source manifest rather than + # staged temp tables, but their jobs still predate ``jobs.session_id``. + # Backfill from the live v15 tables so the exact-session triggers added + # below never reject persisted lineage for legacy jobs. + self._backfill_job_session_scope_from_tables( + "source_vaults", "source_imports", "source_import_items", + ) + return + + # A legacy import job can be referenced by either the source manifest's + # last-seen pointer or an item row. UNION makes duplicate references + # harmless while retaining a deterministic conflict check if corrupted + # data points one job at more than one session vault. + candidates = self.conn.execute( + "SELECT job_id, session_id FROM (" + "SELECT i.last_seen_job_id AS job_id, v.session_id " + "FROM temp._source_imports_v15 i " + "JOIN temp._source_vaults_v15 v ON v.id=i.vault_id " + "WHERE i.last_seen_job_id IS NOT NULL AND v.session_id IS NOT NULL " + "UNION " + "SELECT item.job_id, v.session_id " + "FROM temp._source_import_items_v15 item " + "JOIN temp._source_imports_v15 i ON i.id=item.source_id " + "JOIN temp._source_vaults_v15 v ON v.id=i.vault_id " + "WHERE item.job_id IS NOT NULL AND v.session_id IS NOT NULL" + ") ORDER BY job_id, session_id" + ).fetchall() + self._apply_job_session_backfill(candidates) + + def _backfill_job_session_scope_from_tables( + self, vaults_table: str, imports_table: str, items_table: str, + ) -> None: + """Backfill ``jobs.session_id`` from a (temp or live) source manifest. + + Shared by the staged v14 path and the live v15 path so legacy jobs that + predate ``jobs.session_id`` keep their authoritative vault session scope + before the v16 exact-session triggers compile. + """ + try: + vault_exists = self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + (vaults_table,), + ).fetchone() is not None + imports_exists = self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + (imports_table,), + ).fetchone() is not None + items_exists = self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + (items_table,), + ).fetchone() is not None + except Exception: # noqa: BLE001 — a temp table may not be readable here + return + if not (vault_exists and imports_exists and items_exists): + return + candidates = self.conn.execute( + "SELECT job_id, session_id FROM (" + "SELECT i.last_seen_job_id AS job_id, v.session_id " + f"FROM {imports_table} i " + f"JOIN {vaults_table} v ON v.id=i.vault_id " + "WHERE i.last_seen_job_id IS NOT NULL AND v.session_id IS NOT NULL " + "UNION " + "SELECT item.job_id, v.session_id " + f"FROM {items_table} item " + f"JOIN {imports_table} i ON i.id=item.source_id " + f"JOIN {vaults_table} v ON v.id=i.vault_id " + "WHERE item.job_id IS NOT NULL AND v.session_id IS NOT NULL" + ") ORDER BY job_id, session_id" + ).fetchall() + self._apply_job_session_backfill(candidates) + + def _apply_job_session_backfill(self, candidates) -> None: + """Apply a job→session backfill with the same conflict rules as the staged path.""" + by_job: dict[str, str] = {} + for row in candidates: + job_id = str(row["job_id"]) + session_id = str(row["session_id"]) + prior = by_job.get(job_id) + if prior is not None and prior != session_id: + raise RuntimeError( + "cannot backfill job session scope: staged job maps to " + "multiple source sessions" + ) + by_job[job_id] = session_id + + for job_id, session_id in by_job.items(): + current = self.conn.execute( + "SELECT session_id FROM jobs WHERE id=?", (job_id,) + ).fetchone() + if current is None: + continue + current_session = current["session_id"] + if current_session is not None and str(current_session) != session_id: + raise RuntimeError( + "cannot backfill job session scope: existing job session " + "conflicts with its source vault" + ) + if current_session is None: + self.conn.execute( + "UPDATE jobs SET session_id=? WHERE id=?", + (session_id, job_id), + ) + + def _restore_source_manifest_v15(self) -> None: + """Restore source identities staged by :meth:`_prepare_source_manifest_v15`.""" + self.conn.execute( + "INSERT INTO source_vaults SELECT * FROM temp._source_vaults_v15" + ) + self.conn.execute( + "INSERT INTO source_imports SELECT * FROM temp._source_imports_v15" + ) + self.conn.execute( + "INSERT INTO source_import_items(" + "id,job_id,source_id,relative_path,planned_action,result_state," + "warning_count,error_code,created_at,finished_at) " + "SELECT id,job_id,source_id,relative_path,planned_action,result_state," + "warning_count,error_code,created_at,finished_at " + "FROM temp._source_import_items_v15" + ) + for name in ( + "_source_import_items_v15", "_source_imports_v15", "_source_vaults_v15", + ): + self.conn.execute(f"DROP TABLE temp.{name}") + # ── schema ────────────────────────────────────────────────────────────── def init_schema(self) -> None: objects = self.conn.execute( @@ -1432,94 +2052,10 @@ def init_schema(self) -> None: str(row["name"]) for row in self.conn.execute("PRAGMA table_info(memories)").fetchall() } - memories_need_expired_at = ( - "memories" in object_names and "expired_at" not in memory_columns - ) - memories_need_valid_to = ( - "memories" in object_names and "valid_to" not in memory_columns - ) memories_need_modified_hlc = ( "memories" in object_names and "modified_hlc" not in memory_columns ) self._memories_need_modified_hlc = memories_need_modified_hlc - self._memories_need_valid_to = memories_need_valid_to - self._memories_need_expired_at = memories_need_expired_at - memory_entity_columns: set[str] = set() - if "memory_entities" in object_names: - memory_entity_columns = { - str(row["name"]) - for row in self.conn.execute( - "PRAGMA table_info(memory_entities)" - ).fetchall() - } - self._memory_entities_need_valid_to = ( - "memory_entities" in object_names - and "valid_to" not in memory_entity_columns - ) - self._memory_entities_need_expired_at = ( - "memory_entities" in object_names - and "expired_at" not in memory_entity_columns - ) - edge_columns: set[str] = set() - if "edges" in object_names: - edge_columns = { - str(row["name"]) - for row in self.conn.execute("PRAGMA table_info(edges)").fetchall() - } - self._edges_need_valid_to = ( - "edges" in object_names and "valid_to" not in edge_columns - ) - self._edges_need_expired_at = ( - "edges" in object_names and "expired_at" not in edge_columns - ) - edge_support_columns: set[str] = set() - if "edge_supports" in object_names: - edge_support_columns = { - str(row["name"]) - for row in self.conn.execute( - "PRAGMA table_info(edge_supports)" - ).fetchall() - } - self._edge_supports_need_valid_to = ( - "edge_supports" in object_names - and "valid_to" not in edge_support_columns - ) - self._edge_supports_need_expired_at = ( - "edge_supports" in object_names - and "expired_at" not in edge_support_columns - ) - code_file_history_columns: set[str] = set() - if "code_file_history" in object_names: - code_file_history_columns = { - str(row["name"]) - for row in self.conn.execute( - "PRAGMA table_info(code_file_history)" - ).fetchall() - } - self._code_file_history_need_valid_to = ( - "code_file_history" in object_names - and "valid_to" not in code_file_history_columns - ) - self._code_file_history_need_expired_at = ( - "code_file_history" in object_names - and "expired_at" not in code_file_history_columns - ) - code_memory_link_columns: set[str] = set() - if "code_memory_links" in object_names: - code_memory_link_columns = { - str(row["name"]) - for row in self.conn.execute( - "PRAGMA table_info(code_memory_links)" - ).fetchall() - } - self._code_memory_links_need_valid_to = ( - "code_memory_links" in object_names - and "valid_to" not in code_memory_link_columns - ) - self._code_memory_links_need_expired_at = ( - "code_memory_links" in object_names - and "expired_at" not in code_memory_link_columns - ) session_columns: set[str] = set() if "sessions" in object_names: session_columns = { @@ -1553,19 +2089,7 @@ def init_schema(self) -> None: needs_backup = bool(object_names) and ( previous_version < SCHEMA_VERSION or mem_links_need_temporal_backfill - or memories_need_valid_to - or memories_need_expired_at or memories_need_modified_hlc - or getattr(self, "_memory_entities_need_valid_to", False) - or getattr(self, "_memory_entities_need_expired_at", False) - or getattr(self, "_edges_need_valid_to", False) - or getattr(self, "_edges_need_expired_at", False) - or getattr(self, "_edge_supports_need_valid_to", False) - or getattr(self, "_edge_supports_need_expired_at", False) - or getattr(self, "_code_file_history_need_valid_to", False) - or getattr(self, "_code_file_history_need_expired_at", False) - or getattr(self, "_code_memory_links_need_valid_to", False) - or getattr(self, "_code_memory_links_need_expired_at", False) or sessions_need_handoff or tombstones_need_export_class or sync_exports_need_table @@ -1593,68 +2117,61 @@ def _apply_schema(self, previous_version: int) -> None: "PRAGMA table_info(operation_receipts)" ).fetchall() ) - for table, needs_expired_at in ( - ( - "memories", - getattr(self, "_memories_need_valid_to", False) - or getattr(self, "_memories_need_expired_at", False), - ), - ( - "memory_entities", - getattr(self, "_memory_entities_need_valid_to", False) - or getattr(self, "_memory_entities_need_expired_at", False), - ), - ( - "edges", - getattr(self, "_edges_need_valid_to", False) - or getattr(self, "_edges_need_expired_at", False), - ), - ( - "edge_supports", - getattr(self, "_edge_supports_need_valid_to", False) - or getattr(self, "_edge_supports_need_expired_at", False), - ), - ( - "code_file_history", - getattr(self, "_code_file_history_need_valid_to", False) - or getattr(self, "_code_file_history_need_expired_at", False), - ), - ( - "code_memory_links", - getattr(self, "_code_memory_links_need_valid_to", False) - or getattr(self, "_code_memory_links_need_expired_at", False), - ), + restore_source_manifest = self._prepare_source_manifest_v15(previous_version) + self._prepare_job_session_scope_v16() + if previous_version < 16: + # v15 source-job triggers bound only workspace/repo because generic + # jobs did not yet persist a session target. Recreate them through + # SCHEMA_SQL after installing ``jobs.session_id`` above. + for name in ( + "trg_source_import_seen_job_insert", + "trg_source_import_seen_job_update", + "trg_source_import_job_insert", + "trg_source_import_job_update", + ): + self.conn.execute(f"DROP TRIGGER IF EXISTS {name}") + # A partially upgraded v2 database can be missing one of the temporal + # columns that SCHEMA_SQL indexes. Repair those columns before running + # the script; SQLite parses CREATE INDEX before the additive ALTER loop + # below gets a chance to run. Keep ``valid_from`` out of this repair so + # the v4 code-link migration can still recognize and rebuild its legacy + # table-level uniqueness shape. + existing_tables = { + str(row[0]) + for row in self.conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + } + for table in ( + "memories", "memory_entities", "edges", "edge_supports", + "code_file_history", "code_memory_links", ): - if needs_expired_at: - try: - self.conn.execute(f"ALTER TABLE {table} ADD COLUMN valid_to REAL") - except sqlite3.OperationalError: - pass - self.conn.execute(f"ALTER TABLE {table} ADD COLUMN expired_at REAL") + if table not in existing_tables: + continue + columns = { + str(row[1]) + for row in self.conn.execute(f"PRAGMA table_info({table})").fetchall() + } + for column in ("valid_to", "expired_at"): + if column not in columns: + self.conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} REAL") self._execute_script_transactional(SCHEMA_SQL) + if restore_source_manifest: + self._restore_source_manifest_v15() self.has_fts5 = _fts5_available(self.conn) self.conn.execute(FTS_SQL_FTS5 if self.has_fts5 else FTS_SQL_FALLBACK) # Additive columns for DBs created before they existed — CREATE TABLE IF NOT # EXISTS above is a no-op on an already-existing table, so new columns need an # explicit, idempotent ALTER TABLE here (SQLite has no "ADD COLUMN IF NOT EXISTS"). - # - # A few long-lived tables were introduced before their current bi-temporal - # ``expired_at`` field existed. Repair them here before any live indexes or - # queries mention that column, so partially migrated databases continue to - # open instead of crashing during startup. for stmt in ( "ALTER TABLE memories ADD COLUMN sort_order REAL", "ALTER TABLE memories ADD COLUMN pinned_at REAL", "ALTER TABLE memories ADD COLUMN unpinned_at REAL", "ALTER TABLE memories ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0", - "ALTER TABLE memories ADD COLUMN expired_at REAL", "ALTER TABLE memories ADD COLUMN subject_key TEXT DEFAULT ''", "ALTER TABLE memories ADD COLUMN claim_kind TEXT DEFAULT ''", "ALTER TABLE memories ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE memories ADD COLUMN modified_hlc TEXT NOT NULL DEFAULT ''", - "ALTER TABLE memory_entities ADD COLUMN expired_at REAL", - "ALTER TABLE edges ADD COLUMN expired_at REAL", - "ALTER TABLE edge_supports ADD COLUMN expired_at REAL", "ALTER TABLE edges ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE entities ADD COLUMN normalized_name TEXT NOT NULL DEFAULT ''", "ALTER TABLE entities ADD COLUMN canonical_method TEXT NOT NULL DEFAULT 'exact'", @@ -1678,7 +2195,6 @@ def _apply_schema(self, previous_version: int) -> None: "ALTER TABLE code_edges ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE code_edges ADD COLUMN ingested_at REAL", "ALTER TABLE code_edges ADD COLUMN expired_at REAL", - "ALTER TABLE code_memory_links ADD COLUMN expired_at REAL", "ALTER TABLE edges ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE edge_supports ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE memory_entities ADD COLUMN valid_to_recorded_at REAL", @@ -1687,6 +2203,7 @@ def _apply_schema(self, previous_version: int) -> None: "ALTER TABLE operation_receipts ADD COLUMN sequence INTEGER", "ALTER TABLE jobs ADD COLUMN runner_id TEXT", "ALTER TABLE jobs ADD COLUMN heartbeat_at REAL", + "ALTER TABLE jobs ADD COLUMN session_id TEXT REFERENCES sessions(id) ON DELETE SET NULL", "ALTER TABLE memory_tombstones ADD COLUMN repo_id TEXT", "ALTER TABLE memory_tombstones ADD COLUMN export_class TEXT NOT NULL " "DEFAULT 'never_export' CHECK(" @@ -2861,6 +3378,14 @@ def __exit__(self, exc_type, exc, traceback) -> None: @contextmanager def _write_operation(self, name: str, *, commit: bool): """Isolate one compound write without settling a caller-owned transaction.""" + # Inside ``defer_commits`` the caller's outer savepoint owns the whole + # operation. Opening another savepoint here is legal but leaves it to be + # released by the deferral teardown; a failed helper that escapes the + # deferral context could otherwise strand an unreleased inner savepoint. + # Join the outer boundary directly so failure semantics stay with its owner. + if getattr(self.conn._pin, "defer_commits", 0): + yield + return owns_transaction = not self.conn.transaction_owned_by_current_thread() savepoint = "" try: @@ -2885,6 +3410,336 @@ def _write_operation(self, name: str, *, commit: bool): self.conn.execute(f"RELEASE SAVEPOINT {savepoint}") raise + # ── local source-import manifest ───────────────────────────────────────── + def _authorize_source_workspace_id(self, workspace_id: str) -> str: + row = self.conn.execute( + "SELECT name FROM workspaces WHERE id=?", (str(workspace_id),) + ).fetchone() + if row is None: + raise ValueError("source import workspace was not found") + self._authorize_workspace(str(row["name"])) + return str(workspace_id) + + def _source_vault_row(self, vault_id: str) -> Optional[dict]: + row = self.conn.execute( + "SELECT * FROM source_vaults WHERE id=?", (str(vault_id),) + ).fetchone() + if row is None: + return None + result = dict(row) + self._authorize_source_workspace_id(str(result["workspace_id"])) + return result + + def _authorize_source_job(self, job_id: str) -> dict: + row = self.conn.execute( + "SELECT id, workspace_id, repo_id, kind FROM jobs WHERE id=?", + (str(job_id),), + ).fetchone() + if row is None or str(row["kind"]) not in { + "document_import", "obsidian_import", + }: + raise ValueError("source import job was not found") + result = dict(row) + self._authorize_source_workspace_id(str(result["workspace_id"])) + return result + + def get_source_vault(self, vault_id: str) -> Optional[dict]: + return self._source_vault_row(vault_id) + + def get_source_vault_by_root_digest(self, *, kind: str, root_digest: str, + workspace_id: str, repo_id: Optional[str] = None, + session_id: Optional[str] = None) -> Optional[dict]: + self._authorize_source_workspace_id(workspace_id) + row = self.conn.execute( + "SELECT * FROM source_vaults WHERE kind=? AND root_digest=? AND workspace_id=? " + "AND repo_id IS ? AND session_id IS ?", + (kind, root_digest, workspace_id, repo_id, session_id), + ).fetchone() + return dict(row) if row is not None else None + + def list_source_vaults(self, *, workspace_id: Optional[str] = None, + kind: Optional[str] = None, limit: int = 100) -> list[dict]: + clauses, params = [], [] + if workspace_id is not None: + self._authorize_source_workspace_id(workspace_id) + clauses.append("workspace_id=?") + params.append(workspace_id) + if self.allowed_workspaces is not None: + names = sorted(str(name) for name in self.allowed_workspaces) + clauses.append( + "workspace_id IN (SELECT id FROM workspaces WHERE name IN (" + + ",".join("?" for _ in names) + "))" + ) + params.extend(names) + if kind is not None: + clauses.append("kind=?") + params.append(kind) + sql = "SELECT * FROM source_vaults" + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY updated_at DESC, id LIMIT ?" + params.append(max(1, min(10_000, int(limit)))) + return [dict(row) for row in self.conn.execute(sql, params).fetchall()] + + def register_source_vault(self, *, kind: str, root_digest: str, workspace_id: str, + repo_id: Optional[str] = None, session_id: Optional[str] = None, + display_name: str = "", scope: str = "workspace", + memory_type: str = "semantic", importer_version: str = "", + commit: bool = True) -> str: + """Create or refresh a local source-vault identity without storing its root.""" + kind, root_digest = str(kind).strip(), str(root_digest).strip().casefold() + if kind not in {"documents", "obsidian"} or _SOURCE_DIGEST_RE.fullmatch(root_digest) is None: + raise ValueError( + "source collection requires kind='documents' or 'obsidian' and a root digest" + ) + self._authorize_source_workspace_id(workspace_id) + try: + selected_scope = Scope(str(scope)) + except ValueError as exc: + raise ValueError("source vault scope must be workspace, repo, or session") from exc + try: + selected_memory_type = MemoryType(str(memory_type)) + except ValueError as exc: + raise ValueError("source vault memory_type is invalid") from exc + if selected_scope == Scope.WORKSPACE and (repo_id is not None or session_id is not None): + raise ValueError("workspace source vault scope cannot include a repo or session") + if selected_scope == Scope.REPO and (repo_id is None or session_id is not None): + raise ValueError("repo source vault scope requires repo_id and no session_id") + if repo_id is not None: + repo = self.conn.execute( + "SELECT workspace_id FROM repos WHERE id=?", (repo_id,) + ).fetchone() + if repo is None or str(repo["workspace_id"]) != str(workspace_id): + raise ValueError("repo_id does not belong to the source vault workspace") + if selected_scope == Scope.SESSION: + if session_id is None: + raise ValueError("session source vault scope requires session_id") + session = self.get_session(session_id) + if session is None or session["workspace_id"] != workspace_id: + raise ValueError("session_id does not belong to the source vault workspace") + if repo_id is not None and session.get("repo_id") != repo_id: + raise ValueError("session_id does not belong to the source vault repo") + repo_id = repo_id or session.get("repo_id") + safe_display_name = str(display_name or "")[:200] + safe_importer_version = str(importer_version or "")[:64] + stamp = now_ts() + with self._write_operation("source_vault", commit=commit): + existing = self.get_source_vault_by_root_digest( + kind=kind, root_digest=root_digest, workspace_id=workspace_id, + repo_id=repo_id, session_id=session_id, + ) + if existing is not None: + self.conn.execute( + "UPDATE source_vaults SET display_name=?, scope=?, memory_type=?, " + "importer_version=?, updated_at=? WHERE id=?", + (safe_display_name, selected_scope.value, selected_memory_type.value, + safe_importer_version, stamp, existing["id"]), + ) + return str(existing["id"]) + vault_id = ids.new_id("vault") + self.conn.execute( + "INSERT INTO source_vaults(id, kind, root_digest, display_name, workspace_id, " + "repo_id, session_id, scope, memory_type, importer_version, created_at, updated_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (vault_id, kind, root_digest, safe_display_name, workspace_id, + repo_id, session_id, selected_scope.value, selected_memory_type.value, + safe_importer_version, stamp, stamp), + ) + return vault_id + + def get_source_import_item(self, *, vault_id: str, source_key: str) -> Optional[dict]: + if self._source_vault_row(vault_id) is None: + return None + row = self.conn.execute( + "SELECT * FROM source_imports WHERE vault_id=? AND source_key=?", + (vault_id, source_key), + ).fetchone() + return dict(row) if row is not None else None + + def list_source_import_items(self, *, vault_id: str, states: Optional[list[str]] = None, + limit: int = 10_000) -> list[dict]: + if self._source_vault_row(vault_id) is None: + return [] + params: list[Any] = [vault_id] + sql = "SELECT * FROM source_imports WHERE vault_id=?" + if states is not None: + if not states: + return [] + sql += " AND state IN (" + ",".join("?" for _ in states) + ")" + params.extend(str(state) for state in states) + sql += " ORDER BY relative_path LIMIT ?" + params.append(max(1, min(100_000, int(limit)))) + return [dict(row) for row in self.conn.execute(sql, params).fetchall()] + + def upsert_source_import_item(self, *, vault_id: str, source_key: str, relative_path: str, + source_id: Optional[str] = None, + memory_id: Optional[str] = None, subject_key: str = "", + content_sha256: str = "", canonical_sha256: str = "", + file_size: int = 0, + file_mtime_ns: Optional[int] = None, + importer_version: str = "", state: str = "imported", + import_id: Optional[str] = None, + last_error: str = "", seen_at: Optional[float] = None, + commit: bool = True) -> str: + """Atomically create/update one source identity; callers may join their memory write.""" + if self._source_vault_row(vault_id) is None: + raise ValueError("source vault was not found") + try: + relative_path = normalize_document_path(relative_path) + except ValueError as exc: + raise ValueError( + "source import item requires a safe relative path and source key" + ) from exc + source_key = str(source_key).strip().casefold() + if _SOURCE_DIGEST_RE.fullmatch(source_key) is None: + raise ValueError("source import item requires a safe relative path and source key") + for label, digest in ( + ("content_sha256", content_sha256), + ("canonical_sha256", canonical_sha256), + ): + value = str(digest or "").strip().casefold() + if value and _SOURCE_DIGEST_RE.fullmatch(value) is None: + raise ValueError(f"source import item {label} is invalid") + content_sha256 = str(content_sha256 or "").strip().casefold() + canonical_sha256 = str(canonical_sha256 or "").strip().casefold() + if state not in {"imported", "unchanged", "skipped", "rejected", "error", "conflict", "missing", "renamed"}: + raise ValueError("invalid source import item state") + stamp = now_ts() if seen_at is None else float(seen_at) + with self._write_operation("source_item", commit=commit): + existing = self.get_source_import_item(vault_id=vault_id, source_key=source_key) + item_id = ( + str(existing["id"]) + if existing is not None + else str(source_id or ids.new_id("source")) + ) + if not item_id.startswith("src_"): + raise ValueError("source import item requires a typed source id") + self.conn.execute( + "INSERT INTO source_imports(id, vault_id, source_key, relative_path, memory_id, " + "subject_key, content_sha256, canonical_sha256, file_size, file_mtime_ns, importer_version, " + "last_seen_job_id, state, first_imported_at, last_imported_at, last_seen_at, " + "missing_at, last_error) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL,?) " + "ON CONFLICT(vault_id, source_key) DO UPDATE SET relative_path=excluded.relative_path, " + "memory_id=COALESCE(excluded.memory_id, source_imports.memory_id), " + "subject_key=excluded.subject_key, content_sha256=excluded.content_sha256, " + "canonical_sha256=excluded.canonical_sha256, file_size=excluded.file_size, " + "file_mtime_ns=excluded.file_mtime_ns, " + "importer_version=excluded.importer_version, " + "last_seen_job_id=excluded.last_seen_job_id, state=excluded.state, " + "last_imported_at=excluded.last_imported_at, last_seen_at=excluded.last_seen_at, " + "missing_at=NULL, last_error=excluded.last_error", + (item_id, vault_id, source_key, relative_path, memory_id, + str(subject_key or ""), content_sha256, + canonical_sha256, max(0, int(file_size)), file_mtime_ns, + str(importer_version or "")[:64], import_id, state, stamp, stamp, stamp, + _content_free_source_error(last_error)), + ) + return item_id + + def rename_source_import_item(self, *, vault_id: str, source_key: str, + relative_path: str, commit: bool = True) -> bool: + if self._source_vault_row(vault_id) is None: + return False + try: + relative_path = normalize_document_path(relative_path) + except ValueError as exc: + raise ValueError("source import item requires a safe relative path") from exc + with self._write_operation("source_rename", commit=commit): + return bool(self.conn.execute( + "UPDATE source_imports SET relative_path=?, state='renamed', last_seen_at=?, " + "missing_at=NULL WHERE vault_id=? AND source_key=?", + (relative_path, now_ts(), vault_id, source_key), + ).rowcount) + + def mark_source_import_items_missing( + self, *, vault_id: str, seen_before: float, + preserve_paths: Iterable[str] = (), commit: bool = True, + ) -> int: + if self._source_vault_row(vault_id) is None: + return 0 + with self._write_operation("source_missing", commit=commit): + for relative_path in {str(path) for path in preserve_paths if str(path)}: + self.conn.execute( + "UPDATE source_imports SET last_seen_at=? WHERE vault_id=? " + "AND relative_path=? AND state NOT IN ('missing','conflict')", + (float(seen_before), vault_id, relative_path), + ) + return int(self.conn.execute( + "UPDATE source_imports SET state='missing', missing_at=? WHERE vault_id=? " + "AND (last_seen_at IS NULL OR last_seen_at Optional[dict]: + row = self.conn.execute("SELECT * FROM source_imports WHERE id=?", (import_id,)).fetchone() + if row is None: + return None + result = dict(row) + if self._source_vault_row(str(result["vault_id"])) is None: + return None + return result + + def list_source_imports(self, *, vault_id: str, limit: int = 100) -> list[dict]: + if self._source_vault_row(vault_id) is None: + return [] + return [dict(row) for row in self.conn.execute( + "SELECT * FROM source_imports WHERE vault_id=? ORDER BY last_seen_at DESC, id DESC LIMIT ?", + (vault_id, max(1, min(10_000, int(limit)))), + ).fetchall()] + + def record_source_import_job_item(self, *, job_id: str, + relative_path: str, planned_action: str, + result_state: str = "pending", + source_id: Optional[str] = None, + source_format: str = "", + warning_count: int = 0, error_code: str = "", + commit: bool = True) -> str: + """Upsert one content-free per-job plan/result row.""" + self._authorize_source_job(job_id) + try: + relative_path = normalize_document_path(relative_path) + except ValueError as exc: + raise ValueError("source import job item requires a safe relative path") from exc + planned = str(planned_action) + result = str(result_state) + source_format = str(source_format or "").strip() + if len(source_format) > 64 or re.fullmatch(r"[A-Za-z0-9_.+-]*", source_format) is None: + raise ValueError("invalid source import format") + if planned not in {"imported", "updated", "skipped", "rejected", "renamed", "missing", "conflict"}: + raise ValueError("invalid planned source import action") + if result not in {"pending", "imported", "updated", "skipped", "rejected", "renamed", "missing", "conflict", "error", "warning"}: + raise ValueError("invalid source import result") + stamp = now_ts() + with self._write_operation("source_job_item", commit=commit): + row = self.conn.execute( + "SELECT id FROM source_import_items WHERE job_id=? AND relative_path=? " + "AND planned_action=?", + (job_id, relative_path, planned), + ).fetchone() + item_id = str(row["id"]) if row is not None else ids.new_id("source") + self.conn.execute( + "INSERT INTO source_import_items(id, job_id, source_id, relative_path, " + "source_format, planned_action, result_state, warning_count, error_code, " + "created_at, finished_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(job_id, relative_path, planned_action) " + "DO UPDATE SET source_id=COALESCE(excluded.source_id, source_import_items.source_id), " + "source_format=excluded.source_format, result_state=excluded.result_state, " + "warning_count=excluded.warning_count, " + "error_code=excluded.error_code, finished_at=excluded.finished_at", + (item_id, job_id, source_id, relative_path, source_format, planned, result, + max(0, int(warning_count)), str(error_code or "")[:100], stamp, + stamp if result != "pending" else None), + ) + return item_id + + def list_source_import_job_items(self, *, job_id: str, limit: int = 100_000) -> list[dict]: + self._authorize_source_job(job_id) + return [dict(row) for row in self.conn.execute( + "SELECT * FROM source_import_items WHERE job_id=? ORDER BY relative_path, id LIMIT ?", + (job_id, max(1, min(100_000, int(limit)))), + ).fetchall()] + # ── tenancy ─────────────────────────────────────────────────────────────── def _authorize_workspace(self, name: str) -> str: """When this Store is bound to a workspace allow-list, refuse to create or @@ -2896,6 +3751,20 @@ def _authorize_workspace(self, name: str) -> str: raise ValueError(f"workspace '{name}' is not permitted on this instance") return name + def _authorize_workspace_id(self, workspace_id: Optional[str]) -> Optional[str]: + """Apply the instance allow-list to an already-resolved workspace id.""" + if self.allowed_workspaces is None: + return workspace_id + if workspace_id is None: + raise ValueError("workspace is not permitted on this instance") + row = self.conn.execute( + "SELECT name FROM workspaces WHERE id=?", (str(workspace_id),) + ).fetchone() + if row is None: + raise ValueError("workspace was not found") + self._authorize_workspace(str(row["name"])) + return workspace_id + def create_workspace(self, name: str, *, settings: Optional[dict] = None) -> str: self._authorize_workspace(name) wid = ids.new_id("workspace") @@ -2940,6 +3809,7 @@ def get_or_create_workspace( raise def create_repo(self, workspace_id: str, name: str, **kw: Any) -> str: + self._authorize_workspace_id(workspace_id) rid = ids.new_id("repo") self.conn.execute( "INSERT INTO repos(id, workspace_id, name, root_path, vcs_remote, primary_lang, " @@ -2952,6 +3822,7 @@ def create_repo(self, workspace_id: str, name: str, **kw: Any) -> str: def get_or_create_repo(self, workspace_id: str, name: str, **kw: Any) -> str: """Return one scoped repository id, creating it atomically when absent.""" + self._authorize_workspace_id(workspace_id) row = self.conn.execute( "SELECT id FROM repos WHERE workspace_id=? AND name=?", (workspace_id, name) ).fetchone() @@ -2995,6 +3866,7 @@ def get_or_create_repo(self, workspace_id: str, name: str, **kw: Any) -> str: def start_session(self, workspace_id: str, repo_id: Optional[str] = None, *, agent: str = "", user_id: str = "", goal: str = "", commit: bool = True) -> str: + self._authorize_workspace_id(workspace_id) sid = ids.new_id("session") self.conn.execute( "INSERT INTO sessions(id, workspace_id, repo_id, agent, user_id, goal, status, " @@ -3190,6 +4062,7 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, internal compatibility path. Sync may separately preserve the empty pre-v13 descriptive clock; ordinary local writes always mint a real HLC. """ + self._authorize_workspace_id(rec.workspace_id) if ( _enum(rec.scope) == Scope.USER.value and not _allow_legacy_user_scope @@ -3428,6 +4301,11 @@ def _add_memory_impl( def get_memory(self, memory_id: str) -> Optional[MemoryRecord]: row = self.conn.execute("SELECT * FROM memories WHERE id=?", (memory_id,)).fetchone() + if row is not None and self.allowed_workspaces is not None: + try: + self._authorize_workspace_id(row["workspace_id"]) + except ValueError: + return None return _row_to_record(row) if row else None def get_memories(self, memory_ids: Iterable[str]) -> dict[str, MemoryRecord]: @@ -3450,6 +4328,11 @@ def get_memories(self, memory_ids: Iterable[str]) -> dict[str, MemoryRecord]: rows = self.conn.fetchall( f"SELECT * FROM memories WHERE id IN ({marks})", chunk) for row in rows: + if self.allowed_workspaces is not None: + try: + self._authorize_workspace_id(row["workspace_id"]) + except ValueError: + continue out[row["id"]] = _row_to_record(row) return out @@ -4003,6 +4886,37 @@ def _has_table(conn, name: str) -> bool: "SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name=?", (name,) ).fetchone() is not None + @classmethod + def _secure_erase_targets(cls, conn, memory_id: str) -> list[str]: + """Include deterministic sync-conflict successors in one erase operation.""" + if not cls._has_table(conn, "memories"): + return [memory_id] + rows = conn.execute( + "SELECT id, metadata, provenance FROM memories" + ).fetchall() + parents: dict[str, set[str]] = {} + for row in rows: + metadata = _loads(row["metadata"], {}) + provenance = _loads(row["provenance"], {}) + metadata = metadata if isinstance(metadata, dict) else {} + provenance = provenance if isinstance(provenance, dict) else {} + sync_conflict = metadata.get("sync_conflict") + candidates = {provenance.get("conflict_of")} + if isinstance(sync_conflict, dict): + candidates.add(sync_conflict.get("memory_id")) + for parent in candidates: + parent_id = str(parent or "") + if parent_id: + parents.setdefault(parent_id, set()).add(str(row["id"])) + targets = [memory_id] + seen = {memory_id} + for parent in targets: + for child in sorted(parents.get(parent, set())): + if child not in seen: + seen.add(child) + targets.append(child) + return targets + @classmethod def _erase_memory_rows(cls, conn, memory_id: str, *, actor: str = "user") -> dict: """Remove a memory and all known local derivatives from one SQLite database. @@ -4214,8 +5128,19 @@ def _recognised_local_backups(self) -> list[Path]: for pattern in patterns: for candidate in parent.glob(pattern): try: - if candidate.is_file() and candidate.resolve() != primary: - found.append(candidate.resolve()) + stat_result = candidate.lstat() + reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + if candidate.is_symlink() or ( + reparse and getattr(stat_result, "st_file_attributes", 0) & reparse + ): + continue + resolved = candidate.resolve() + if ( + resolved != primary + and resolved.parent == parent + and resolved.is_file() + ): + found.append(resolved) except OSError: continue return sorted(set(found), key=lambda value: str(value)) @@ -4236,31 +5161,41 @@ def secure_erase_memory(self, memory_id: str, *, actor: str = "user") -> dict: # the destructive transaction means the deletion and terminal tombstone # commit (or roll back) as one unit. device_id = self.device_id() - current = self._erase_memory_rows(self.conn, memory_id, actor=actor) - if not current["present"]: + targets = self._secure_erase_targets(self.conn, memory_id) + current_rows = [] + for target_id in targets: + marker = self.get_memory_sync_export(target_id) + current = self._erase_memory_rows(self.conn, target_id, actor=actor) + if current["present"]: + current_rows.append((target_id, current, marker)) + current = next( + (row for row in current_rows if row[0] == memory_id), None + ) + if current is None: raise KeyError(f"no memory with id '{memory_id}'") - export_marker = self.get_memory_sync_export(memory_id) - if ( - export_marker is not None - and export_marker["workspace_id"] == current.get("workspace_id") - ): - export_class = TOMBSTONE_REMOTE_ERASURE - tombstone_workspace_id = export_marker["workspace_id"] - tombstone_repo_id = export_marker["repo_id"] - else: - export_class = TOMBSTONE_NEVER_EXPORT - tombstone_workspace_id = current.get("workspace_id") - tombstone_repo_id = current.get("repo_id") - # Current scope/sensitivity cannot prove that an id ever crossed a sync - # boundary. Only the durable content-free marker can authorize a remote - # erasure; absent or scope-conflicting evidence fails closed to local-only. - self.add_memory_tombstone( - memory_id, deleted_at=now_ts(), - device_id=device_id, - workspace_id=tombstone_workspace_id, - repo_id=tombstone_repo_id, - export_class=export_class, - ) + primary_export_class = TOMBSTONE_NEVER_EXPORT + for target_id, erased, export_marker in current_rows: + if ( + export_marker is not None + and export_marker["workspace_id"] == erased.get("workspace_id") + ): + export_class = TOMBSTONE_REMOTE_ERASURE + tombstone_workspace_id = export_marker["workspace_id"] + tombstone_repo_id = export_marker["repo_id"] + else: + export_class = TOMBSTONE_NEVER_EXPORT + tombstone_workspace_id = erased.get("workspace_id") + tombstone_repo_id = erased.get("repo_id") + # Current scope/sensitivity cannot prove that an id ever crossed a + # sync boundary. Only the durable content-free marker can authorize + # a remote erasure; absent or scope-conflicting evidence fails closed. + self.add_memory_tombstone( + target_id, deleted_at=now_ts(), device_id=device_id, + workspace_id=tombstone_workspace_id, + repo_id=tombstone_repo_id, export_class=export_class, + ) + if target_id == memory_id: + primary_export_class = export_class if owns_transaction and self.conn.transaction_owned_by_current_thread(): self.conn.commit() except BaseException: @@ -4276,10 +5211,11 @@ def secure_erase_memory(self, memory_id: str, *, actor: str = "user") -> dict: conn = None try: conn = self._open_connection(str(backup)) - erased = self._erase_memory_rows(conn, memory_id, actor="secure_erase") + for target_id in targets: + self._erase_memory_rows(conn, target_id, actor="secure_erase") conn.commit() self._checkpoint_and_vacuum(conn, durable=True) - if erased["present"]: + if current_rows: backup_processed += 1 except Exception: # pragma: no cover - keyed/corrupt/locked backups vary by deployment backup_failed += 1 @@ -4292,7 +5228,7 @@ def secure_erase_memory(self, memory_id: str, *, actor: str = "user") -> dict: return { "id": memory_id, "status": "securely_erased", - "export_class": export_class, + "export_class": primary_export_class, "maintenance": maintenance, "recognised_backups_erased": backup_processed, "recognised_backups_failed": backup_failed, @@ -5980,13 +6916,25 @@ def neighbors(self, node_ids: list[str], *, at: Optional[float] = None, source_ids = set().union(*( set(_provenance_memory_ids(edge.provenance)) for edge in edges )) if edges else set() - memories = self.get_memories(sorted(source_ids)) + support_rows = self.edge_supports_in_scope( + [edge.id for edge in edges], at=valid_at, flt=flt, + ) if edges else [] + support_ids = {str(row["memory_id"]) for row in support_rows + if row.get("memory_id")} + memories = self.get_memories(sorted(source_ids | support_ids)) + supports_by_edge: dict[str, set[str]] = {} + for support in support_rows: + supports_by_edge.setdefault(str(support["edge_id"]), set()).add( + str(support["memory_id"]) + ) for edge in edges: if not _edge_is_prompt_eligible(edge.provenance): continue - sources = _provenance_memory_ids(edge.provenance) + sources = set(_provenance_memory_ids(edge.provenance)) + sources.update(supports_by_edge.get(edge.id, set())) if sources and not all( (memory := memories.get(memory_id)) + and (flt is None or memory_matches_filter(memory, flt, at=valid_at)) and _row_is_prompt_eligible(memory.provenance, memory.metadata) for memory_id in sources ): @@ -6321,15 +7269,13 @@ def count_symbols(self, repo_id: str) -> int: def link_memory_symbol(self, *, repo_id: str, symbol_id: str, memory_id: str, relation: str = "mentions", confidence: float = 1.0, commit: bool = True) -> str: - # The partial unique index makes the check-and-insert safe across concurrent - # callers. Keep the whole sequence in one write operation and read the winner - # back after INSERT OR IGNORE; returning the freshly generated id after a - # uniqueness conflict would violate the idempotency contract. + # Keep the existence check and insert in one serialized write operation. + # Separate statements otherwise allow two threads to observe no live link + # before either INSERT runs, defeating the idempotent API contract. with self._write_operation("link_memory_symbol", commit=commit): existing = self.conn.execute( "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " - "AND memory_id=? AND relation=? AND valid_to IS NULL " - "AND expired_at IS NULL", + "AND memory_id=? AND relation=? AND valid_to IS NULL AND expired_at IS NULL", (repo_id, symbol_id, memory_id, relation), ).fetchone() if existing is not None: @@ -6344,13 +7290,7 @@ def link_memory_symbol(self, *, repo_id: str, symbol_id: str, memory_id: str, (link_id, repo_id, symbol_id, memory_id, relation, max(0.0, min(1.0, float(confidence))), stamp, stamp, stamp), ) - winner = self.conn.execute( - "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " - "AND memory_id=? AND relation=? AND valid_to IS NULL " - "AND expired_at IS NULL", - (repo_id, symbol_id, memory_id, relation), - ).fetchone() - return winner["id"] if winner is not None else link_id + return link_id def clear_code_memory_links(self, repo_id: str, *, commit: bool = True) -> None: stamp = now_ts() @@ -6558,28 +7498,32 @@ def memories_mentioning(self, repo_id: str, text: str, *, # ── events & audit ────────────────────────────────────────────────────── def append_event(self, *, kind: str, content: str, workspace_id: str = "", repo_id: str = "", session_id: str = "", refs: Optional[list] = None, - interaction_level: str = "") -> str: + interaction_level: str = "", ts: Optional[float] = None) -> str: # Events are not memories, but are durable, searchable agent context too. Do # not create a side channel that can retain a credential after memory capture is # blocked. reject_secrets((("event content", content), ("event refs", refs))) eid = ids.new_id("event") - owns_session_transaction = False + owns_transaction = not self.conn.transaction_owned_by_current_thread() + event_ts = _finite_timestamp(ts, "event timestamp") + if event_ts is None: + event_ts = now_ts() try: if session_id: - owns_session_transaction = self.begin_session_write( + self.begin_session_write( session_id, workspace_id=workspace_id, repo_id=repo_id or None ) self.conn.execute( "INSERT INTO events(id, workspace_id, repo_id, session_id, kind, content, refs, " "interaction_level, ts) VALUES (?,?,?,?,?,?,?,?,?)", (eid, workspace_id, repo_id, session_id, kind, content, _dumps(refs or []), - interaction_level, now_ts()), + interaction_level, event_ts), ) - self.conn.commit() + if owns_transaction and self.conn.transaction_owned_by_current_thread(): + self.conn.commit() return eid except BaseException: - if (owns_session_transaction + if (owns_transaction and self.conn.transaction_owned_by_current_thread()): self.conn.rollback() raise @@ -7802,6 +8746,16 @@ def _where(self, flt: Optional[SearchFilter], include_invalid: bool, p = f"{alias}." if alias else "" where: list[str] = [] params: list[Any] = [] + if self.allowed_workspaces is not None: + names = sorted(str(name) for name in self.allowed_workspaces) + if not names: + where.append("0") + else: + marks = ",".join("?" for _ in names) + where.append( + f"{p}workspace_id IN (SELECT id FROM workspaces WHERE name IN ({marks}))" + ) + params.extend(names) if flt: if flt.workspace_id: where.append(f"{p}workspace_id=?") diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 92023dd8..bb2b480a 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -52,7 +52,7 @@ import logging import math import re -from typing import Any, Optional +from typing import Any, Iterable, Optional from engraphis.core.graph_layers import merge_graph_layers, normalize_graph_layer from engraphis.core.interfaces import ( @@ -65,6 +65,7 @@ normalize_modified_hlc, parse_modified_hlc, vector_index_requires_sync, + vector_index_shares_store_transaction, ) from engraphis.core.poisoning import ( PoisoningDecision, @@ -86,6 +87,8 @@ logger = logging.getLogger("engraphis.sync") +_VectorIndexAction = tuple[str, str, Any, str] + # ── bundle format ───────────────────────────────────────────────────────────── SYNC_FORMAT = "engraphis-sync" SYNC_VERSION = 3 @@ -884,7 +887,8 @@ def export_bundle(self, workspace_id: str, *, repo_id: Optional[str] = None, # become exportable. links_among() below receives only the retained ids, which also # prevents a link from disclosing a filtered endpoint. mems = [m for m in self.store.list_memories(flt, include_invalid=True) - if m.sensitivity != "secret" and m.scope != Scope.SESSION] + if m.sensitivity != "secret" + and m.scope not in (Scope.SESSION, Scope.USER)] if repo_id is not None: repo_rows = self.store.conn.execute( "SELECT id, name FROM repos WHERE workspace_id=? AND id=?", @@ -1203,6 +1207,7 @@ def _apply_memories(self, mem_dicts: list, report: dict, live_tombstones[tomb["id"]] = (timestamp, mapped_repo) for start in range(0, len(mem_dicts), APPLY_BATCH): batch = mem_dicts[start:start + APPLY_BATCH] + pending_index_actions: list[_VectorIndexAction] = [] parsed = [dict_to_record(d) for d in batch] # One IN(...) lookup for the whole batch instead of get_memory() per row. # ``known`` doubles as the write-through cache so a duplicate id LATER in the @@ -1220,13 +1225,19 @@ def _apply_memories(self, mem_dicts: list, report: dict, for d, rec in zip(batch, parsed): self._apply_one(d, rec, report, accepted, known, local_ws, repo_remap, only_repo_id, src_device, dry_run, - live_tombstones) + live_tombstones, pending_index_actions) if not dry_run: self.store.conn.commit() + self._publish_index_actions(pending_index_actions) def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, local_ws, repo_remap: dict, only_repo_id, src_device, - dry_run: bool, live_tombstones: Optional[dict] = None) -> None: + dry_run: bool, live_tombstones: Optional[dict] = None, + pending_index_actions: Optional[list[_VectorIndexAction]] = None, + ) -> None: + pending_index_actions = ( + pending_index_actions if pending_index_actions is not None else [] + ) if rec is None: report["rejected"] += 1 return @@ -1340,9 +1351,11 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, if existing is not None and provenance_is_approved(existing.provenance): content_changed = rec.content != existing.content self._rehome_external_record(rec, src_device=src_device) - self._preserve_hlc_conflict( + conflict_action = self._preserve_hlc_conflict( existing, rec, report=report, known=known, dry_run=dry_run, ) + if conflict_action is not None: + pending_index_actions.append(conflict_action) if not dry_run and content_changed: self.store.audit( "sync:%s" % _clamp_str(src_device or "peer", 128), @@ -1402,12 +1415,16 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, rec.valid_to_recorded_at = now_ts() rec.embedding = None if existing is not None: - self._preserve_hlc_conflict( + conflict_action = self._preserve_hlc_conflict( existing, rec, report=report, known=known, dry_run=dry_run, ) + if conflict_action is not None: + pending_index_actions.append(conflict_action) if existing is None: if not dry_run: - self._write(rec, commit=False) + index_action = self._write(rec, commit=False) + if index_action is not None: + pending_index_actions.append(index_action) self.store.audit( "sync:%s" % _clamp_str(src_device or "peer", 128), "sync_add", rec.id, @@ -1431,7 +1448,9 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, report["unchanged"] += 1 else: if not dry_run: - self._write(merged, commit=False) + index_action = self._write(merged, commit=False) + if index_action is not None: + pending_index_actions.append(index_action) # A synced bundle overwriting existing content is exactly the # memory-poisoning surface (SECURITY.md): record who/what so the # overwrite is never silent and "why is this known?" stays answerable. @@ -1695,7 +1714,7 @@ def _audit_index_failure( "index_%s_failed" % action, memory_id, "failure_type=%s" % failure_type, - commit=False, + commit=not self.store.conn.transaction_owned_by_current_thread(), ) except Exception as audit_exc: logger.warning( @@ -1729,11 +1748,11 @@ def _preserve_hlc_conflict( report: dict, known: dict, dry_run: bool, - ) -> None: + ) -> Optional[_VectorIndexAction]: """Keep the losing concurrent edit as one deterministic untrusted successor.""" conflict = self._hlc_conflict(existing, incoming) if conflict is None: - return + return None physical, logical, existing_hash, incoming_hash = conflict winner = ( existing @@ -1830,9 +1849,10 @@ def _preserve_hlc_conflict( or not _same_sync_payload(already_preserved, preserved) ): raise SyncError("sync conflict identity collision") - return + return None + index_action = None if not dry_run: - self._write(preserved, commit=False) + index_action = self._write(preserved, commit=False) self.store.audit( "sync", "sync_conflict_preserved", @@ -1846,13 +1866,17 @@ def _preserve_hlc_conflict( ) known[conflict_id] = preserved report["conflicts_preserved"] += 1 + return index_action - def _write(self, rec: MemoryRecord, *, commit: bool = True) -> None: + def _write( + self, rec: MemoryRecord, *, commit: bool = True, + ) -> Optional[_VectorIndexAction]: """Persist a merged/new record verbatim (ids + timestamps preserved) and keep derived state coherent: re-embed for the vector arm when an embedder is wired. ``commit=False`` leaves the transaction open for the caller's batch (apply_bundle).""" quarantined = metadata_is_quarantined(rec.metadata) + external_index_action = None persistent_store = ( self.store.path != ":memory:" and not self.store.path.startswith("file::memory:") @@ -1905,29 +1929,71 @@ def _write(self, rec: MemoryRecord, *, commit: bool = True) -> None: self.index is not None and vector_index_requires_sync(self.index, self.store) ): - try: - self.index.delete([rec.id], commit=False) - except Exception as exc: - self._audit_index_failure("delete", rec.id, exc) + if vector_index_shares_store_transaction(self.index, self.store): + try: + self.index.delete([rec.id], commit=False) + except Exception as exc: + self._audit_index_failure("delete", rec.id, exc) + else: + external_index_action = ("delete", rec.id, None, "") if commit: self.store.conn.commit() - return + self._publish_index_actions([external_index_action]) + return None + return external_index_action if ( rec.embedding is not None and not quarantined and self.index is not None and vector_index_requires_sync(self.index, self.store) ): - try: - self.index.upsert( - [rec.id], rec.embedding.reshape(1, -1), - [{"model": self.embedding_space}], - commit=False, + if vector_index_shares_store_transaction(self.index, self.store): + try: + self.index.upsert( + [rec.id], rec.embedding.reshape(1, -1), + [{"model": self.embedding_space}], + commit=False, + ) + except Exception as exc: + self._audit_index_failure("upsert", rec.id, exc) + else: + external_index_action = ( + "upsert", rec.id, rec.embedding.copy(), self.embedding_space, ) - except Exception as exc: - self._audit_index_failure("upsert", rec.id, exc) if commit: self.store.conn.commit() + self._publish_index_actions([external_index_action]) + return None + return external_index_action + + def _publish_index_actions( + self, actions: Iterable[Optional[_VectorIndexAction]], + ) -> None: + """Publish committed Store vectors to a separately-backed index. + + Coalescing by id avoids exposing intermediate vectors when a bundle repeats one + memory inside a batch. Provider failures remain content-free repair debt while + the already-committed canonical memory stays available. + """ + latest: dict[str, _VectorIndexAction] = {} + for action in actions: + if action is not None: + latest[action[1]] = action + index = self.index + if index is None: + return + for operation, memory_id, vector, model in latest.values(): + try: + if operation == "delete": + index.delete([memory_id]) + elif operation == "upsert" and vector is not None: + index.upsert( + [memory_id], vector.reshape(1, -1), [{"model": model}], + ) + else: # pragma: no cover - actions are constructed locally + raise RuntimeError("invalid deferred vector-index action") + except Exception as exc: # noqa: BLE001 - canonical Store state is committed + self._audit_index_failure(operation, memory_id, exc) @staticmethod def _rehome_external_record(rec: MemoryRecord, *, src_device: object) -> None: diff --git a/engraphis/dashboard_app.py b/engraphis/dashboard_app.py index 440602d8..c39d8aa7 100644 --- a/engraphis/dashboard_app.py +++ b/engraphis/dashboard_app.py @@ -7,16 +7,21 @@ from __future__ import annotations import asyncio +import hashlib import importlib.util import hmac +import inspect +import json import logging -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from urllib.parse import urlsplit import os as _os import secrets +import threading +import time -from fastapi import FastAPI, Request +from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles @@ -24,6 +29,7 @@ from engraphis import licensing from engraphis.config import settings +from engraphis.core.documents import supported_document_extensions from engraphis.http_security import wants_https from engraphis.local_auth import ( BROWSER_SESSION_COOKIE, @@ -36,6 +42,7 @@ from engraphis.routes import v2_api from engraphis.service import ( MAX_IMPORT_FILES, + MAX_IMPORT_RESOURCE_BYTES, MAX_IMPORT_TOTAL_BYTES, MemoryService, ) @@ -46,6 +53,7 @@ _CLASSIC_ASSETS = Path(__file__).resolve().parent / "classic_assets" _V2_ASSETS = Path(__file__).resolve().parent / "dashboard_assets" _INDEX = _V2_ASSETS / "index.html" +_DOCUMENT_SUFFIXES = frozenset(supported_document_extensions()) _DASHBOARD_JSON_REQUEST_BYTES = 8 * 1024 * 1024 _DASHBOARD_UPLOAD_REQUEST_BYTES = ( @@ -211,6 +219,8 @@ class _DashboardApprovalReq(BaseModel): _REVIEW_CSRF_HEADER = "X-Engraphis-Review-CSRF" +_DOCUMENT_REVIEW_TTL_SECONDS = 5 * 60 +_DOCUMENT_REVIEW_LIMIT = 256 def _embedder_status(embedder, configured_model: str) -> str: @@ -313,7 +323,7 @@ async def _lifespan(app: FastAPI): settings.loop_interval, ) try: - if _mcp_asgi is not None: + if _mcp_asgi is not None and _mcp_mgr is not None: async with _mcp_mgr.run(): yield else: @@ -421,6 +431,11 @@ def _discard_unbound_service() -> None: # credential. It is minted alongside a short-lived browser session and exists only # to authorize the narrowly scoped human-approval dashboard action below. app.state.review_csrf_tokens = {} + # A document preview authorizes exactly one subsequent import of the reviewed + # bytes into the reviewed target. Records contain only a request digest, the + # already-opaque browser-session value, and a short process-local expiry. + app.state.document_import_reviews = {} + app.state.document_import_review_lock = threading.Lock() try: import sys as _sys _ed = svc.engine.embedder @@ -514,8 +529,31 @@ def dashboard_review_approve(req: _DashboardApprovalReq, request: Request): reason = req.reason.strip() if not reason: return JSONResponse({"error": "review reason required"}, status_code=422) - source = svc.store.get_memory(req.memory_id) + try: + source = svc.store.get_memory(req.memory_id) + except ValueError: + return JSONResponse( + {"error": "workspace approval is not permitted"}, status_code=403 + ) if source is None: + # A bound Store intentionally redacts foreign rows as ``None``. Keep the + # dashboard's authorization contract distinct from a genuinely missing id + # by checking only the content-free owner identity before returning 404. + raw = svc.store.conn.execute( + "SELECT workspace_id FROM memories WHERE id=?", (req.memory_id,) + ).fetchone() + if raw is not None: + workspace = svc.store.conn.execute( + "SELECT name FROM workspaces WHERE id=?", (raw["workspace_id"],) + ).fetchone() + if workspace is not None: + try: + svc._authorize_workspace(workspace["name"]) + except ValueError: + return JSONResponse( + {"error": "workspace approval is not permitted"}, + status_code=403, + ) return JSONResponse({"error": "memory not found"}, status_code=404) workspace = svc.store.conn.execute( "SELECT name FROM workspaces WHERE id=?", (source.workspace_id,), @@ -566,6 +604,537 @@ def dashboard_review_csrf(request: Request): response.headers["Cache-Control"] = "no-store" return response + def _require_document_browser_owner(request: Request) -> str: + """Keep local document uploads off generic bearer-token API/MCP surfaces. + + A browser owner must hold the HttpOnly dashboard session *and* echo its + process-local CSRF nonce. The general API middleware deliberately accepts a + bearer for automation clients; that authority is insufficient to upload a + selected local documents or make their imported content trusted. + """ + if not settings.api_token: + raise HTTPException( + status_code=409, + detail={"error": "document import requires ENGRAPHIS_API_TOKEN"}, + ) + session_value = request.cookies.get(BROWSER_SESSION_COOKIE) + if not browser_session_ok(session_value, settings.api_token): + raise HTTPException(status_code=401, detail={"error": "browser session required"}) + if request.headers.get("X-Engraphis-Browser-Session") != "1": + raise HTTPException(status_code=403, detail={"error": "browser session header required"}) + expected = app.state.review_csrf_tokens.get(session_value) + supplied = request.headers.get(_REVIEW_CSRF_HEADER, "") + if not expected or not hmac.compare_digest(supplied, expected): + raise HTTPException(status_code=403, detail={"error": "owner confirmation required"}) + # Include the process-local per-login nonce, not only the signed cookie. + # Two owner logins minted in the same second can otherwise share the same + # deterministic cookie value. Rotating the owner confirmation must also + # invalidate any outstanding import review minted by the earlier login. + return hashlib.sha256( + f"{session_value}\0{expected}".encode("utf-8"), + ).hexdigest() + + def _document_review_digest( + *, uploads: list[tuple[str, bytes]], attachments: list[dict], + workspace: str, repo: str, session_id: str, scope: str, + memory_type: str, source_id: str, source_label: str, + on_conflict: str, source_mode: str, + ) -> str: + """Bind one preview to its exact local bytes, provenance, and write target.""" + + material = { + "attachments": sorted( + ( + {"path": str(item["path"]), "size": int(item["size"])} + for item in attachments + ), + key=lambda item: (item["path"].casefold(), item["path"]), + ), + "files": sorted( + ( + { + "path": path, + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + for path, raw in uploads + ), + key=lambda item: (item["path"].casefold(), item["path"]), + ), + "target": { + "memory_type": str(memory_type), + "on_conflict": str(on_conflict), + "repo": str(repo), + "scope": str(scope), + "session_id": str(session_id), + "source_id": str(source_id), + "source_label": str(source_label), + "source_mode": str(source_mode), + "workspace": str(workspace), + }, + } + encoded = json.dumps( + material, ensure_ascii=True, sort_keys=True, separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def _prune_document_reviews(now: float) -> None: + reviews = app.state.document_import_reviews + for token, record in list(reviews.items()): + if float(record["expires_at"]) <= now: + reviews.pop(token, None) + while len(reviews) >= _DOCUMENT_REVIEW_LIMIT: + oldest = min( + reviews, key=lambda token: float(reviews[token]["expires_at"]), + ) + reviews.pop(oldest, None) + + def _issue_document_review( + report: dict, *, owner_binding: str, digest: str, + ) -> dict: + now = time.monotonic() + token = secrets.token_urlsafe(32) + with app.state.document_import_review_lock: + _prune_document_reviews(now) + app.state.document_import_reviews[token] = { + "owner_binding": owner_binding, + "digest": digest, + "expires_at": now + _DOCUMENT_REVIEW_TTL_SECONDS, + } + response = dict(report) + response["review_token"] = token + response["review_expires_in"] = _DOCUMENT_REVIEW_TTL_SECONDS + return response + + def _consume_document_review( + token: str, *, owner_binding: str, digest: str, + ) -> None: + supplied = str(token or "") + if not 32 <= len(supplied) <= 200: + raise HTTPException( + status_code=403, + detail={"error": "a fresh matching import preview is required"}, + ) + now = time.monotonic() + with app.state.document_import_review_lock: + _prune_document_reviews(now) + record = app.state.document_import_reviews.pop(supplied, None) + if ( + record is None + or not hmac.compare_digest( + str(record["owner_binding"]), owner_binding, + ) + or not hmac.compare_digest(str(record["digest"]), digest) + ): + raise HTTPException( + status_code=403, + detail={"error": "a fresh matching import preview is required"}, + ) + + def _document_relative_path(value: object) -> str: + raw = str(value or "").replace("\\", "/") + candidate = PurePosixPath(raw) + windows = PureWindowsPath(raw) + if ( + not raw + or "\x00" in raw + or candidate.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or any(part in {"", ".", ".."} for part in candidate.parts) + or any(ord(character) < 32 for character in raw) + ): + raise HTTPException(status_code=400, detail={"error": "invalid upload path"}) + return candidate.as_posix() + + async def _document_uploads( + files: list[UploadFile], *, source_mode: str, + ) -> list[tuple[str, bytes]]: + if not files or len(files) > MAX_IMPORT_FILES: + raise HTTPException(status_code=413, detail={"error": "invalid document file count"}) + uploads: list[tuple[str, bytes]] = [] + seen_paths: set[str] = set() + total = 0 + for upload in files: + relative_path = _document_relative_path(upload.filename) + path_key = relative_path.casefold() + if path_key in seen_paths: + raise HTTPException(status_code=400, detail={"error": "duplicate upload path"}) + seen_paths.add(path_key) + suffix = PurePosixPath(relative_path).suffix.lower() + if source_mode == "obsidian": + if suffix != ".md": + raise HTTPException( + status_code=400, + detail={"error": "Obsidian mode accepts Markdown note bytes only"}, + ) + elif suffix not in _DOCUMENT_SUFFIXES: + raise HTTPException( + status_code=400, + detail={"error": "unsupported document format"}, + ) + raw = await upload.read(MAX_IMPORT_RESOURCE_BYTES + 1) + if len(raw) > MAX_IMPORT_RESOURCE_BYTES: + raise HTTPException(status_code=413, detail={"error": "document file is too large"}) + total += len(raw) + if total > MAX_IMPORT_TOTAL_BYTES: + raise HTTPException(status_code=413, detail={"error": "document upload is too large"}) + uploads.append((relative_path, raw)) + return uploads + + def _document_attachments(raw_manifest: str, *, source_mode: str) -> list[dict]: + if source_mode != "obsidian" and raw_manifest not in {"", "[]", None}: + raise HTTPException( + status_code=400, + detail={"error": "attachment manifests are only supported in Obsidian mode"}, + ) + try: + manifest = json.loads(raw_manifest or "[]") + except (TypeError, ValueError, RecursionError) as exc: + raise HTTPException(status_code=400, detail={"error": "invalid attachment manifest"}) from exc + if not isinstance(manifest, list) or len(manifest) > MAX_IMPORT_FILES * 20: + raise HTTPException(status_code=400, detail={"error": "invalid attachment manifest"}) + safe = [] + seen_paths: set[str] = set() + for entry in manifest: + if not isinstance(entry, dict): + raise HTTPException(status_code=400, detail={"error": "invalid attachment manifest"}) + path = _document_relative_path(entry.get("path")) + path_key = path.casefold() + if path_key in seen_paths: + raise HTTPException( + status_code=400, + detail={"error": "duplicate attachment path"}, + ) + seen_paths.add(path_key) + size = entry.get("size") + if type(size) is not int or not 0 <= size <= MAX_IMPORT_RESOURCE_BYTES: + raise HTTPException(status_code=400, detail={"error": "invalid attachment manifest"}) + safe.append({"path": path, "size": size}) + return safe + + def _reject_document_path_overlap( + uploads: list[tuple[str, bytes]], attachments: list[dict], + ) -> None: + uploaded = {path.casefold() for path, _raw in uploads} + if uploaded.intersection(str(item["path"]).casefold() for item in attachments): + raise HTTPException( + status_code=400, + detail={"error": "upload and attachment paths overlap"}, + ) + + def _document_source_identity(source_id: str, source_label: str) -> tuple[str, str]: + """Require an explicit identity before creating a browser-upload source.""" + clean_id = source_id.strip() + clean_label = source_label.strip() + if not clean_id and not clean_label: + raise HTTPException( + status_code=400, + detail={"error": "source label is required for a new source"}, + ) + return clean_id, clean_label + + def _document_service_call( + generic_name: str, legacy_name: str, *, generic_kwargs: dict, + legacy_kwargs: dict, + ): + """Call the universal facade when available, retaining old local databases. + + Service rollout is intentionally independent from dashboard rollout. Filter + arguments for an explicit service signature so an older local service keeps + serving its Obsidian compatibility routes during a staged upgrade. + """ + method = getattr(svc, generic_name, None) + kwargs = generic_kwargs if method is not None else legacy_kwargs + if method is None: + method = getattr(svc, legacy_name) + signature = inspect.signature(method) + accepts_kwargs = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + if not accepts_kwargs: + kwargs = { + name: value for name, value in kwargs.items() + if name in signature.parameters + } + return method(**kwargs) + + def _document_sources(workspace: str): + method = getattr(svc, "list_source_vaults", None) + if method is None: + method = getattr(svc, "list_document_sources", None) + if method is None: + method = getattr(svc, "list_obsidian_vaults") + return method(workspace) + + @app.get("/api/workspaces/import-documents/sources", include_in_schema=False) + def document_sources(workspace: str, request: Request): + _require_document_browser_owner(request) + try: + return {"sources": _document_sources(workspace)} + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + + @app.get("/api/workspaces/import-documents/formats", include_in_schema=False) + def document_formats(request: Request): + """Expose the local parser registry to the owner-only browser wizard. + + This avoids treating the client-side picker as an authority while keeping + its supported-format hint synchronized with the server that validates every + uploaded byte. + """ + _require_document_browser_owner(request) + return {"extensions": sorted(_DOCUMENT_SUFFIXES)} + + @app.post("/api/workspaces/import-documents/preview", include_in_schema=False) + async def document_preview( + request: Request, + workspace: str = Form(...), repo: str = Form(""), session_id: str = Form(""), + scope: str = Form("workspace"), memory_type: str = Form("semantic"), + source_id: str = Form(""), source_label: str = Form(""), on_conflict: str = Form("error"), + source_mode: str = Form("documents"), + confirmed: str = Form("false"), attachment_manifest: str = Form("[]"), + files: list[UploadFile] = File(...), + ): + owner_binding = _require_document_browser_owner(request) + if source_mode not in {"documents", "obsidian"}: + raise HTTPException(status_code=400, detail={"error": "invalid document source mode"}) + source_id, source_label = _document_source_identity(source_id, source_label) + uploads = await _document_uploads(files, source_mode=source_mode) + attachments = _document_attachments(attachment_manifest, source_mode=source_mode) + _reject_document_path_overlap(uploads, attachments) + try: + if source_mode == "obsidian": + # Obsidian mode must preserve its existing vlt_ identity, rich + # Markdown provenance, and ``obsidian_import`` job lineage. The + # source-neutral document adapter parses Markdown too, but its + # ``documents`` identity cannot resume an Obsidian source. + report = svc.preview_obsidian_upload( + files=uploads, attachment_manifest=attachments, + workspace=workspace, repo=repo or None, + session_id=session_id or None, scope=scope, + memory_type=memory_type, vault_id=source_id or None, + vault_label=source_label, on_conflict=on_conflict, + confirmed=confirmed.strip().lower() == "true", + ) + else: + report = _document_service_call( + "preview_document_upload", "preview_obsidian_upload", + generic_kwargs={ + "files": uploads, "attachment_manifest": attachments, + "workspace": workspace, "repo": repo or None, + "session_id": session_id or None, "scope": scope, + "memory_type": memory_type, "source_id": source_id or None, + "source_label": source_label, "on_conflict": on_conflict, + }, + legacy_kwargs={ + "files": uploads, "attachment_manifest": attachments, + "workspace": workspace, "repo": repo or None, + "session_id": session_id or None, "scope": scope, + "memory_type": memory_type, "vault_id": source_id or None, + "vault_label": source_label, "on_conflict": on_conflict, + "confirmed": confirmed.strip().lower() == "true", + }, + ) + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + digest = _document_review_digest( + uploads=uploads, attachments=attachments, workspace=workspace, + repo=repo, session_id=session_id, scope=scope, + memory_type=memory_type, source_id=source_id, + source_label=source_label, on_conflict=on_conflict, + source_mode=source_mode, + ) + return _issue_document_review( + report, owner_binding=owner_binding, digest=digest, + ) + + @app.post("/api/workspaces/import-documents/run", include_in_schema=False) + async def document_run( + request: Request, + workspace: str = Form(...), repo: str = Form(""), session_id: str = Form(""), + scope: str = Form("workspace"), memory_type: str = Form("semantic"), + source_id: str = Form(""), source_label: str = Form(""), on_conflict: str = Form("error"), + source_mode: str = Form("documents"), + confirmed: str = Form("false"), review_token: str = Form(""), + attachment_manifest: str = Form("[]"), + files: list[UploadFile] = File(...), + ): + owner_binding = _require_document_browser_owner(request) + if confirmed.strip().lower() != "true": + raise HTTPException(status_code=403, detail={"error": "owner confirmation required"}) + if source_mode not in {"documents", "obsidian"}: + raise HTTPException(status_code=400, detail={"error": "invalid document source mode"}) + source_id, source_label = _document_source_identity(source_id, source_label) + uploads = await _document_uploads(files, source_mode=source_mode) + attachments = _document_attachments(attachment_manifest, source_mode=source_mode) + _reject_document_path_overlap(uploads, attachments) + digest = _document_review_digest( + uploads=uploads, attachments=attachments, workspace=workspace, + repo=repo, session_id=session_id, scope=scope, + memory_type=memory_type, source_id=source_id, + source_label=source_label, on_conflict=on_conflict, + source_mode=source_mode, + ) + _consume_document_review( + review_token, owner_binding=owner_binding, digest=digest, + ) + try: + if source_mode == "obsidian": + return svc.import_obsidian_upload( + files=uploads, attachment_manifest=attachments, + workspace=workspace, repo=repo or None, + session_id=session_id or None, scope=scope, + memory_type=memory_type, vault_id=source_id or None, + vault_label=source_label, on_conflict=on_conflict, + confirmed=True, + ) + return _document_service_call( + "import_document_upload", "import_obsidian_upload", + generic_kwargs={ + "files": uploads, "attachment_manifest": attachments, + "workspace": workspace, "repo": repo or None, + "session_id": session_id or None, "scope": scope, + "memory_type": memory_type, "source_id": source_id or None, + "source_label": source_label, "on_conflict": on_conflict, + "confirmed": True, + }, + legacy_kwargs={ + "files": uploads, "attachment_manifest": attachments, + "workspace": workspace, "repo": repo or None, + "session_id": session_id or None, "scope": scope, + "memory_type": memory_type, "vault_id": source_id or None, + "vault_label": source_label, "on_conflict": on_conflict, + "confirmed": True, + }, + ) + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + + @app.get("/api/workspaces/import-documents/jobs/{job_id}", include_in_schema=False) + def document_job(job_id: str, workspace: str, request: Request): + _require_document_browser_owner(request) + try: + return _document_service_call( + "get_document_import_job", "get_obsidian_import_job", + generic_kwargs={"job_id": job_id, "workspace": workspace}, + legacy_kwargs={"job_id": job_id, "workspace": workspace}, + ) + except (ValueError, KeyError): + raise HTTPException(status_code=404, detail={"error": "import job not found"}) from None + + @app.post("/api/workspaces/import-documents/jobs/{job_id}/cancel", include_in_schema=False) + def cancel_document_job(job_id: str, request: Request, workspace: str = Form(...)): + _require_document_browser_owner(request) + try: + return _document_service_call( + "cancel_document_import_job", "cancel_obsidian_import_job", + generic_kwargs={"job_id": job_id, "workspace": workspace}, + legacy_kwargs={"job_id": job_id, "workspace": workspace}, + ) + except (ValueError, KeyError): + raise HTTPException(status_code=404, detail={"error": "import job not found"}) from None + + # The short-lived Obsidian routes remain browser-owner-only compatibility aliases + # for saved dashboard links. The universal wizard uses import-documents above. + @app.get("/api/workspaces/import-obsidian/vaults", include_in_schema=False) + def obsidian_vaults(workspace: str, request: Request): + _require_document_browser_owner(request) + try: + return {"vaults": svc.list_obsidian_vaults(workspace)} + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + + @app.post("/api/workspaces/import-obsidian/preview", include_in_schema=False) + async def obsidian_preview_alias( + request: Request, workspace: str = Form(...), repo: str = Form(""), + session_id: str = Form(""), scope: str = Form("workspace"), + memory_type: str = Form("semantic"), vault_id: str = Form(""), + vault_label: str = Form(""), on_conflict: str = Form("error"), + confirmed: str = Form("false"), attachment_manifest: str = Form("[]"), + files: list[UploadFile] = File(...), + ): + owner_binding = _require_document_browser_owner(request) + vault_id, vault_label = _document_source_identity(vault_id, vault_label) + uploads = await _document_uploads(files, source_mode="obsidian") + attachments = _document_attachments(attachment_manifest, source_mode="obsidian") + _reject_document_path_overlap(uploads, attachments) + try: + report = svc.preview_obsidian_upload( + files=uploads, attachment_manifest=attachments, workspace=workspace, + repo=repo or None, session_id=session_id or None, scope=scope, + memory_type=memory_type, vault_id=vault_id or None, + vault_label=vault_label, on_conflict=on_conflict, + confirmed=confirmed.strip().lower() == "true", + ) + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + digest = _document_review_digest( + uploads=uploads, attachments=attachments, workspace=workspace, + repo=repo, session_id=session_id, scope=scope, + memory_type=memory_type, source_id=vault_id, + source_label=vault_label, on_conflict=on_conflict, + source_mode="obsidian", + ) + return _issue_document_review( + report, owner_binding=owner_binding, digest=digest, + ) + + @app.post("/api/workspaces/import-obsidian/run", include_in_schema=False) + async def obsidian_run_alias( + request: Request, workspace: str = Form(...), repo: str = Form(""), + session_id: str = Form(""), scope: str = Form("workspace"), + memory_type: str = Form("semantic"), vault_id: str = Form(""), + vault_label: str = Form(""), on_conflict: str = Form("error"), + confirmed: str = Form("false"), review_token: str = Form(""), + attachment_manifest: str = Form("[]"), + files: list[UploadFile] = File(...), + ): + owner_binding = _require_document_browser_owner(request) + if confirmed.strip().lower() != "true": + raise HTTPException(status_code=403, detail={"error": "owner confirmation required"}) + vault_id, vault_label = _document_source_identity(vault_id, vault_label) + uploads = await _document_uploads(files, source_mode="obsidian") + attachments = _document_attachments(attachment_manifest, source_mode="obsidian") + _reject_document_path_overlap(uploads, attachments) + digest = _document_review_digest( + uploads=uploads, attachments=attachments, workspace=workspace, + repo=repo, session_id=session_id, scope=scope, + memory_type=memory_type, source_id=vault_id, + source_label=vault_label, on_conflict=on_conflict, + source_mode="obsidian", + ) + _consume_document_review( + review_token, owner_binding=owner_binding, digest=digest, + ) + try: + return svc.import_obsidian_upload( + files=uploads, attachment_manifest=attachments, workspace=workspace, + repo=repo or None, session_id=session_id or None, scope=scope, + memory_type=memory_type, vault_id=vault_id or None, + vault_label=vault_label, on_conflict=on_conflict, confirmed=True, + ) + except (ValueError, KeyError): + raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None + + @app.get("/api/workspaces/import-obsidian/jobs/{job_id}", include_in_schema=False) + def obsidian_job_alias(job_id: str, workspace: str, request: Request): + _require_document_browser_owner(request) + try: + return svc.get_obsidian_import_job(job_id, workspace=workspace) + except (ValueError, KeyError): + raise HTTPException(status_code=404, detail={"error": "import job not found"}) from None + + @app.post("/api/workspaces/import-obsidian/jobs/{job_id}/cancel", include_in_schema=False) + def cancel_obsidian_job_alias(job_id: str, request: Request, workspace: str = Form(...)): + _require_document_browser_owner(request) + try: + return svc.cancel_obsidian_import_job(job_id, workspace=workspace) + except (ValueError, KeyError): + raise HTTPException(status_code=404, detail={"error": "import job not found"}) from None + from engraphis.netutil import is_local_request @app.middleware("http") diff --git a/engraphis/dashboard_assets/index.html b/engraphis/dashboard_assets/index.html index ee90afc3..2550a7a1 100644 --- a/engraphis/dashboard_assets/index.html +++ b/engraphis/dashboard_assets/index.html @@ -1,642 +1,704 @@ - - - - - - - - Engraphis Ledger - - - - -
- - -
-

- - -
-
-
-
-

Today ·

-

What changed in this workspace

-

Everything below comes from this workspace’s memory records and audit trail.

-
- -
-
—Memories
-
—Visible memories
-
—Workspaces
-
—Sessions
-
- -
-
-

Needs a decision

High-signal records surfaced from local memory.

- -
-
-

Reviewing active memory…

-
-
- -
-
-

Recent activity

Privacy-safe operations from the audit log.

- -
-
- - - -
WhenActorActionScopeReceipt
Loading activity…
-
-
-
- - -
-
- -
-
-
-

Ask · grounded retrieval

-

Answer from what the store can support

-

Every claim links to a memory. If the evidence is weak, Engraphis says so.

-
- -
- - -
- - -
-
- -
-

Ask a question to begin.

-
- -
- Inspect retrieval -
-

Raw retrieval appears after an answer.

-
-
-
-
- -
-
-
-
-

Library · active memory

-

Browse, add and govern memories

-

Live records stay editable without erasing their temporal history.

-
-
- - - -
-
- -
- - - 0 memories -
- -
-
-

Loading memories…

-
- -
-
-

Selected memory

-

Choose a memory

-

Select a memory from the library to inspect its content, scope, provenance and history.

-
- - -
-
-
-
- -
-
-
-
-
-

Graph & Relationships · evidence graph

-

How this workspace connects

-
-
- -
Open Graph & Relationships to load the graph.
-
- 0 entities · 0 relations - Community islands -
-

The graph is a visual summary. Open the Analyse tab to inspect entities and relations with keyboard controls.

-
- - -
-
- -
-
-
-

Provenance · temporal truth

-

Why the store believes what it believes

-

Inspect support, supersessions and privacy-safe receipts without flattening history.

-
- -
- - - - -
- -
-
- - -
-

Search for a claim to inspect its live support and what it replaced.

-
- -
-
- - -
-

Search a topic to travel through its valid-time history.

-
- -
-
-

Recorded operations

Actor, action, scope and verification state.

-
- - -
-
-

Loading context savings…

-

Loading audit records…

-
- -
-
- - -
-

Search a topic to compare closed and current records.

-
-
-
- -
-
-
-

Manage · local operations

-

Operate the engine deliberately

-

Workspace, consolidation, hosted services and interface preferences in one place.

-
- -
- - - - - - - - -
- -
-
-

Workspaces

Each workspace is an independent visibility boundary.

- -
- -

Loading workspaces…

-
-
-
-

Pro · end-to-end encrypted

-

Sync eligible shared workspaces

-

Push this device’s changes and pull peer changes for every eligible shared workspace. Secret, session-scoped, and personal-workspace memories stay local.

-

Open this tab to check the Cloud Sync connection.

-
-
- -
-
-
-

Sleep-time maintenance

-

Review before memory evolves

-

A dry run finds recurring episodes and decayed transients. Nothing is committed until you explicitly apply it.

-
-
- - - -
-
-

No preview has been run.

-
- -
-
-

Pro · hosted compute

-

Portfolio analytics without moving secret memory

-

Aggregate health, growth and reinforcement trends are computed through the connected Engraphis Cloud account.

-

Open this tab to check availability.

- Subscribe to Pro -
-
- -
-
-

Pro · managed maintenance

-

Schedule consolidation with an explicit upload boundary

-

Hosted automation receives a bounded workspace snapshot. Secret and session-scoped memory stays local.

-

Open this tab to check availability.

- Subscribe to Pro -
-
- -
-
-

Team · hosted control plane

-

Shared workspaces, member roles and named seats

-

The local dashboard stays single-user. Team authorization and remote-agent access live in the hosted service.

-

Checking local connection state…

- Compare Team -
-
- -
-
-

Plans & billing

Free is local forever. Pay only for hosted services.

- -
-
-
- - - - - - - - - - - - - - - -
CapabilityFreeProTeam
Local memory engineIncludedIncludedIncluded
Grounded recallIncludedIncludedIncluded
Bi-temporal provenanceIncludedIncludedIncluded
Relations graphIncludedIncludedIncluded
Manual consolidationIncludedIncludedIncluded
Cloud sync—IncludedIncluded
Managed automation—IncludedIncluded
Portfolio analytics—IncludedIncluded
Shared workspaces——Included
Members and roles——Included
Remote agent access——Included
-
-
- -
-
-
-

Interface

-

Dashboard

-

Both interfaces read the same store. The choice changes presentation only.

- -
-
-

Appearance

-

Theme

-

The preference stays on this device and is shared with Classic.

- -
-
-

Pro · hosted account

-

Engraphis Cloud

-

Manage your subscription, connected devices, and hosted account settings in Engraphis Cloud.

- -
-
-
-

Optional synthesis

-

Connect an LLM

-

Use a provider only when you want schema-validated extraction. Recall and storage remain local by default.

-
-

Checking local configuration…

-
-
-

Engine

-

Local runtime

-
API
127.0.0.1:8700
Engine
v2 · bi-temporal
Storage
local SQLite
- Open Classic tools -
-
-
-
-
-
-
- - -
-
-
-

Remote deployment

-

Connect to this Engraphis deployment

-
-
-

Enter the deployment API token. It is exchanged for an HttpOnly browser session and is never stored in the page or URL.

- - -
- - -
-
- - - -
-
-
-

Graph connections

-

Connected nodes

-
- -
-

-
-
-

Memories

-
-
-
- - - - - + + + + + + + + Engraphis Ledger + + + + + +
+ + +
+

+ + + +
+
+

Runtime savings

+

Estimated context saved

+

Loading receipt-backed estimate…

+
+
+ — + tokens avoided +
+
+ — + +
+
+ +
+
+
+
+

Today ·

+

What changed in this workspace

+

Everything below comes from this workspace’s memory records and audit trail.

+
+ +
+
—Live memories
+
—All versions, including history
+
—Workspaces
+
—Sessions
+
+ +
+
+

Runtime savings

Estimated context saved

+ +
+

Loading receipt-backed estimate…

+

Measures estimated prompt-context reduction; it does not measure provider billing.

+
+ +
+
+

Needs a decision

High-signal records surfaced from local memory.

+ +
+
+

Reviewing active memory…

+
+
+ +
+
+

Recent activity

Privacy-safe operations from the audit log.

+ +
+
+ + + +
WhenActorActionScopeReceipt
Loading activity…
+
+
+
+ + +
+
+ +
+
+
+

Ask · grounded retrieval

+

Answer from what the store can support

+

Every claim links to a memory. If the evidence is weak, Engraphis says so.

+
+ +
+ + +
+ + +
+
+ +
+

Ask a question to begin.

+
+ +
+ Inspect retrieval +
+

Raw retrieval appears after an answer.

+
+
+
+
+ +
+
+
+
+

Library · active memory

+

Browse, add and govern memories

+

Live records stay editable without erasing their temporal history.

+
+
+ + + + +
+
+ +
+ + + 0 memories +
+ +
+
+

Loading memories…

+
+ +
+
+

Selected memory

+

Choose a memory

+

Select a memory from the library to inspect its content, scope, provenance and history.

+
+ + +
+
+
+
+ +
+
+
+
+
+

Graph & Relationships · evidence graph

+

How this workspace connects

+
+
+ +
Open Graph & Relationships to load the graph.
+
+ 0 entities · 0 relations + Community islands +
+

The graph is a visual summary. Open the Analyse tab to inspect entities and relations with keyboard controls.

+
+ + +
+
+ +
+
+
+

Provenance · temporal truth

+

Why the store believes what it believes

+

Inspect support, supersessions and privacy-safe receipts without flattening history.

+
+ +
+ + + + +
+ +
+
+ + +
+

Search for a claim to inspect its live support and what it replaced.

+
+ +
+
+ + +
+

Search a topic to travel through its valid-time history.

+
+ +
+
+

Recorded operations

Actor, action, scope and verification state.

+
+ + +
+
+

Loading context savings…

+

Loading audit records…

+
+ +
+
+ + +
+

Search a topic to compare closed and current records.

+
+
+
+ +
+
+
+

Manage · local operations

+

Operate the engine deliberately

+

Workspace, consolidation, hosted services and interface preferences in one place.

+
+ +
+ + + + + + + + +
+ +
+
+

Workspaces

Each workspace is an independent visibility boundary.

+ +
+ +

Loading workspaces…

+
+
+
+

Pro · end-to-end encrypted

+

Sync eligible shared workspaces

+

Push this device’s changes and pull peer changes for every eligible shared workspace. Secret, session-scoped, and personal-workspace memories stay local.

+

Open this tab to check the Cloud Sync connection.

+
+
+ +
+
+
+

Sleep-time maintenance

+

Review before memory evolves

+

A dry run finds recurring episodes and decayed transients. Nothing is committed until you explicitly apply it.

+
+
+ + + +
+
+

No preview has been run.

+
+ +
+
+

Pro · hosted compute

+

Portfolio analytics without moving secret memory

+

Aggregate health, growth and reinforcement trends are computed through the connected Engraphis Cloud account.

+

Open this tab to check availability.

+ Subscribe to Pro +
+
+ +
+
+

Pro · managed maintenance

+

Schedule consolidation with an explicit upload boundary

+

Hosted automation receives a bounded workspace snapshot. Secret and session-scoped memory stays local.

+

Open this tab to check availability.

+ Subscribe to Pro +
+
+ +
+
+

Team · hosted control plane

+

Shared workspaces, member roles and named seats

+

The local dashboard stays single-user. Team authorization and remote-agent access live in the hosted service.

+

Checking local connection state…

+ Compare Team +
+
+ +
+
+

Plans & billing

Free is local forever. Pay only for hosted services.

+ +
+
+
+ + + + + + + + + + + + + + + +
CapabilityFreeProTeam
Local memory engineIncludedIncludedIncluded
Grounded recallIncludedIncludedIncluded
Bi-temporal provenanceIncludedIncludedIncluded
Relations graphIncludedIncludedIncluded
Manual consolidationIncludedIncludedIncluded
Cloud sync—IncludedIncluded
Managed automation—IncludedIncluded
Portfolio analytics—IncludedIncluded
Shared workspaces——Included
Members and roles——Included
Remote agent access——Included
+
+
+ +
+
+
+

Settings · local preferences

+

Make the workspace yours

+

Choose how Engraphis looks and connects while keeping memory, recall, and storage local by default.

+
+
Local-first runtime
+
+
+
+

Interface

+

Dashboard

+

Both interfaces read the same store. The choice changes presentation only.

+ +
+
+

Appearance

+

Theme

+

The preference stays on this device and is shared with Classic.

+ +
+
+

Pro · hosted account

+

Engraphis Cloud

+

Manage your subscription, connected devices, and hosted account settings in Engraphis Cloud.

+ +
+
+
+

Optional synthesis

+

Connect an LLM

+

Use a provider only when you want schema-validated extraction. Recall and storage remain local by default.

+
+

Checking local configuration…

+
+
+

Engine

+

Local runtime

+
API
127.0.0.1:8700
Engine
v2 · bi-temporal
Storage
local SQLite
+ Open Classic tools +
+
+
+
+
+
+
+ + +
+
+
+

Remote deployment

+

Connect to this Engraphis deployment

+
+
+

Enter the deployment API token. It is exchanged for an HttpOnly browser session and is never stored in the page or URL.

+ + +
+ + +
+
+ + + +
+
+
+

Graph connections

+

Connected nodes

+
+ +
+

+
+
+

Memories

+
+
+
+ + + +
+
+

Local document import

Import local documents

+ +
+

Choose individual files or a folder. Engraphis previews supported document formats before it writes anything; uploaded bytes are processed locally and are not kept as dashboard upload copies.

+
+ + + + + + + + + + + +
+ +
Choose files or a folder to preview its import.
+ +

No preview yet.

+
+ + + +
+
+ + + + + diff --git a/engraphis/dashboard_assets/ledger.css b/engraphis/dashboard_assets/ledger.css index b7735b77..5a3941af 100644 --- a/engraphis/dashboard_assets/ledger.css +++ b/engraphis/dashboard_assets/ledger.css @@ -290,6 +290,31 @@ body[data-theme="paper"] .theme-switcher select { color-scheme: light; } font-size: 12px; } .update-dismiss:hover { color: var(--c-fg); } +.notice-banner { + margin: 12px 22px 0; + padding: 9px 12px; + border: 1px solid var(--c-info); + border-radius: 4px; + background: color-mix(in srgb, var(--c-info) 12%, var(--c-surface)); + color: var(--c-fg); + font-size: 12px; +} +.notice-banner[data-tone="error"] { + border-color: var(--c-bad); + background: color-mix(in srgb, var(--c-bad) 12%, var(--c-surface)); +} +.skip-link { + position: fixed; + top: 8px; + left: 8px; + z-index: 1000; + padding: 8px 12px; + border-radius: 4px; + background: var(--c-acc); + color: var(--c-bg); + transform: translateY(-150%); +} +.skip-link:focus { transform: translateY(0); } .primary-nav, .manage-nav { display: grid; gap: 1px; } .primary-nav { flex: 1 0 auto; } .manage-nav { flex: 0 0 auto; } @@ -419,13 +444,23 @@ body[data-theme="paper"] .theme-switcher select { color-scheme: light; } .activity-table td:first-child, .activity-table td:last-child { color: var(--c-dim); font-family: var(--mono); font-size: 10.5px; } .compact-row { display: grid; + width: 100%; gap: 3px; padding: 10px 0; border-bottom: 1px solid var(--c-line); + border-inline: 0; + border-top: 0; + background: transparent; + color: var(--c-fg); + font: inherit; + text-align: left; + cursor: pointer; } +.compact-row:hover { background: var(--c-acc-soft); } +.compact-row:focus-visible { outline-offset: -2px; } .compact-row:last-child { border-bottom: 0; } .compact-row strong { font: 500 14px/1.3 var(--serif); } -.compact-row span { color: var(--c-mid); font-size: 11.5px; line-height: 1.45; } +.compact-row span { display: -webkit-box; overflow: hidden; color: var(--c-mid); font-size: 11.5px; line-height: 1.45; -webkit-box-orient: vertical; -webkit-line-clamp: 2; } .type-bars { display: grid; gap: 10px; } .type-bar { display: grid; grid-template-columns: 88px minmax(0, 1fr) auto; align-items: center; gap: 9px; color: var(--c-mid); font-size: 11px; } .type-bar progress { @@ -996,6 +1031,50 @@ input::placeholder, textarea::placeholder { color: var(--c-dim); opacity: 1; } .llm-test-result[data-tone="error"] { color: var(--c-bad); } .llm-test-result[data-tone="muted"] { color: var(--c-dim); } +.persistent-savings-summary { + display: grid; + grid-template-columns: minmax(0, 1fr) auto auto; + align-items: center; + gap: 18px; + margin: 0 0 18px; + padding: 16px 18px; + border: 1px solid var(--c-line2); + border-left: 3px solid var(--c-acc); + border-radius: 6px; + background: linear-gradient(105deg, var(--c-surface), var(--c-inset)); +} +.persistent-savings-copy { min-width: 0; } +.persistent-savings-copy .eyebrow { margin-bottom: 4px; } +.persistent-savings-copy h2 { margin: 0; font: 600 18px/1.2 var(--serif); } +.persistent-savings-copy p:last-child { margin: 5px 0 0; color: var(--c-mid); font-size: 12px; } +.persistent-savings-value { display: grid; gap: 2px; white-space: nowrap; } +.persistent-savings-value strong { color: var(--c-fg); font: 650 clamp(21px, 2vw, 30px)/1 var(--mono); letter-spacing: -.04em; } +.persistent-savings-value span { color: var(--c-dim); font-size: 11px; text-transform: uppercase; letter-spacing: .08em; } +.persistent-savings-action { display: flex; align-items: center; gap: 14px; white-space: nowrap; } +.savings-inline-rate { color: var(--c-ok); font: 650 12px/1 var(--mono); } + +.settings-intro { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 18px; + margin: 0 0 16px; + padding: 18px 20px; + border: 1px solid var(--c-line); + border-radius: 6px; + background: var(--c-inset); +} +.settings-intro h2 { margin: 0; font: 600 23px/1.15 var(--serif); } +.settings-intro p:last-child { max-width: 680px; margin: 7px 0 0; color: var(--c-mid); line-height: 1.55; } +.settings-intro-status { display: inline-flex; align-items: center; gap: 7px; flex: 0 0 auto; padding: 7px 10px; border: 1px solid var(--c-line2); border-radius: 999px; color: var(--c-ok); font: 650 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; } +.settings-intro-status .status-dot { width: 6px; height: 6px; } +.settings-grid { grid-template-columns: minmax(0, 1.1fr) minmax(260px, .9fr); gap: 14px; } +.setting-card { min-height: 0; padding: 22px; border-radius: 6px; } +.settings-grid > .setting-card:nth-child(1), .settings-grid > .setting-card:nth-child(2) { min-height: 210px; } +.settings-grid > .setting-card:nth-child(3), .settings-grid > .setting-card:nth-child(5) { min-height: 190px; } +.settings-grid > .setting-card:nth-child(5) { grid-column: 2; grid-row: 2; } +.llm-setting-card { grid-row: 3; } + @keyframes page-in { from { transform: translateY(6px); } to { transform: none; } @@ -1012,7 +1091,7 @@ input::placeholder, textarea::placeholder { color: var(--c-dim); opacity: 1; } @media (max-width: 860px) { .app-shell { display: block; } .sidebar { - position: sticky; + position: relative; display: grid; grid-template-columns: auto minmax(150px, 1fr); grid-template-rows: auto auto; @@ -1043,6 +1122,9 @@ input::placeholder, textarea::placeholder { color: var(--c-dim); opacity: 1; } .plan-cards { grid-template-columns: 1fr; } } @media (max-width: 640px) { + .persistent-savings-summary { grid-template-columns: minmax(0, 1fr) auto; gap: 12px; } + .persistent-savings-action { grid-column: 1 / -1; justify-content: space-between; } + .settings-intro { flex-direction: column; } .sidebar { grid-template-columns: 1fr; } .workspace-switcher { grid-column: 1; grid-row: 2; } .dashboard-switcher { grid-column: 1; grid-row: 3; } @@ -1063,6 +1145,7 @@ input::placeholder, textarea::placeholder { color: var(--c-dim); opacity: 1; } .library-list { max-height: 420px; } .inline-form, .split-callout { grid-template-columns: 1fr; } .settings-grid { grid-template-columns: 1fr; } + .settings-grid > .setting-card:nth-child(5), .llm-setting-card { grid-column: auto; grid-row: auto; } .llm-picker-grid { grid-template-columns: 1fr; } .llm-snippet-wrap { grid-template-columns: 1fr; } .llm-copy-button { justify-self: start; } @@ -1076,12 +1159,13 @@ input::placeholder, textarea::placeholder { color: var(--c-dim); opacity: 1; } .update-banner { align-items: flex-start; flex-direction: column; margin: 0 16px; } .update-actions { width: 100%; justify-content: flex-start; } .graph-header { padding: 8px; } - .graph-header h1 { font-size: 12px; } + .graph-header h1 { font-size: 18px; } } @media (max-width: 420px) { - .primary-nav .nav-item { font-size: 12px; } + .primary-nav { display: flex; overflow-x: auto; scrollbar-width: thin; } + .primary-nav .nav-item { min-width: 88px; font-size: 12px; } .primary-nav .nav-item span { font-size: 13px; } - .workspace-switcher { display: none; } + .workspace-switcher { display: grid; } .dashboard-switcher { grid-row: 2; } .theme-switcher { grid-row: 3; } .primary-nav { grid-row: 4; } diff --git a/engraphis/dashboard_assets/ledger.js b/engraphis/dashboard_assets/ledger.js index 79119088..2520b72d 100644 --- a/engraphis/dashboard_assets/ledger.js +++ b/engraphis/dashboard_assets/ledger.js @@ -1,677 +1,702 @@ -(() => { - 'use strict'; - - const apiRoot = `${location.origin}/api`; - const state = { - workspace: '', - workspaces: [], - stats: {}, - memories: [], - selectedMemory: '', - editorMemory: null, - editorReturnFocus: null, - view: 'today', - provenanceTab: 'belief', - savingsPreset: 'all', - manageTab: 'workspaces', - refreshEpoch: 0, - graphWorkspace: '', - graphData: null, - graphDataMode: 'overview', - graphDataIncludeCode: false, - graphDataShowUnlinked: false, - graphDataAsOf: null, - graphMeta: null, - graphMode: 'overview', - graphShowUnlinked: false, - graphEngine: null, - graphLoadPromise: null, - graphLoadWorkspace: '', - graphLoadMode: '', - graphLoadIncludeCode: false, - graphLoadShowUnlinked: false, - graphLoadAsOf: null, - graphLoadController: null, - graphConnectionsRequest: 0, - graphConnectionsController: null, - graphMetrics: {}, - graphFrozen: false, - graphIncludeCode: false, - graphSavedView: 'schema', - consolidationReview: null, - reviewCsrf: '', - hostedLoaded: new Set(), - scopedRequests: Object.create(null), - syncStatus: null, - license: null, - }; - - const byId = id => document.getElementById(id); - const all = selector => [...document.querySelectorAll(selector)]; - const text = value => value == null ? '' : String(value); - const number = value => Number.isFinite(Number(value)) ? Number(value) : 0; - const CLOUD_SYNC_PRIVACY_NOTICE = 'Cloud Sync encrypts eligible shared-workspace changes end-to-end before they leave this device. Engraphis Cloud cannot read their contents; secret and session-scoped memories stay local.'; - const EXTERNAL_LLM_PRIVACY_NOTICE = 'Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; - const truncate = (value, length = 260) => { - const source = text(value).trim(); - return source.length > length ? `${source.slice(0, length - 1)}…` : source; - }; - const empty = (message, className = 'empty-state') => { - const node = document.createElement('p'); - node.className = className; - node.textContent = message; - return node; - }; - const node = (tag, className = '', content = '') => { - const element = document.createElement(tag); - if (className) element.className = className; - if (content !== '') element.textContent = text(content); - return element; - }; - const button = (label, className, action) => { - const control = node('button', className, label); - control.type = 'button'; - control.addEventListener('click', action); - return control; - }; - const option = (value, label, selected = false) => { - const item = node('option', '', label); - item.value = value; - item.selected = selected; - return item; - }; - const query = (name = state.workspace) => `workspace=${encodeURIComponent(name || '')}`; - const beginScopedRequest = kind => { - const generation = number(state.scopedRequests[kind]) + 1; - state.scopedRequests[kind] = generation; - return { - kind, - generation, - workspace: state.workspace, - epoch: state.refreshEpoch, - }; - }; - const isCurrentScopedRequest = request => Boolean(request - && request.workspace === state.workspace - && request.epoch === state.refreshEpoch - && state.scopedRequests[request.kind] === request.generation); - const invalidateScopedRequests = () => { - Object.keys(state.scopedRequests).forEach(kind => { - state.scopedRequests[kind] = number(state.scopedRequests[kind]) + 1; - }); - }; - const GRAPH_INITIAL_NODE_LIMIT = 320; - const GRAPH_FULL_NODE_LIMIT = 20_000; - const GRAPH_LOAD_TIMEOUT_MS = 12_000; - const GRAPH_FULL_LOAD_TIMEOUT_MS = 30_000; - const GRAPH_CONNECTION_MEMORIES_TIMEOUT_MS = 8_000; - const GRAPH_PREFERENCES_KEY = 'engraphis-ledger-graph-preferences-v1'; - const GRAPH_CUSTOM_VIEW_KEY = 'engraphis-ledger-graph-custom-view-v1'; - const GRAPH_LAYERS = ['temporal', 'entity', 'causal', 'semantic', 'code']; - const GRAPH_DEFAULT_LAYERS = { temporal: true, entity: true, causal: true, semantic: true, code: false }; - const GRAPH_TUNING = [ - { id: 'graph-repel', key: 'repel', fallback: 48 }, - { id: 'graph-link', key: 'link', fallback: 16 }, - { id: 'graph-gravity', key: 'gravity', fallback: 48 }, - { id: 'graph-node-size', key: 'size', fallback: 3 }, - { id: 'graph-text-size', key: 'font', fallback: 12 }, - { id: 'graph-line-width', key: 'linkw', fallback: 0.72, precision: 2 }, - { id: 'graph-label-density', key: 'labelDensity', fallback: 24 }, - ]; - const GRAPH_PRESET_TUNING = { - original: { repel: 120, link: 30, gravity: 14, font: 13, size: 3, linkw: 1, labelDensity: 40 }, - compact: { repel: 42, link: 20, gravity: 26, font: 12, size: 3, linkw: 0.7, labelDensity: 30 }, - communities: { repel: 48, link: 16, gravity: 48, font: 12, size: 3, linkw: 0.72, labelDensity: 24 }, - radial: { repel: 68, link: 26, gravity: 12, font: 13, size: 3, linkw: 0.75, labelDensity: 55 }, - constellation: { repel: 34, link: 16, gravity: 38, font: 12, size: 3, linkw: 0.65, labelDensity: 35 }, - }; - const GRAPH_SAVED_VIEWS = { - operations: { - preset: 'compact', style: 'cyber', color: 'connections', palette: 'contrast', - layers: { temporal: false, entity: true, causal: true, semantic: false, code: false }, - minDegree: 2, depth: 1, showUnlinked: false, includeCode: false, - }, - schema: { - preset: 'communities', style: 'cyber', color: 'community', palette: 'theme', - layers: { ...GRAPH_DEFAULT_LAYERS }, minDegree: 1, depth: 2, showUnlinked: false, includeCode: false, - }, - people: { - preset: 'radial', style: 'galaxy', color: 'community', palette: 'aurora', - layers: { temporal: false, entity: true, causal: false, semantic: true, code: false }, - minDegree: 1, depth: 2, showUnlinked: false, includeCode: false, - }, - code: { - preset: 'constellation', style: 'cyber', color: 'type', palette: 'ocean', - layers: { temporal: false, entity: true, causal: false, semantic: true, code: true }, - minDegree: 1, depth: 2, showUnlinked: false, includeCode: true, - }, - }; - const GRAPH_PRESET_LABELS = { - original: 'Spacious', - compact: 'Compact', - communities: 'Islands', - radial: 'Radial', - constellation: 'Constellation', - }; - const GRAPH_STYLE_NOTES = { - cyber: 'Iridescent PVD over graphite — cyan, violet, and magenta across each node.', - galaxy: 'Deep anodized alloy with a cool blue-violet directional sheen.', - solar: 'Brushed copper faces with amber bezels and warm radial grain.', - classic: 'Neutral satin gunmetal with a restrained cool steel edge.', - }; - const GRAPH_CUSTOM_PALETTE = { - person_or_concept: '#8d82e3', - mention: '#5ba1a6', - hashtag: '#c9a15b', - email: '#8eb3e6', - organization: '#d48173', - location: '#7ebf8e', - memory: '#5ba1a6', - repo: '#c9a15b', - file: '#8eb3e6', - }; - const relative = value => { - const raw = typeof value === 'number' && value < 1e12 ? value * 1000 : value; - const time = typeof raw === 'number' ? raw : Date.parse(raw); - if (!Number.isFinite(time)) return 'stored locally'; - const seconds = Math.max(0, Math.round((Date.now() - time) / 1000)); - if (seconds < 60) return 'just now'; - if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; - if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; - if (seconds < 604800) return `${Math.floor(seconds / 86400)}d ago`; - return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium' }).format(time); - }; - const errorMessage = (payload, status) => { - const detail = payload && (payload.detail || payload.error); - if (typeof detail === 'string') return detail; - if (detail && typeof detail.error === 'string') return detail.error; - return `Request failed (${status})`; - }; - - async function api(path, options = {}) { - const init = { ...options, headers: { ...(options.headers || {}) } }; - init.headers['X-Engraphis-Browser-Session'] = '1'; - if (init.body && !(init.body instanceof FormData) && typeof init.body !== 'string') { - init.headers['Content-Type'] = 'application/json'; - init.body = JSON.stringify(init.body); - } - const response = await fetch(`${apiRoot}${path}`, init); - const payload = await response.json().catch(() => null); - if (!response.ok) { - const error = new Error(errorMessage(payload, response.status)); - error.status = response.status; - throw error; - } - return payload; - } - - function promptBrowserToken(message = '') { - const dialog = byId('browser-auth-dialog'); - const form = byId('browser-auth-form'); - const input = byId('browser-auth-token'); - const error = byId('browser-auth-error'); - const cancel = byId('browser-auth-cancel'); - if (!dialog || !form || !input || !error || !cancel) return Promise.resolve(''); - - error.textContent = message; - error.hidden = !message; - input.value = ''; - const returnFocus = document.activeElement; - - return new Promise(resolve => { - let settled = false; - const cleanup = () => { - form.removeEventListener('submit', submit); - cancel.removeEventListener('click', dismiss); - dialog.removeEventListener('cancel', dismiss); - dialog.removeEventListener('close', closed); - }; - const finish = value => { - if (settled) return; - settled = true; - cleanup(); - input.value = ''; - if (dialog.open) dialog.close(); - if (returnFocus && typeof returnFocus.focus === 'function') returnFocus.focus(); - resolve(value); - }; - const submit = event => { - event.preventDefault(); - const value = input.value.trim(); - if (!value) { - error.textContent = 'Enter the deployment token.'; - error.hidden = false; - input.focus(); - return; - } - finish(value); - }; - const dismiss = event => { - if (event) event.preventDefault(); - finish(''); - }; - const closed = () => finish(''); - - form.addEventListener('submit', submit); - cancel.addEventListener('click', dismiss); - dialog.addEventListener('cancel', dismiss); - dialog.addEventListener('close', closed); - if (!dialog.open) dialog.showModal(); - input.focus(); - }); - } - - async function authenticateBrowser() { - let token = ''; - let failure = ''; - try { - const fragment = new URLSearchParams(location.hash.slice(1)); - token = fragment.get('token') || ''; - if (token) history.replaceState(null, '', `${location.pathname}${location.search}`); - } catch (_) {} - while (true) { - if (!token) token = await promptBrowserToken(failure); - if (!token) return false; - let submitted = token; - token = ''; - try { - const session = await api('/auth/session', { - method: 'POST', - body: { token: submitted }, - }); - state.reviewCsrf = text(session && session.review_csrf_token); - submitted = ''; - return true; - } catch (error) { - submitted = ''; - failure = error.message; - showNotice(`Authentication failed: ${failure}`); - } - } - } - - async function reviewCsrfToken() { - if (state.reviewCsrf) return state.reviewCsrf; - const response = await fetch(`${location.origin}/dashboard/review/csrf`, { - headers: { 'X-Engraphis-Browser-Session': '1' }, - }); - const payload = await response.json().catch(() => null); - if (!response.ok || !payload || !payload.review_csrf_token) { - const error = new Error(errorMessage(payload, response.status)); - error.status = response.status; - throw error; - } - state.reviewCsrf = text(payload.review_csrf_token); - return state.reviewCsrf; - } - - async function approveForPrompt(memory) { - if (!memory || !memory.id) return; - const provenance = memory.provenance || {}; - const reviewState = provenance.review_state || 'pending'; - const reason = window.prompt( - `Why is this ${reviewState} record safe to include in model context?`, - ); - if (reason === null) return; - if (!reason.trim()) { - showNotice('A non-empty review reason is required.'); - return; - } - if (!window.confirm( - 'Approve this record for model context? This creates a fresh, audited approved memory; the reviewed source remains preserved.', - )) return; - try { - const csrf = await reviewCsrfToken(); - const response = await fetch(`${location.origin}/dashboard/review/approve`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Engraphis-Browser-Session': '1', - 'X-Engraphis-Review-CSRF': csrf, - }, - body: JSON.stringify({ memory_id: memory.id, reason: reason.trim() }), - }); - const payload = await response.json().catch(() => null); - if (!response.ok) { - const error = new Error(errorMessage(payload, response.status)); - error.status = response.status; - throw error; - } - showNotice('Approved successor created. The reviewed source remains in the audit trail.'); - await selectWorkspace(state.workspace); - if (payload.id) await selectMemory(payload.id); - } catch (error) { - showNotice(`Could not approve this memory: ${error.message}`); - } - } - - let graphAssetsPromise = null; - function loadScript(src, globalName) { - if (window[globalName]) return Promise.resolve(); - return new Promise((resolve, reject) => { - const script = document.createElement('script'); - script.src = src; - script.onload = () => window[globalName] - ? resolve() - : reject(new Error(`${globalName} did not register`)); - script.onerror = () => reject(new Error(`could not load ${src}`)); - document.head.append(script); - }); - } - - function ensureGraphAssets() { - if (window.ForceGraph && window.EngraphisGraph) return Promise.resolve(); - if (!graphAssetsPromise) { - const attempt = loadScript( - '/v2-assets/vendor/d3.min.js?v=20260727-final', - 'd3', - ).then(() => loadScript( - '/v2-assets/vendor/force-graph.min.js?v=20260727-final', - 'ForceGraph', - )).then(() => loadScript( - '/v2-assets/engraphis-graph.js?v=20260809-pin-only-physics', - 'EngraphisGraph', - )); - graphAssetsPromise = attempt; - attempt.catch(() => { - if (graphAssetsPromise === attempt) graphAssetsPromise = null; - }); - } - return graphAssetsPromise; - } - - function showNotice(message) { - byId('notice-text').textContent = message; - } - - function updateReleaseUrl(value) { - const fallback = 'https://github.com/Coding-Dev-Tools/engraphis/releases'; - try { - const url = new URL(value || fallback, location.href); - return ['http:', 'https:'].includes(url.protocol) ? url.href : fallback; - } catch (_) { - return fallback; - } - } - - // A compromised or misconfigured license server could otherwise push a crafted - // upgrade_url (e.g. `javascript:...`) that executes script when the plan link is - // clicked. Only http(s) survives; anything else — including a relative/empty value — - // returns '' so the caller falls back to an inert '#' href. - function safeUrl(value) { - if (!value || typeof value !== 'string') return ''; - try { - const url = new URL(value, location.href); - return ['http:', 'https:'].includes(url.protocol) ? url.href : ''; - } catch (_) { - return ''; - } - } - - function licenseAccessState(license = state.license) { - const value = license && license.access_state; - return ['active', 'trial', 'trial_expired', 'lapsed'].includes(value) ? value : 'inactive'; - } - - function licensePlanKey(license = state.license) { - const value = String((license && license.plan) || 'local').toLowerCase(); - return value === 'pro' || value === 'team' ? value : ''; - } - - function licenseTrialAvailable(license = state.license) { - return Boolean(license && license.trial && license.trial.available - && licenseAccessState(license) === 'inactive' && license.plan_source === 'local'); - } - - function licenseHasHostedAccess(license = state.license) { - const access = licenseAccessState(license); - return access === 'active' || access === 'trial'; - } - - function withCtaAttribution(raw, content, medium = 'product') { - const safe = safeUrl(raw); - if (!safe) return ''; - try { - const url = new URL(safe, location.href); - url.searchParams.set('utm_source', 'engraphis'); - url.searchParams.set('utm_medium', medium); - url.searchParams.set('utm_campaign', 'pro_conversion'); - url.searchParams.set('utm_content', content || 'plans'); - return url.href; - } catch (_) { - return safe; - } - } - - function hostedPlanUrl(plan, trial, interval = 'monthly', content = plan) { - const cadence = interval === 'annual' ? 'annual' : 'monthly'; - const license = state.license || {}; - const raw = license[`${plan}_${cadence}_upgrade_url`] - || license[`${plan}_upgrade_url`] || license.upgrade_url; - const safe = safeUrl(raw); - if (!safe) return ''; - try { - const url = new URL(safe, location.href); - url.searchParams.set('plan', plan); - url.searchParams.set('interval', cadence); - if (trial) url.searchParams.set('trial', plan); - if (!url.hash) url.hash = 'billing'; - return withCtaAttribution(url.href, content); - } catch (_) { - return safe; - } - } - - function hostedAccountUrl(content = 'account') { - const license = state.license || {}; - return withCtaAttribution(license.account_url || license.upgrade_url, content); - } - - function hostedCta(plan = 'pro', content = 'plans', interval = 'monthly') { - const stateName = licenseAccessState(); - const currentPlan = licensePlanKey(); - const name = plan === 'team' ? 'Team' : 'Pro'; - if (stateName === 'lapsed') { - return { label: 'Update billing', href: hostedAccountUrl(content), kind: 'account' }; - } - if (licenseHasHostedAccess() && (currentPlan === plan - || (currentPlan === 'team' && plan === 'pro'))) { - return { - label: currentPlan === 'team' && plan === 'team' ? 'Open Team Cloud' : 'Open Engraphis Cloud', - href: hostedAccountUrl(content), - kind: 'account', - }; - } - const trial = licenseTrialAvailable() && stateName === 'inactive'; - return { - label: trial ? `Start 3-day ${name} trial` : `Subscribe to ${name}`, - href: hostedPlanUrl(plan, trial, interval, content), - kind: trial ? 'trial' : 'subscribe', - }; - } - - function updatePlanBadge() { - // The side-menu header intentionally has no plan or upgrade badge. - } - - function renderSidebarCta() { - const copy = byId('sidebar-pro-copy'); - const detail = byId('sidebar-pro-detail'); - const link = byId('sidebar-pro-cta'); - if (!copy || !detail || !link || !state.license) return; - const canonicalProCtaLabel = 'Subscribe to Pro'; - const renderFeatureCtas = () => { - [ - ['analytics-pro-cta', 'analytics', 'pro'], - ['automation-pro-cta', 'automation', 'pro'], - ['team-cloud-cta', 'team', 'team'], - ].forEach(([id, content, plan]) => { - const featureLink = byId(id); - if (!featureLink) return; - const featureCta = hostedCta(plan, content); - featureLink.textContent = featureCta.label; - featureLink.href = featureCta.href || '#'; - featureLink.setAttribute('aria-disabled', featureCta.href ? 'false' : 'true'); - }); - }; - if (licenseHasHostedAccess()) { - const cta = hostedCta(licensePlanKey() || 'pro', 'sidebar'); - copy.textContent = 'Thank you for supporting Engraphis.'; - detail.textContent = 'Your subscription funds hosted infrastructure and ongoing development.'; - link.hidden = false; - link.textContent = cta.label; - link.href = cta.href || '#'; - link.setAttribute('aria-disabled', cta.href ? 'false' : 'true'); - renderFeatureCtas(); - return; - } - const cta = hostedCta('pro', 'sidebar'); - copy.textContent = 'Support continued Engraphis development with Pro.'; - detail.textContent = 'Cloud Sync, Analytics, and managed memory maintenance.'; - link.hidden = false; - link.textContent = cta.label; - link.href = cta.href || '#'; - link.setAttribute('aria-disabled', cta.href ? 'false' : 'true'); - link.dataset.proCtaLabel = canonicalProCtaLabel; - link.dataset.proCta = 'sidebar'; - renderFeatureCtas(); - } - - function renderCloudAccountSettings() { - const target = byId('cloud-account-settings'); - if (!target) return; - target.replaceChildren(); - const plan = licensePlanKey() || 'pro'; - const cta = hostedCta(plan, 'settings'); - const live = licenseHasHostedAccess(); - const detail = live - ? 'Your hosted account is connected. Manage membership in Cloud, or edit this workspace’s hosted maintenance policy locally.' - : licenseAccessState() === 'lapsed' - ? 'Your hosted subscription needs attention. Update billing in Engraphis Cloud to restore hosted features.' - : 'Open Engraphis Cloud to start a trial, subscribe, or manage a connected hosted account.'; - const action = node('a', 'primary-button', cta.label); - action.href = cta.href || '#'; - if (cta.href) { - action.target = '_blank'; - action.rel = 'noopener'; - } else { - action.addEventListener('click', event => { - event.preventDefault(); - showNotice('Connect this installation to Engraphis Cloud to open hosted account settings.'); - }); - } - const actions = node('div', 'automation-policy-actions'); - actions.append(action); - if (live) actions.append(button('Configure hosted policy', 'secondary-button', () => switchManageTab('automation'))); - target.append(node('p', 'automation-policy-note', detail), actions); - } - - function renderUpdateBanner(update) { - const target = byId('update-banner'); - if (!target) return; - target.replaceChildren(); - if (!update || !update.enabled || !update.update_available || !update.latest) { - target.hidden = true; - return; - } - let dismissed = ''; - try { - dismissed = localStorage.getItem('engraphis-update-dismissed') || ''; - } catch (_) {} - if (dismissed === update.latest) { - target.hidden = true; - return; - } - const copy = node('div', 'update-copy'); - copy.append( - node('strong', '', 'Update available'), - document.createTextNode(` — Engraphis ${text(update.latest)} is out (you have ${text(update.current || '?')}). Upgrade with `), - node('code', '', 'pip install -U engraphis'), - document.createTextNode('.'), - ); - const actions = node('div', 'update-actions'); - const release = node('a', 'text-button', 'View release →'); - release.href = updateReleaseUrl(update.url); - release.target = '_blank'; - release.rel = 'noopener'; - const dismiss = button('Dismiss', 'update-dismiss', () => { - try { - localStorage.setItem('engraphis-update-dismissed', text(update.latest)); - } catch (_) {} - target.hidden = true; - target.replaceChildren(); - }); - actions.append(release, dismiss); - target.append(copy, actions); - target.hidden = false; - } - - function setConnection(message, healthy = true) { - byId('connection-status').textContent = message; - const dot = document.querySelector('.status-dot'); - dot.classList.toggle('unhealthy', !healthy); - } - - function memoryType(memory) { - return memory.memory_type || memory.mtype || 'semantic'; - } - - function memoryTime(memory) { - return memory.ingested_at || memory.valid_from || memory.last_access; - } - - function memoryMeta(memory) { - const meta = node('div', 'memory-meta'); - meta.append( - node('span', 'type-chip', memoryType(memory)), - node('span', '', memory.scope || 'workspace'), - node('span', '', relative(memoryTime(memory))), - ); - if (memory.pinned) meta.append(node('span', '', 'pinned')); - return meta; - } - - function renderMetricValues(stats) { - const values = [ - stats.memories, - stats.total_rows, - stats.workspaces || state.workspaces.length, - stats.sessions, - ]; - all('#metrics strong').forEach((element, index) => { - element.textContent = values[index] == null ? '—' : number(values[index]).toLocaleString(); - }); - } - - function renderTypeBars(stats) { - const target = byId('type-bars'); - target.replaceChildren(); - const types = stats.by_type || {}; - const entries = Object.entries(types).sort((a, b) => number(b[1]) - number(a[1])); - if (!entries.length) { - target.append(empty('No typed memories yet.')); - return; - } - const max = Math.max(1, ...entries.map(([, value]) => number(value))); - entries.forEach(([name, value]) => { - const row = node('div', 'type-bar'); - row.append(node('span', '', name)); - const bar = document.createElement('progress'); - bar.max = max; - bar.value = number(value); - bar.setAttribute('aria-label', `${name}: ${number(value)}`); - row.append(bar, node('strong', '', number(value).toLocaleString())); - target.append(row); - }); - } - - function savingsQuery(workspace, preset = 'all') { - const base = query(workspace); - if (preset === 'current') return `${base}&release_version=1.5.0`; - if (preset === '7d') return `${base}&from_ts=${encodeURIComponent(Date.now() / 1000 - 604800)}`; - return base; - } - +(() => { + 'use strict'; + + const apiRoot = `${location.origin}/api`; + const state = { + workspace: '', + workspaces: [], + stats: {}, + memories: [], + selectedMemory: '', + editorMemory: null, + editorReturnFocus: null, + view: 'today', + provenanceTab: 'belief', + savingsPreset: 'all', + manageTab: 'workspaces', + refreshEpoch: 0, + graphWorkspace: '', + graphData: null, + graphDataMode: 'overview', + graphDataIncludeCode: false, + graphDataShowUnlinked: false, + graphDataAsOf: null, + graphMeta: null, + graphMode: 'overview', + graphShowUnlinked: false, + graphEngine: null, + graphLoadPromise: null, + graphLoadWorkspace: '', + graphLoadMode: '', + graphLoadIncludeCode: false, + graphLoadShowUnlinked: false, + graphLoadAsOf: null, + graphLoadController: null, + graphConnectionsRequest: 0, + graphConnectionsController: null, + graphMetrics: {}, + graphFrozen: false, + graphIncludeCode: false, + graphSavedView: 'schema', + consolidationReview: null, + reviewCsrf: '', + hostedLoaded: new Set(), + scopedRequests: Object.create(null), + syncStatus: null, + license: null, + }; + + const byId = id => document.getElementById(id); + const all = selector => [...document.querySelectorAll(selector)]; + const text = value => value == null ? '' : String(value); + const number = value => Number.isFinite(Number(value)) ? Number(value) : 0; + const CLOUD_SYNC_PRIVACY_NOTICE = 'Cloud Sync encrypts eligible shared-workspace changes end-to-end before they leave this device. Engraphis Cloud cannot read their contents; secret and session-scoped memories stay local.'; + const EXTERNAL_LLM_PRIVACY_NOTICE = 'Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; + const truncate = (value, length = 260) => { + const source = text(value).trim(); + return source.length > length ? `${source.slice(0, length - 1)}…` : source; + }; + const empty = (message, className = 'empty-state') => { + const node = document.createElement('p'); + node.className = className; + node.textContent = message; + return node; + }; + const node = (tag, className = '', content = '') => { + const element = document.createElement(tag); + if (className) element.className = className; + if (content !== '') element.textContent = text(content); + return element; + }; + const button = (label, className, action) => { + const control = node('button', className, label); + control.type = 'button'; + control.addEventListener('click', action); + return control; + }; + const option = (value, label, selected = false) => { + const item = node('option', '', label); + item.value = value; + item.selected = selected; + return item; + }; + const query = (name = state.workspace) => `workspace=${encodeURIComponent(name || '')}`; + const beginScopedRequest = kind => { + const generation = number(state.scopedRequests[kind]) + 1; + state.scopedRequests[kind] = generation; + return { + kind, + generation, + workspace: state.workspace, + epoch: state.refreshEpoch, + }; + }; + const isCurrentScopedRequest = request => Boolean(request + && request.workspace === state.workspace + && request.epoch === state.refreshEpoch + && state.scopedRequests[request.kind] === request.generation); + const invalidateScopedRequests = () => { + Object.keys(state.scopedRequests).forEach(kind => { + state.scopedRequests[kind] = number(state.scopedRequests[kind]) + 1; + }); + }; + const GRAPH_INITIAL_NODE_LIMIT = 320; + const GRAPH_FULL_NODE_LIMIT = 20_000; + const GRAPH_LOAD_TIMEOUT_MS = 12_000; + const GRAPH_FULL_LOAD_TIMEOUT_MS = 30_000; + const GRAPH_CONNECTION_MEMORIES_TIMEOUT_MS = 8_000; + const GRAPH_PREFERENCES_KEY = 'engraphis-ledger-graph-preferences-v1'; + const GRAPH_CUSTOM_VIEW_KEY = 'engraphis-ledger-graph-custom-view-v1'; + const GRAPH_LAYERS = ['temporal', 'entity', 'causal', 'semantic', 'code']; + const GRAPH_DEFAULT_LAYERS = { temporal: true, entity: true, causal: true, semantic: true, code: false }; + const GRAPH_TUNING = [ + { id: 'graph-repel', key: 'repel', fallback: 48 }, + { id: 'graph-link', key: 'link', fallback: 16 }, + { id: 'graph-gravity', key: 'gravity', fallback: 48 }, + { id: 'graph-node-size', key: 'size', fallback: 3 }, + { id: 'graph-text-size', key: 'font', fallback: 12 }, + { id: 'graph-line-width', key: 'linkw', fallback: 0.72, precision: 2 }, + { id: 'graph-label-density', key: 'labelDensity', fallback: 24 }, + ]; + const GRAPH_PRESET_TUNING = { + original: { repel: 120, link: 30, gravity: 14, font: 13, size: 3, linkw: 1, labelDensity: 40 }, + compact: { repel: 42, link: 20, gravity: 26, font: 12, size: 3, linkw: 0.7, labelDensity: 30 }, + communities: { repel: 48, link: 16, gravity: 48, font: 12, size: 3, linkw: 0.72, labelDensity: 24 }, + radial: { repel: 68, link: 26, gravity: 12, font: 13, size: 3, linkw: 0.75, labelDensity: 55 }, + constellation: { repel: 34, link: 16, gravity: 38, font: 12, size: 3, linkw: 0.65, labelDensity: 35 }, + }; + const GRAPH_SAVED_VIEWS = { + operations: { + preset: 'compact', style: 'cyber', color: 'connections', palette: 'contrast', + layers: { temporal: false, entity: true, causal: true, semantic: false, code: false }, + minDegree: 2, depth: 1, showUnlinked: false, includeCode: false, + }, + schema: { + preset: 'communities', style: 'cyber', color: 'community', palette: 'theme', + layers: { ...GRAPH_DEFAULT_LAYERS }, minDegree: 1, depth: 2, showUnlinked: false, includeCode: false, + }, + people: { + preset: 'radial', style: 'galaxy', color: 'community', palette: 'aurora', + layers: { temporal: false, entity: true, causal: false, semantic: true, code: false }, + minDegree: 1, depth: 2, showUnlinked: false, includeCode: false, + }, + code: { + preset: 'constellation', style: 'cyber', color: 'type', palette: 'ocean', + layers: { temporal: false, entity: true, causal: false, semantic: true, code: true }, + minDegree: 1, depth: 2, showUnlinked: false, includeCode: true, + }, + }; + const GRAPH_PRESET_LABELS = { + original: 'Spacious', + compact: 'Compact', + communities: 'Islands', + radial: 'Radial', + constellation: 'Constellation', + }; + const GRAPH_STYLE_NOTES = { + cyber: 'Iridescent PVD over graphite — cyan, violet, and magenta across each node.', + galaxy: 'Deep anodized alloy with a cool blue-violet directional sheen.', + solar: 'Brushed copper faces with amber bezels and warm radial grain.', + classic: 'Neutral satin gunmetal with a restrained cool steel edge.', + }; + const GRAPH_CUSTOM_PALETTE = { + person_or_concept: '#8d82e3', + mention: '#5ba1a6', + hashtag: '#c9a15b', + email: '#8eb3e6', + organization: '#d48173', + location: '#7ebf8e', + memory: '#5ba1a6', + repo: '#c9a15b', + file: '#8eb3e6', + }; + const relative = value => { + const raw = typeof value === 'number' && value < 1e12 ? value * 1000 : value; + const time = typeof raw === 'number' ? raw : Date.parse(raw); + if (!Number.isFinite(time)) return 'stored locally'; + const seconds = Math.max(0, Math.round((Date.now() - time) / 1000)); + if (seconds < 60) return 'just now'; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + if (seconds < 604800) return `${Math.floor(seconds / 86400)}d ago`; + return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium' }).format(time); + }; + const errorMessage = (payload, status) => { + const detail = payload && (payload.detail || payload.error); + if (typeof detail === 'string') return detail; + if (detail && typeof detail.error === 'string') return detail.error; + return `Request failed (${status})`; + }; + + async function api(path, options = {}) { + const init = { ...options, headers: { ...(options.headers || {}) } }; + init.headers['X-Engraphis-Browser-Session'] = '1'; + if (init.body && !(init.body instanceof FormData) && typeof init.body !== 'string') { + init.headers['Content-Type'] = 'application/json'; + init.body = JSON.stringify(init.body); + } + const response = await fetch(`${apiRoot}${path}`, init); + const payload = await response.json().catch(() => null); + if (!response.ok) { + const error = new Error(errorMessage(payload, response.status)); + error.status = response.status; + throw error; + } + return payload; + } + + function promptBrowserToken(message = '') { + const dialog = byId('browser-auth-dialog'); + const form = byId('browser-auth-form'); + const input = byId('browser-auth-token'); + const error = byId('browser-auth-error'); + const cancel = byId('browser-auth-cancel'); + if (!dialog || !form || !input || !error || !cancel) return Promise.resolve(''); + + error.textContent = message; + error.hidden = !message; + input.value = ''; + const returnFocus = document.activeElement; + + return new Promise(resolve => { + let settled = false; + const cleanup = () => { + form.removeEventListener('submit', submit); + cancel.removeEventListener('click', dismiss); + dialog.removeEventListener('cancel', dismiss); + dialog.removeEventListener('close', closed); + }; + const finish = value => { + if (settled) return; + settled = true; + cleanup(); + input.value = ''; + if (dialog.open) dialog.close(); + if (returnFocus && typeof returnFocus.focus === 'function') returnFocus.focus(); + resolve(value); + }; + const submit = event => { + event.preventDefault(); + const value = input.value.trim(); + if (!value) { + error.textContent = 'Enter the deployment token.'; + error.hidden = false; + input.focus(); + return; + } + finish(value); + }; + const dismiss = event => { + if (event) event.preventDefault(); + finish(''); + }; + const closed = () => finish(''); + + form.addEventListener('submit', submit); + cancel.addEventListener('click', dismiss); + dialog.addEventListener('cancel', dismiss); + dialog.addEventListener('close', closed); + if (!dialog.open) dialog.showModal(); + input.focus(); + }); + } + + async function authenticateBrowser() { + let token = ''; + let failure = ''; + try { + const fragment = new URLSearchParams(location.hash.slice(1)); + token = fragment.get('token') || ''; + if (token) history.replaceState(null, '', `${location.pathname}${location.search}`); + } catch (_) {} + while (true) { + if (!token) token = await promptBrowserToken(failure); + if (!token) return false; + let submitted = token; + token = ''; + try { + const session = await api('/auth/session', { + method: 'POST', + body: { token: submitted }, + }); + state.reviewCsrf = text(session && session.review_csrf_token); + submitted = ''; + return true; + } catch (error) { + submitted = ''; + failure = error.message; + showNotice(`Authentication failed: ${failure}`); + } + } + } + + async function reviewCsrfToken() { + if (state.reviewCsrf) return state.reviewCsrf; + const response = await fetch(`${location.origin}/dashboard/review/csrf`, { + headers: { 'X-Engraphis-Browser-Session': '1' }, + }); + const payload = await response.json().catch(() => null); + if (!response.ok || !payload || !payload.review_csrf_token) { + const error = new Error(errorMessage(payload, response.status)); + error.status = response.status; + throw error; + } + state.reviewCsrf = text(payload.review_csrf_token); + return state.reviewCsrf; + } + + async function approveForPrompt(memory) { + if (!memory || !memory.id) return; + const provenance = memory.provenance || {}; + const reviewState = provenance.review_state || 'pending'; + const reason = window.prompt( + `Why is this ${reviewState} record safe to include in model context?`, + ); + if (reason === null) return; + if (!reason.trim()) { + showNotice('A non-empty review reason is required.'); + return; + } + if (!window.confirm( + 'Approve this record for model context? This creates a fresh, audited approved memory; the reviewed source remains preserved.', + )) return; + try { + const csrf = await reviewCsrfToken(); + const response = await fetch(`${location.origin}/dashboard/review/approve`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Engraphis-Browser-Session': '1', + 'X-Engraphis-Review-CSRF': csrf, + }, + body: JSON.stringify({ memory_id: memory.id, reason: reason.trim() }), + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + const error = new Error(errorMessage(payload, response.status)); + error.status = response.status; + throw error; + } + showNotice('Approved successor created. The reviewed source remains in the audit trail.'); + await selectWorkspace(state.workspace); + if (payload.id) await selectMemory(payload.id); + } catch (error) { + showNotice(`Could not approve this memory: ${error.message}`); + } + } + + let graphAssetsPromise = null; + function loadScript(src, globalName) { + if (window[globalName]) return Promise.resolve(); + return new Promise((resolve, reject) => { + const script = document.createElement('script'); + script.src = src; + script.onload = () => window[globalName] + ? resolve() + : reject(new Error(`${globalName} did not register`)); + script.onerror = () => reject(new Error(`could not load ${src}`)); + document.head.append(script); + }); + } + + function ensureGraphAssets() { + if (window.ForceGraph && window.EngraphisGraph) return Promise.resolve(); + if (!graphAssetsPromise) { + const attempt = loadScript( + '/v2-assets/vendor/d3.min.js?v=20260727-final', + 'd3', + ).then(() => loadScript( + '/v2-assets/vendor/force-graph.min.js?v=20260727-final', + 'ForceGraph', + )).then(() => loadScript( + '/v2-assets/engraphis-graph.js?v=20260809-pin-only-physics', + 'EngraphisGraph', + )); + graphAssetsPromise = attempt; + attempt.catch(() => { + if (graphAssetsPromise === attempt) graphAssetsPromise = null; + }); + } + return graphAssetsPromise; + } + + function showNotice(message) { + const text = String(message || ''); + byId('notice-text').textContent = text; + const banner = byId('notice-banner'); + if (!banner) return; + banner.textContent = text; + banner.hidden = !text; + banner.dataset.tone = /\b(could not|unavailable|failed|broken|error)\b/i.test(text) ? 'error' : 'info'; + } + + function updateReleaseUrl(value) { + const fallback = 'https://github.com/Coding-Dev-Tools/engraphis/releases'; + try { + const url = new URL(value || fallback, location.href); + return ['http:', 'https:'].includes(url.protocol) ? url.href : fallback; + } catch (_) { + return fallback; + } + } + + // A compromised or misconfigured license server could otherwise push a crafted + // upgrade_url (e.g. `javascript:...`) that executes script when the plan link is + // clicked. Only http(s) survives; anything else — including a relative/empty value — + // returns '' so the caller falls back to an inert '#' href. + function safeUrl(value) { + if (!value || typeof value !== 'string') return ''; + try { + const url = new URL(value, location.href); + return ['http:', 'https:'].includes(url.protocol) ? url.href : ''; + } catch (_) { + return ''; + } + } + + function licenseAccessState(license = state.license) { + const value = license && license.access_state; + return ['active', 'trial', 'trial_expired', 'lapsed'].includes(value) ? value : 'inactive'; + } + + function licensePlanKey(license = state.license) { + const value = String((license && license.plan) || 'local').toLowerCase(); + return value === 'pro' || value === 'team' ? value : ''; + } + + function licenseTrialAvailable(license = state.license) { + return Boolean(license && license.trial && license.trial.available + && licenseAccessState(license) === 'inactive' && license.plan_source === 'local'); + } + + function licenseHasHostedAccess(license = state.license) { + const access = licenseAccessState(license); + return access === 'active' || access === 'trial'; + } + + function withCtaAttribution(raw, content, medium = 'product') { + const safe = safeUrl(raw); + if (!safe) return ''; + try { + const url = new URL(safe, location.href); + url.searchParams.set('utm_source', 'engraphis'); + url.searchParams.set('utm_medium', medium); + url.searchParams.set('utm_campaign', 'pro_conversion'); + url.searchParams.set('utm_content', content || 'plans'); + return url.href; + } catch (_) { + return safe; + } + } + + function hostedPlanUrl(plan, trial, interval = 'monthly', content = plan) { + const cadence = interval === 'annual' ? 'annual' : 'monthly'; + const license = state.license || {}; + const raw = license[`${plan}_${cadence}_upgrade_url`] + || license[`${plan}_upgrade_url`] || license.upgrade_url; + const safe = safeUrl(raw); + if (!safe) return ''; + try { + const url = new URL(safe, location.href); + url.searchParams.set('plan', plan); + url.searchParams.set('interval', cadence); + if (trial) url.searchParams.set('trial', plan); + if (!url.hash) url.hash = 'billing'; + return withCtaAttribution(url.href, content); + } catch (_) { + return safe; + } + } + + function hostedAccountUrl(content = 'account') { + const license = state.license || {}; + return withCtaAttribution(license.account_url || license.upgrade_url, content); + } + + function hostedCta(plan = 'pro', content = 'plans', interval = 'monthly') { + const stateName = licenseAccessState(); + const currentPlan = licensePlanKey(); + const name = plan === 'team' ? 'Team' : 'Pro'; + if (stateName === 'lapsed') { + return { label: 'Update billing', href: hostedAccountUrl(content), kind: 'account' }; + } + if (licenseHasHostedAccess() && (currentPlan === plan + || (currentPlan === 'team' && plan === 'pro'))) { + return { + label: currentPlan === 'team' && plan === 'team' ? 'Open Team Cloud' : 'Open Engraphis Cloud', + href: hostedAccountUrl(content), + kind: 'account', + }; + } + const trial = licenseTrialAvailable() && stateName === 'inactive'; + return { + label: trial ? `Start 3-day ${name} trial` : `Subscribe to ${name}`, + href: hostedPlanUrl(plan, trial, interval, content), + kind: trial ? 'trial' : 'subscribe', + }; + } + + function updatePlanBadge() { + const badge = byId('plan-badge'); + if (!badge || !state.license) return; + const access = licenseAccessState(); + const plan = licensePlanKey(); + const trial = licenseTrialAvailable(); + const label = access === 'active' ? plan.toUpperCase() + : access === 'trial' ? 'TRIAL' + : access === 'lapsed' ? 'BILLING' + : trial ? 'TRY PRO' : 'GET PRO'; + const aria = licenseHasHostedAccess() ? 'Open Engraphis Cloud account' + : access === 'lapsed' ? 'Update billing in Plans and billing' + : trial ? 'Start the 3-day Pro trial in Plans and billing' + : 'Subscribe to Pro in Plans and billing'; + badge.textContent = label; + badge.setAttribute('aria-label', aria); + badge.title = aria; + const cta = hostedCta(plan || 'pro', 'header'); + const opensAccount = cta.kind === 'account' && Boolean(cta.href); + badge.href = opensAccount ? cta.href : '#'; + badge.target = opensAccount ? '_blank' : ''; + badge.rel = opensAccount ? 'noopener' : ''; + badge.dataset.opensAccount = String(opensAccount); + } + + function renderSidebarCta() { + const copy = byId('sidebar-pro-copy'); + const detail = byId('sidebar-pro-detail'); + const link = byId('sidebar-pro-cta'); + if (!copy || !detail || !link || !state.license) return; + const renderFeatureCtas = () => { + [ + ['analytics-pro-cta', 'analytics', 'pro'], + ['automation-pro-cta', 'automation', 'pro'], + ['team-cloud-cta', 'team', 'team'], + ].forEach(([id, content, plan]) => { + const featureLink = byId(id); + if (!featureLink) return; + const featureCta = hostedCta(plan, content); + featureLink.textContent = featureCta.label; + featureLink.href = featureCta.href || '#'; + featureLink.setAttribute('aria-disabled', featureCta.href ? 'false' : 'true'); + }); + }; + if (licenseHasHostedAccess()) { + const cta = hostedCta(licensePlanKey() || 'pro', 'sidebar'); + copy.textContent = 'Thank you for supporting Engraphis.'; + detail.textContent = 'Your subscription funds hosted infrastructure and ongoing development.'; + link.hidden = false; + link.textContent = cta.label; + link.href = cta.href || '#'; + link.setAttribute('aria-disabled', cta.href ? 'false' : 'true'); + renderFeatureCtas(); + return; + } + const cta = hostedCta('pro', 'sidebar'); + copy.textContent = 'Support continued Engraphis development with Pro.'; + detail.textContent = 'Cloud Sync, Analytics, and managed memory maintenance.'; + link.hidden = false; + link.textContent = cta.label; + link.href = cta.href || '#'; + link.setAttribute('aria-disabled', cta.href ? 'false' : 'true'); + link.dataset.proCta = 'sidebar'; + renderFeatureCtas(); + } + + function renderCloudAccountSettings() { + const target = byId('cloud-account-settings'); + if (!target) return; + target.replaceChildren(); + const plan = licensePlanKey() || 'pro'; + const cta = hostedCta(plan, 'settings'); + const live = licenseHasHostedAccess(); + const detail = live + ? 'Your hosted account is connected. Manage membership in Cloud, or edit this workspace’s hosted maintenance policy locally.' + : licenseAccessState() === 'lapsed' + ? 'Your hosted subscription needs attention. Update billing in Engraphis Cloud to restore hosted features.' + : 'Open Engraphis Cloud to start a trial, subscribe, or manage a connected hosted account.'; + const action = node('a', 'primary-button', cta.label); + action.href = cta.href || '#'; + if (cta.href) { + action.target = '_blank'; + action.rel = 'noopener'; + } else { + action.addEventListener('click', event => { + event.preventDefault(); + showNotice('Connect this installation to Engraphis Cloud to open hosted account settings.'); + }); + } + const actions = node('div', 'automation-policy-actions'); + actions.append(action); + if (live) actions.append(button('Configure hosted policy', 'secondary-button', () => switchManageTab('automation'))); + target.append(node('p', 'automation-policy-note', detail), actions); + } + + function renderUpdateBanner(update) { + const target = byId('update-banner'); + if (!target) return; + target.replaceChildren(); + if (!update || !update.enabled || !update.update_available || !update.latest) { + target.hidden = true; + return; + } + let dismissed = ''; + try { + dismissed = localStorage.getItem('engraphis-update-dismissed') || ''; + } catch (_) {} + if (dismissed === update.latest) { + target.hidden = true; + return; + } + const copy = node('div', 'update-copy'); + copy.append( + node('strong', '', 'Update available'), + document.createTextNode(` — Engraphis ${text(update.latest)} is out (you have ${text(update.current || '?')}). Upgrade with `), + node('code', '', 'pip install -U engraphis'), + document.createTextNode('.'), + ); + const actions = node('div', 'update-actions'); + const release = node('a', 'text-button', 'View release →'); + release.href = updateReleaseUrl(update.url); + release.target = '_blank'; + release.rel = 'noopener'; + const dismiss = button('Dismiss', 'update-dismiss', () => { + try { + localStorage.setItem('engraphis-update-dismissed', text(update.latest)); + } catch (_) {} + target.hidden = true; + target.replaceChildren(); + }); + actions.append(release, dismiss); + target.append(copy, actions); + target.hidden = false; + } + + function setConnection(message, healthy = true) { + byId('connection-status').textContent = message; + const dot = document.querySelector('.status-dot'); + dot.classList.toggle('unhealthy', !healthy); + } + + function memoryType(memory) { + return memory.memory_type || memory.mtype || 'semantic'; + } + + function memoryTime(memory) { + return memory.ingested_at || memory.valid_from || memory.last_access; + } + + function memoryMeta(memory) { + const meta = node('div', 'memory-meta'); + meta.append( + node('span', 'type-chip', memoryType(memory)), + node('span', '', memory.scope || 'workspace'), + node('span', '', relative(memoryTime(memory))), + ); + if (memory.pinned) meta.append(node('span', '', 'pinned')); + return meta; + } + + function renderMetricValues(stats) { + const values = [ + stats.memories, + stats.total_rows, + stats.workspaces || state.workspaces.length, + stats.sessions, + ]; + all('#metrics strong').forEach((element, index) => { + element.textContent = values[index] == null ? '—' : number(values[index]).toLocaleString(); + }); + } + + function renderTypeBars(stats) { + const target = byId('type-bars'); + target.replaceChildren(); + const types = stats.by_type || {}; + const entries = Object.entries(types).sort((a, b) => number(b[1]) - number(a[1])); + if (!entries.length) { + target.append(empty('No typed memories yet.')); + return; + } + const max = Math.max(1, ...entries.map(([, value]) => number(value))); + entries.forEach(([name, value]) => { + const row = node('div', 'type-bar'); + row.append(node('span', '', name)); + const bar = document.createElement('progress'); + bar.max = max; + bar.value = number(value); + bar.setAttribute('aria-label', `${name}: ${number(value)}`); + row.append(bar, node('strong', '', number(value).toLocaleString())); + target.append(row); + }); + } + + function savingsQuery(workspace, preset = 'all') { + const base = query(workspace); + if (preset === 'current') return `${base}&release_version=1.6`; + if (preset === '7d') return `${base}&from_ts=${encodeURIComponent(Date.now() / 1000 - 604800)}`; + return base; + } + function formatSavingsTokens(value) { return Math.max(0, Math.round(number(value))).toLocaleString(); } @@ -701,2715 +726,3250 @@ progress.setAttribute('aria-label', `${(ratio * 100).toFixed(1)}% estimated context reduction`); return { hero, progress }; } - - function savingsCounts(payload) { - const estimate = payload && payload.estimated ? payload.estimated : {}; - return { - estimate, - eligible: number(estimate.eligible_receipt_count), - excluded: number(estimate.excluded_receipt_count) - + number(estimate.unclassified_receipt_count) - + number(estimate.invalid_estimate_count), - }; - } - - function renderSavingsOverview(payload) { - const target = byId('context-savings-summary-body'); - if (!target) return; - const { estimate, eligible, excluded } = savingsCounts(payload); - target.replaceChildren(); - if (!eligible) { - target.append( - empty('No receipt-backed context savings yet.'), - node('p', 'field-note', `${excluded} excluded or unclassified ${excluded === 1 ? 'delivery' : 'deliveries'} so far.`), - ); - return; - } + + function savingsCounts(payload) { + const estimate = payload && payload.estimated ? payload.estimated : {}; + return { + estimate, + eligible: number(estimate.eligible_receipt_count), + excluded: number(estimate.excluded_receipt_count) + + number(estimate.unclassified_receipt_count) + + number(estimate.invalid_estimate_count), + }; + } + + function renderSavingsOverview(payload) { + const target = byId('context-savings-summary-body'); + const { estimate, eligible, excluded } = savingsCounts(payload); + const persistentValue = byId('context-savings-persistent-value'); + const persistentMeta = byId('context-savings-persistent-meta'); + const persistentRate = byId('context-savings-persistent-rate'); + const setPersistent = (value, meta, rate = '—') => { + if (persistentValue) persistentValue.textContent = value; + if (persistentMeta) persistentMeta.textContent = meta; + if (persistentRate) persistentRate.textContent = rate; + }; + if (!target) { + setPersistent('—', 'Savings estimate unavailable.'); + return; + } + target.replaceChildren(); + if (!eligible) { + setPersistent('—', excluded ? `${excluded} excluded or unclassified deliveries so far.` : 'Tracking starts with the first eligible delivery.'); + target.append( + empty('No receipt-backed context savings yet.'), + node('p', 'field-note', `${excluded} excluded or unclassified deliver${excluded === 1 ? 'y' : 'ies'} so far.`), + ); + return; + } const metric = savingsMetric(estimate); + const ratio = savingsRatio(estimate.savings_ratio); + setPersistent( + formatSavingsTokens(estimate.saved_tokens), + `Across ${eligible.toLocaleString()} eligible context deliveries · ${estimate.confidence || 'unknown'} confidence`, + `${(ratio * 100).toFixed(1)}% estimated reduction`, + ); target.append( metric.hero, metric.progress, node('p', 'savings-summary', `Across ${eligible} eligible context deliveries`), node('p', 'field-note', `Baseline ${formatSavingsTokens(estimate.baseline_tokens)} → emitted ${formatSavingsTokens(estimate.emitted_tokens)} · confidence: ${text(estimate.confidence || 'unknown')}`), - node('p', 'field-note', `${excluded} excluded or unclassified ${excluded === 1 ? 'delivery' : 'deliveries'}.`), - ); - } - - function renderSavingsDetail(payload) { - const target = byId('savings-detail'); - if (!target) return; - const { estimate, eligible, excluded } = savingsCounts(payload); - target.replaceChildren(); - const metric = eligible ? savingsMetric(estimate) : null; + node('p', 'field-note', `${excluded} excluded or unclassified deliver${excluded === 1 ? 'y' : 'ies'}.`), + ); + } + + function renderSavingsDetail(payload) { + const target = byId('savings-detail'); + if (!target) return; + const { estimate, eligible, excluded } = savingsCounts(payload); + target.replaceChildren(); const header = node('div', 'savings-detail-header'); - const presets = node('div', 'savings-presets'); - [ - ['since', 'Since tracking started'], - ['current', 'Current release'], - ['7d', 'Last 7 days'], - ['all', 'All time'], - ].forEach(([value, label]) => { - const control = button(label, '', () => { - state.savingsPreset = value; - loadAudit(); - }); - control.classList.toggle('active', state.savingsPreset === value); - presets.append(control); - }); + header.append( + node('strong', 'savings-number', `${formatSavingsTokens(estimate.saved_tokens)} tokens`), + node('span', '', eligible + ? `${eligible} eligible deliveries · ${(number(estimate.savings_ratio) * 100).toFixed(1)}% estimated reduction` + : 'No eligible estimates in this range.'), + ); + const presets = node('div', 'savings-presets'); + [ + ['since', 'Since tracking started'], + ['current', 'Current release'], + ['7d', 'Last 7 days'], + ['all', 'All time'], + ].forEach(([value, label]) => { + const control = button(label, '', () => { + state.savingsPreset = value; + loadAudit(); + }); + control.classList.toggle('active', state.savingsPreset === value); + control.setAttribute('aria-pressed', String(state.savingsPreset === value)); + presets.append(control); + }); header.append(presets); - if (metric) target.append(metric.hero, metric.progress); target.append(header); - target.append(node('p', 'savings-summary', eligible - ? `${eligible} eligible deliveries` - : 'No eligible estimates in this range.')); - if (eligible) { - target.append(node('p', 'field-note', `Baseline ${formatSavingsTokens(estimate.baseline_tokens)} → emitted ${formatSavingsTokens(estimate.emitted_tokens)} · confidence: ${text(estimate.confidence || 'unknown')}`)); - target.append(node('p', 'field-note', 'Packed context is packing savings; adaptive history is estimated avoided prompt context.')); - const basisTitle = node('h3', '', 'Savings basis'); - const basisRows = node('div', 'savings-breakdown'); - (estimate.by_basis || []).forEach(row => { - const item = node('div', 'savings-breakdown-row'); - item.append( - node('span', '', `${text(row.basis || 'unclassified').replaceAll('_', ' ')} · ${text(row.confidence || 'unknown')}`), - node('span', '', `${formatSavingsTokens(row.baseline_tokens)} → ${formatSavingsTokens(row.emitted_tokens)} · ${formatSavingsTokens(row.saved_tokens)} saved`), - ); - basisRows.append(item); - }); - target.append(basisTitle, basisRows); - if ((estimate.by_token_counter || []).length) { - target.append(node('h3', '', 'Token counters')); - const counterRows = node('div', 'savings-breakdown'); - (estimate.by_token_counter || []).forEach(row => { - const item = node('div', 'savings-breakdown-row'); - item.append( - node('span', '', text(row.token_counter || 'unknown')), - node('span', '', `${formatSavingsTokens(row.saved_tokens)} saved · ${row.receipt_count || 0} eligible delivery`), - ); - counterRows.append(item); - }); - target.append(counterRows); - } - } - target.append(node('p', 'savings-note', `${excluded} excluded or unclassified delivery${excluded === 1 ? '' : 's'}. Measures estimated prompt-context reduction; it does not measure provider billing.`)); - } - - function renderDecisions(memories) { - const target = byId('decision-list'); - target.replaceChildren(); - const candidates = memories.slice(0, 3); - if (!candidates.length) { - target.append(empty('No high-signal memories need review.')); - return; - } - candidates.forEach(memory => { - const card = node(memory.id ? 'button' : 'article', 'decision-card memory-link-card'); - if (memory.id) { - card.type = 'button'; - card.dataset.memoryId = memory.id; - card.addEventListener('click', () => openMemory(memory)); - } - const header = node('div', 'decision-card-header'); - header.append( - node('span', 'tag', memory.pinned ? 'Pinned' : memoryType(memory)), - node('h3', '', memory.title || memory.id || 'Untitled memory'), - ); - card.append(header, node('p', '', truncate(memory.content || memory.summary, 360))); - target.append(card); - }); - } - - function auditItems(payload) { - if (Array.isArray(payload)) return payload; - return payload.audit || payload.entries || payload.records || payload.events || []; - } - - function receiptItems(payload) { - if (Array.isArray(payload)) return payload; - return payload.receipts || payload.entries || payload.records || []; - } - - function provenanceTimestampMs(item) { - // Audit rows use seconds (`ts`), while receipts use milliseconds (`ts_ms`). - // Normalize before merging so both the newest-first order and 120-row cap are - // chronological across the two independently paginated feeds. - const raw = item && (item.ts_ms ?? item.ts ?? item.timestamp ?? item.created_at); - const numeric = Number(raw); - if (Number.isFinite(numeric)) return numeric < 1e12 ? numeric * 1000 : numeric; - const parsed = Date.parse(raw); - return Number.isFinite(parsed) ? parsed : 0; - } - - function auditField(item, ...names) { - for (const name of names) { - if (item && item[name] != null && item[name] !== '') return item[name]; - } - return ''; - } - - function renderActivity(items) { - const target = byId('activity-body'); - target.replaceChildren(); - if (!items.length) { - const row = node('tr'); - const cell = node('td', '', 'No audit entries yet.'); - cell.colSpan = 5; - row.append(cell); - target.append(row); - return; - } - items.slice(0, 8).forEach(item => { - const row = node('tr'); - const timestamp = auditField(item, 'ts', 'timestamp', 'created_at', 'valid_from'); - const values = [ - relative(timestamp), - auditField(item, 'actor', 'source') || 'local operator', - auditField(item, 'action', 'operation', 'event') || 'recorded', - auditField(item, 'scope', 'workspace', 'target') || state.workspace, - truncate(auditField(item, 'hash', 'id', 'receipt_id'), 14) || '—', - ]; - values.forEach(value => row.append(node('td', '', value))); - target.append(row); - }); - } - - function renderProactive(memories) { - const target = byId('proactive-list'); - target.replaceChildren(); - if (!memories.length) { - target.append(empty('No proactive context is available.')); - return; - } - memories.slice(0, 5).forEach(memory => { - const row = node('button', 'compact-row'); - row.type = 'button'; - if (memory.id) row.dataset.memoryId = memory.id; - row.append( - node('strong', '', memory.title || memory.id || 'Memory'), - node('span', '', truncate(memory.summary || memory.content, 140)), - ); - row.addEventListener('click', () => openMemory(memory)); - target.append(row); - }); - } - - async function loadStats(workspace, epoch) { - const stats = await api(`/stats?${query(workspace)}`); - if (epoch !== state.refreshEpoch) return; - state.stats = stats; - renderMetricValues(stats); - renderTypeBars(stats); - } - - async function loadSavings(workspace, epoch) { - try { - const payload = await api(`/context-savings?${savingsQuery(workspace)}`); - if (epoch !== state.refreshEpoch) return; - renderSavingsOverview(payload); - } catch (error) { - if (epoch !== state.refreshEpoch) return; - byId('context-savings-summary-body').replaceChildren(empty(`Could not load savings: ${error.message}`)); - } - } - - async function loadMemories(workspace, epoch) { - const payload = await api(`/memories?${query(workspace)}&limit=500`); - if (epoch !== state.refreshEpoch) return; - state.memories = payload.memories || []; - renderLibrary(); - } - - async function loadToday(workspace, epoch) { - const [proactiveResult, auditResult] = await Promise.allSettled([ - api(`/proactive?${query(workspace)}&k=8`), - api(`/audit?${query(workspace)}&limit=12`), - ]); - if (epoch !== state.refreshEpoch) return; - const proactive = proactiveResult.status === 'fulfilled' - ? (proactiveResult.value.memories || proactiveResult.value.results || []) - : state.memories.slice(0, 5); - renderProactive(proactive); - renderDecisions(proactive.length ? proactive : state.memories); - renderActivity(auditResult.status === 'fulfilled' ? auditItems(auditResult.value) : []); - } - - function renderWorkspaceNames() { - all('[data-workspace-name]').forEach(element => { - element.textContent = state.workspace || 'this workspace'; - }); - } - - function workspaceName(item) { - return typeof item === 'string' ? item : item.name; - } - function resetScopedPanels() { - const messages = { - 'answer-panel': 'Ask a question to receive a grounded answer with citations.', - 'retrieval-list': 'Retrieved memories will appear here.', - 'why-result': 'Trace a claim to inspect live and superseded support.', - 'timeline-result': 'Search a topic to inspect its temporal history.', - 'supersession-list': 'Search a topic to compare closed and current records.', - 'audit-list': 'Open Audit to load this workspace’s records and receipts.', - 'analytics-result': 'Open this tab to check availability.', - 'automation-result': 'Open this tab to check availability.', - 'team-result': 'Open this tab to check connection state.', - }; - Object.entries(messages).forEach(([id, message]) => { - const target = byId(id); - if (target) target.replaceChildren(empty(message)); - }); - } - - async function selectWorkspace(name) { - if (!name) return; - invalidateConsolidationReview(); - const epoch = ++state.refreshEpoch; - invalidateScopedRequests(); - closeGraphConnections(); - state.workspace = name; - state.graphWorkspace = ''; - state.graphData = null; - state.graphDataIncludeCode = false; - state.graphDataShowUnlinked = false; - state.selectedMemory = ''; - // Detail/editor handlers close over a memory record. Clear both before the - // workspace fetches begin so a stale form cannot write that record into the - // newly selected workspace. - state.editorMemory = null; - byId('memory-editor').hidden = true; - const memoryDetail = byId('memory-detail'); - memoryDetail.replaceChildren(); - memoryDetail.hidden = true; - resetScopedPanels(); - state.syncStatus = null; - if (state.graphEngine) { - state.graphEngine.destroy(); - state.graphEngine = null; - } - byId('workspace-select').value = name; - renderWorkspaceNames(); - try { - localStorage.setItem('engraphis-workspace', name); - } catch (_) {} - showNotice(''); - try { - await Promise.all([ - loadStats(name, epoch), - loadSavings(name, epoch), - loadMemories(name, epoch), - loadToday(name, epoch), - ]); - if (epoch !== state.refreshEpoch) return; - renderWorkspaceList(); - if (state.view === 'relations') await loadGraph(); - if (state.view === 'provenance' && state.provenanceTab === 'audit') await loadAudit(); - if (state.view === 'manage') await loadManageTab(state.manageTab); - } catch (error) { - if (epoch === state.refreshEpoch) showNotice(`Could not refresh ${name}: ${error.message}`); - } - } - - function memoryCard(memory) { - const card = node('button', 'memory-card'); - card.type = 'button'; - card.setAttribute('role', 'option'); - card.dataset.memoryId = memory.id; - card.setAttribute('aria-selected', String(state.selectedMemory === memory.id)); - if (state.selectedMemory === memory.id) card.classList.add('selected'); - card.append( - node('h2', '', memory.title || memory.id || 'Untitled memory'), - node('p', '', truncate(memory.content || memory.summary, 240)), - memoryMeta(memory), - ); - card.addEventListener('click', () => openMemory(memory)); - return card; - } - - function filteredMemories() { - const filter = byId('library-filter').value.trim().toLowerCase(); - const type = byId('library-type').value; - return state.memories.filter(memory => { - const matchesText = !filter || `${memory.title || ''} ${memory.content || ''} ${memory.summary || ''}` - .toLowerCase().includes(filter); - return matchesText && (!type || memoryType(memory) === type); - }); - } - - function renderLibrary() { - const target = byId('library-list'); - target.replaceChildren(); - const memories = filteredMemories(); - byId('library-count').textContent = `${memories.length.toLocaleString()} ${memories.length === 1 ? 'memory' : 'memories'}`; - if (!memories.length) { - target.append(empty(state.memories.length ? 'No memories match these filters.' : 'No active memories in this workspace.')); - return; - } - memories.forEach(memory => target.append(memoryCard(memory))); - } - - function definitionList(entries) { - const list = node('dl', 'definition-list'); - entries.forEach(([term, value]) => { - const row = node('div'); - row.append(node('dt', '', term), node('dd', '', value || '—')); - list.append(row); - }); - return list; - } - - async function selectMemory(id) { - state.selectedMemory = id; - renderLibrary(); - const target = byId('memory-detail'); - target.hidden = false; - byId('memory-editor').hidden = true; - target.replaceChildren(empty('Loading memory…')); - try { - const payload = await api(`/memory/${encodeURIComponent(id)}?${query()}`); - const memory = payload.memory || state.memories.find(item => item.id === id); - if (!memory || state.selectedMemory !== id) return; - state.editorMemory = memory; - target.replaceChildren(); - target.append( - node('p', 'eyebrow', `${memoryType(memory)} · ${memory.scope || 'workspace'}`), - node('h2', '', memory.title || memory.id || 'Untitled memory'), - node('p', '', memory.content || memory.summary || 'No content.'), - memoryMeta(memory), - definitionList([ - ['Memory id', memory.id], - ['Importance', memory.importance == null ? '—' : number(memory.importance).toFixed(2)], - ['Valid from', relative(memory.valid_from)], - ['Valid to', memory.valid_to ? relative(memory.valid_to) : 'current'], - ['Source', memory.provenance && (memory.provenance.source || memory.provenance.kind)], - ['Review', memory.provenance && (memory.provenance.review_state || 'pending')], - ]), - ); - const actions = node('div', 'detail-actions'); - const provenance = memory.provenance || {}; - if (provenance.review_state !== 'approved' || provenance.trusted !== true) { - actions.append(button('Approve for prompt…', 'primary-button', () => approveForPrompt(memory))); - } - actions.append( - button('Edit', 'secondary-button', () => openEditor(memory)), - button(memory.pinned ? 'Unpin' : 'Pin', 'secondary-button', () => togglePin(memory)), - button('View timeline', 'secondary-button', () => openMemoryTimeline(memory)), - button('Retire', 'danger-button', () => retireMemory(memory)), - button('Secure erase leak', 'danger-button', () => secureEraseMemory(memory)), - ); - target.append(actions); - const chain = payload.chain || []; - if (chain.length) { - target.append(node('h3', '', 'Supersession chain')); - const list = node('div', 'timeline-list'); - chain.forEach(item => list.append(simpleMemoryCard(item, 'timeline-card'))); - target.append(list); - } - } catch (error) { - if (state.selectedMemory === id) target.replaceChildren(empty(`Could not inspect memory: ${error.message}`)); - } - } - - function openMemory(memory) { - if (!memory || !memory.id) { - showNotice('This result no longer identifies a memory to inspect.'); - return; - } - switchView('library'); - selectMemory(memory.id); - } - - function simpleMemoryCard(memory, className = 'memory-card') { - const interactive = Boolean(memory && memory.id); - const card = node(interactive ? 'button' : 'article', `${className}${interactive ? ' memory-link-card' : ''}`); - if (interactive) { - card.type = 'button'; - card.dataset.memoryId = memory.id; - card.addEventListener('click', () => openMemory(memory)); - } - card.append( - node('h3', '', memory.title || memory.id || 'Memory'), - node('p', '', truncate(memory.content || memory.summary, 500)), - memoryMeta(memory), - ); - return card; - } - - function openEditor(memory = null) { - state.editorMemory = memory; - state.editorReturnFocus = document.activeElement instanceof HTMLElement - ? document.activeElement : byId('new-memory-button'); - byId('memory-detail').hidden = true; - const editor = byId('memory-editor'); - editor.hidden = false; - byId('editor-title').textContent = memory ? 'Revise memory' : 'New memory'; - byId('editor-memory-title').value = memory ? (memory.title || '') : ''; - byId('editor-memory-type').value = memory ? memoryType(memory) : 'semantic'; - byId('editor-memory-content').value = memory ? (memory.content || memory.summary || '') : ''; - byId('editor-memory-content').removeAttribute('aria-invalid'); - byId('editor-error').hidden = true; - byId('editor-error').textContent = ''; - byId('editor-memory-importance').value = memory && memory.importance != null ? memory.importance : 0.5; - byId('editor-memory-title').focus(); - } - - function closeEditor() { - const returnFocus = state.editorReturnFocus; - byId('memory-editor').hidden = true; - byId('memory-detail').hidden = false; - state.editorMemory = null; - state.editorReturnFocus = null; - if (returnFocus && document.contains(returnFocus) && !returnFocus.hidden - && !returnFocus.disabled) returnFocus.focus(); - else byId('new-memory-button').focus(); - } - - async function saveMemory(event) { - event.preventDefault(); - const current = state.editorMemory; - const title = byId('editor-memory-title').value.trim(); - const memoryTypeValue = byId('editor-memory-type').value; - const content = byId('editor-memory-content').value.trim(); - const importance = number(byId('editor-memory-importance').value); - const currentImportance = current && current.importance != null - ? number(current.importance) : 0.5; - const contentField = byId('editor-memory-content'); - const editorError = byId('editor-error'); - contentField.removeAttribute('aria-invalid'); - editorError.hidden = true; - editorError.textContent = ''; - if (!content) { - contentField.setAttribute('aria-invalid', 'true'); - editorError.textContent = 'Enter memory content before saving.'; - editorError.hidden = false; - showNotice('Enter memory content before saving.'); - contentField.focus(); - return; - } - try { - if (current) { - if (content !== (current.content || current.summary || '')) { - const corrected = await api('/correct', { - method: 'POST', - body: { id: current.id, workspace: state.workspace, content, reason: 'revised in Ledger' }, - }); - // A correction intentionally creates a replacement. The core inherits the - // source importance; carry any label edits to that replacement rather than - // accidentally applying them to the historical source record. - if (title !== (current.title || '') || memoryTypeValue !== memoryType(current) - || importance !== currentImportance) { - await api('/memory/update', { - method: 'POST', - body: { - id: corrected.id, - workspace: state.workspace, - title, - memory_type: memoryTypeValue, - importance, - }, - }); - } - } else if (title !== (current.title || '') || memoryTypeValue !== memoryType(current) - || importance !== currentImportance) { - await api('/memory/update', { - method: 'POST', - body: { - id: current.id, - workspace: state.workspace, - title, - memory_type: memoryTypeValue, - importance, - }, - }); - } - showNotice('Memory revision recorded with temporal history preserved.'); - } else { - await api('/remember', { - method: 'POST', - body: { - workspace: state.workspace, - content, - title, - mtype: memoryTypeValue, - scope: 'workspace', - importance, - source: 'human:ledger', - trusted: true, - }, - }); - showNotice('Memory saved locally.'); - } - closeEditor(); - await selectWorkspace(state.workspace); - } catch (error) { - showNotice(`Could not save memory: ${error.message}`); - } - } - - async function togglePin(memory) { - try { - await api('/pin', { - method: 'POST', - body: { id: memory.id, workspace: state.workspace, pinned: !memory.pinned }, - }); - showNotice(memory.pinned ? 'Memory unpinned.' : 'Memory pinned against decay.'); - await selectWorkspace(state.workspace); - selectMemory(memory.id); - } catch (error) { - showNotice(`Could not change pin: ${error.message}`); - } - } - - async function retireMemory(memory) { - if (!window.confirm(`Retire “${memory.title || memory.id}”? The record stays in temporal history but leaves live recall.`)) return; - try { - await api('/retire', { - method: 'POST', - body: { id: memory.id, workspace: state.workspace, reason: 'retired in Ledger' }, - }); - state.selectedMemory = ''; - byId('memory-detail').replaceChildren(empty('Memory moved out of live recall. Its history is retained.')); - showNotice('Memory retired without hard deletion.'); - await selectWorkspace(state.workspace); - } catch (error) { - showNotice(`Could not retire memory: ${error.message}`); - } - } - - async function secureEraseMemory(memory) { - const name = memory.title || memory.id; - if (!window.confirm(`Securely erase “${name}”? This destroys temporal history and local index copies. Rotate the leaked credential; copied exports, snapshots, remote peers, and an already-compromised agent cannot be erased here.`)) return; - try { - const result = await api('/secure-erase', { - method: 'POST', body: { id: memory.id, workspace: state.workspace }, - }); - state.selectedMemory = ''; - byId('memory-detail').replaceChildren(empty('Memory securely erased from this local store. Review the reported backup limitations and rotate the credential.')); - showNotice(result.vector_index_cleanup === 'failed' - ? 'Memory removed locally; configured vector index needs separate remediation.' - : 'Memory securely erased from local persistence.'); - await selectWorkspace(state.workspace); - } catch (error) { - showNotice(`Could not securely erase memory: ${error.message}`); - } - } - - function openMemoryTimeline(memory) { - switchView('provenance'); - switchProvenanceTab('timeline'); - byId('timeline-input').value = memory.title || truncate(memory.content, 80); - byId('timeline-form').requestSubmit(); - } - - async function importFiles(files) { - if (!files.length) return; - const form = new FormData(); - form.append('workspace', state.workspace); - form.append('memory_type', 'semantic'); - form.append('derive_facts', 'false'); - [...files].forEach(file => form.append('files', file)); - try { - showNotice(`Importing ${files.length} ${files.length === 1 ? 'file' : 'files'} locally…`); - const result = await api('/workspaces/import-files', { method: 'POST', body: form }); - showNotice(`Import complete${result.count != null ? ` · ${result.count} memories` : ''}.`); - await selectWorkspace(state.workspace); - } catch (error) { - showNotice(`Import failed: ${error.message}`); - } finally { - byId('import-files').value = ''; - } - } - - function renderAnswer(result) { - const target = byId('answer-panel'); - target.replaceChildren(); - const meta = node('div', 'answer-meta'); - const grounded = Boolean(result.grounded); - meta.append( - node('span', `support-pill ${grounded ? 'grounded' : 'abstained'}`, grounded ? 'Grounded' : 'Abstained'), - node('span', 'support-pill', `Support ${number(result.support).toFixed(2)}`), - node('span', 'support-pill', `${(result.citations || []).length} citations`), - ); - target.append(meta); - if (!grounded) { - target.append( - node('h2', '', 'Insufficient evidence'), - node('p', 'answer-copy', result.reason || 'The active workspace does not support a grounded answer.'), - ); - return; - } - target.append(node('p', 'answer-copy', result.answer || 'The cited memories support this answer.')); - const citations = node('div', 'citation-list'); - (result.citations || []).forEach(citation => { - const card = node(citation.id ? 'button' : 'article', 'citation-card memory-link-card'); - if (citation.id) { - card.type = 'button'; - card.dataset.memoryId = citation.id; - card.addEventListener('click', () => openMemory(citation)); - } - card.append( - node('h3', '', `[${citation.n || citation.number || '•'}] ${citation.title || citation.id || 'Memory'}`), - node('p', '', citation.content || citation.summary || ''), - node('div', 'memory-meta', `support ${number(citation.support || citation.score).toFixed(2)} · ${citation.id || ''}`), - ); - citations.append(card); - }); - target.append(citations); - } - - async function askMemory(event) { - event.preventDefault(); - const input = byId('ask-input'); - const question = input.value.trim(); - if (!question) { - showNotice('Enter a question before requesting a grounded answer.'); - input.focus(); - return; - } - if (!state.workspace) { - showNotice('Choose a workspace before requesting a grounded answer.'); - return; - } - const request = beginScopedRequest('ask'); - const workspace = request.workspace; - showNotice(''); - const k = number(byId('ask-k').value) || 5; - byId('answer-panel').replaceChildren(empty('Searching, checking support and building citations…')); - byId('retrieval-list').replaceChildren(empty('Retrieving candidate memories…')); - try { - const [answer, retrieval] = await Promise.all([ - api('/answer', { - method: 'POST', - body: { query: question, workspace, k: Math.max(8, k), max_citations: k }, - }), - // The dashboard /recall route is deliberately read-only (reinforce=False). - // Keep it alongside /answer for uncited raw candidates without a second - // reinforcement of the memories that answer already cited. - api(`/recall?q=${encodeURIComponent(question)}&${query(workspace)}&k=${Math.max(8, k)}`), - ]); - if (!isCurrentScopedRequest(request)) return; - renderAnswer(answer); - const target = byId('retrieval-list'); - target.replaceChildren(); - const memories = retrieval.memories || []; - if (!memories.length) target.append(empty('No raw candidates were returned.')); - else memories.forEach(memory => target.append(simpleMemoryCard(memory))); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - byId('answer-panel').replaceChildren(empty(`Grounded Ask is unavailable: ${error.message}`)); - byId('retrieval-list').replaceChildren(empty('Raw retrieval did not complete.')); - } - } - - function graphNodes(payload) { - const source = payload.nodes || payload.entities || []; - return source.map(item => ({ - id: item.id, - name: item.label || item.name || item.id, - label: item.label || item.name || item.id, - etype: item.etype || item.type || 'person_or_concept', - nodeKind: item.node_kind || item.kind || '', - degree: number(item.degree), - community: Number.isFinite(Number(item.community)) ? Number(item.community) : undefined, - repo: item.repo || '', - topic: item.topic || '', - valid_from: item.valid_from, - valid_to: item.valid_to, - })); - } - - function graphLinks(payload) { - const source = payload.edges || payload.links || []; - return source.map((item, index) => ({ - id: item.id || `edge-${index}`, - source: item.from || (item.source && (item.source.id || item.source)), - target: item.to || (item.target && (item.target.id || item.target)), - label: item.label || item.relation || 'related', - layer: item.layer || 'semantic', - valid_from: item.valid_from, - valid_to: item.valid_to, - })).filter(item => item.source && item.target); - } - - function revealGraphNode(id, label = 'Selected entity') { - const engine = state.graphEngine; - if (!engine) return; - let attempts = 0; - const reveal = () => { - if (state.graphEngine !== engine) return; - if (engine.reveal(id)) return; - attempts += 1; - if (attempts < 8) { - window.requestAnimationFrame(reveal); - return; - } - showNotice(`${label} is outside the current graph scope.`); - }; - reveal(); - } - - function cancelGraphConnectionMemoryLoad() { - state.graphConnectionsRequest += 1; - if (state.graphConnectionsController) state.graphConnectionsController.abort(); - state.graphConnectionsController = null; - } - - function closeGraphConnections() { - cancelGraphConnectionMemoryLoad(); - const dialog = byId('graph-connections-dialog'); - if (dialog.open) dialog.close(); - } - - function graphMemoryCard(evidence) { - return { - id: evidence.memory_id || evidence.id, - title: evidence.title || evidence.label || evidence.memory_id || evidence.id, - content: evidence.excerpt || evidence.content || evidence.summary || '', - mtype: evidence.memory_type || evidence.mtype, - valid_from: evidence.valid_from, - valid_to: evidence.valid_to, - ingested_at: evidence.ingested_at, - provenance: evidence.provenance, - }; - } - - function graphMemoryEvidenceCard(memory) { - const card = node('article', 'graph-memory-evidence'); - card.append( - node('h4', '', memory.title || memory.id || 'Memory'), - node('p', '', truncate(memory.content || memory.summary, 500)), - memoryMeta(memory), - ); - if (memory.id) { - card.append(button('Open in Library', 'secondary-button', () => { - closeGraphConnections(); - openMemory(memory); - })); - } - return card; - } - - function renderGraphConnectionMemories(memories, message) { - const target = byId('graph-connection-memory-list'); - target.replaceChildren(); - if (!memories.length) { - const placeholder = empty(message); - placeholder.setAttribute('role', 'listitem'); - target.append(placeholder); - return; - } - memories.forEach(memory => { - const card = graphMemoryEvidenceCard(memory); - card.setAttribute('role', 'listitem'); - target.append(card); - }); - } - - function isGraphMemoryNode(item) { - const kind = String(item.nodeKind || '').toLowerCase(); - const type = String(item.etype || '').toLowerCase(); - return kind === 'memory' || type === 'memory' || type.startsWith('memory_'); - } - - function graphConnectionEntries(item) { - const graph = state.graphEngine && state.graphEngine.exportData - ? state.graphEngine.exportData() : state.graphData; - if (!graph) return []; - const nodes = new Map(graph.nodes.map(candidate => [candidate.id, candidate])); - const connections = new Map(); - graph.links.forEach(link => { - const source = link.source; - const target = link.target; - if (source !== item.id && target !== item.id) return; - const otherId = source === item.id ? target : source; - const other = nodes.get(otherId); - if (!other || other.id === item.id) return; - const entry = connections.get(other.id) || { item: other, relations: new Set() }; - if (link.label) entry.relations.add(link.label); - connections.set(other.id, entry); - }); - return [...connections.values()].sort((left, right) => { - const degree = number(right.item.degree) - number(left.item.degree); - return degree || left.item.name.localeCompare(right.item.name); - }); - } - - async function showGraphConnectionMemories(item) { - if (!item || !item.id || !state.workspace) return; - cancelGraphConnectionMemoryLoad(); - const request = ++state.graphConnectionsRequest; - const workspace = state.workspace; - const title = item.name || item.label || item.id; - byId('graph-connection-memory-title').textContent = `Memories for ${title}`; - renderGraphConnectionMemories([], 'Loading memory evidence…'); - if (isGraphMemoryNode(item)) { - const known = state.memories.find(memory => memory.id === item.id); - if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; - renderGraphConnectionMemories( - [known || graphMemoryCard(item)], 'No memory details are available for this node.', - ); - return; - } - const controller = new AbortController(); - state.graphConnectionsController = controller; - const timeout = window.setTimeout(() => controller.abort(), GRAPH_CONNECTION_MEMORIES_TIMEOUT_MS); - try { - const detail = await api( - `/graph/entities/${encodeURIComponent(item.id)}/memories?${query(workspace)}${graphAsOfQuery()}`, - { signal: controller.signal }, - ); - if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; - const evidence = detail.evidence || []; - const total = number(detail.totals && detail.totals.evidence) || evidence.length; - byId('graph-connection-memory-title').textContent = `${total} ${total === 1 ? 'memory' : 'memories'} for ${title}`; - renderGraphConnectionMemories( - evidence.map(graphMemoryCard), - 'No active memories support this connected node.', - ); - } catch (error) { - if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; - byId('graph-connection-memory-title').textContent = `Memories for ${title}`; - renderGraphConnectionMemories([], error && error.name === 'AbortError' - ? 'Memory evidence loading timed out. Choose this node again to retry.' - : `Could not load memory evidence: ${error.message}`); - } finally { - window.clearTimeout(timeout); - if (state.graphConnectionsController === controller) state.graphConnectionsController = null; - } - } - - function graphConnectionRow(entry) { - const item = entry.item; - const row = node('article', 'graph-connection-row'); - row.setAttribute('role', 'listitem'); - const details = node('div'); - const relations = [...entry.relations]; - const relationLabel = relations.length ? ` · ${relations.join(', ')}` : ''; - details.append( - node('h3', '', item.name), - node('p', '', `${number(item.degree)} connections · ${item.etype}${relationLabel}`), - ); - const actions = node('div', 'graph-connection-actions'); - actions.append( - button('Focus graph', 'secondary-button', () => { - closeGraphConnections(); - revealGraphNode(item.id, item.name); - }), - button('Memories', 'secondary-button', () => showGraphConnectionMemories(item)), - ); - row.append(details, actions); - return row; - } - - function openGraphConnections(item) { - if (!item || !item.id) return; - cancelGraphConnectionMemoryLoad(); - const dialog = byId('graph-connections-dialog'); - const entries = graphConnectionEntries(item); - const title = item.name || item.label || item.id; - byId('graph-connections-title').textContent = `Connected to ${title}`; - byId('graph-connections-meta').textContent = `${entries.length} direct ${entries.length === 1 ? 'connection' : 'connections'} visible in this graph view`; - const target = byId('graph-connections-list'); - target.replaceChildren(); - if (!entries.length) target.append(empty('No connected nodes are visible in this graph view.')); - else entries.forEach(entry => target.append(graphConnectionRow(entry))); - byId('graph-connection-memory-title').textContent = 'Memories'; - renderGraphConnectionMemories([], 'Choose a connected node to inspect its memory evidence.'); - if (!dialog.open) dialog.showModal(); - } - - function updateGraphFacts(data) { - const stats = byId('graph-stats'); - stats.replaceChildren(); - const degrees = data.nodes.map(item => number(item.degree)).sort((a, b) => a - b); - const values = [ - ['Entities', data.nodes.length], - ['Relations', data.links.length], - ['Unlinked', data.nodes.filter(item => !number(item.degree)).length], - ['Median links', degrees.length ? degrees[Math.floor(degrees.length / 2)] : 0], - ]; - values.forEach(([label, value]) => { - const item = node('div', 'stat-item'); - item.append(node('span', '', label), node('strong', '', number(value).toLocaleString())); - stats.append(item); - }); - const top = byId('graph-top'); - top.replaceChildren(); - [...data.nodes].sort((a, b) => number(b.degree) - number(a.degree)).slice(0, 7).forEach(item => { - const control = node('button', 'compact-row'); - control.type = 'button'; - control.append(node('strong', '', item.name), node('span', '', `${number(item.degree)} connections · ${item.etype}`)); - control.addEventListener('click', () => openGraphConnections(item)); - top.append(control); - }); - } - - function updateGraphModeControls() { - const full = state.graphMode === 'full'; - ['graph-min-degree', 'graph-tune-min-degree', 'graph-collapse'].forEach(id => { - const scopeControl = byId(id); - scopeControl.disabled = full; - scopeControl.title = full - ? 'Full node graph always includes unlinked nodes and never collapses clusters.' - : ''; - }); - const preset = GRAPH_PRESET_LABELS[byId('graph-preset').value] || 'Islands'; - byId('graph-mode').textContent = `${full ? 'Full node graph' : 'Responsive overview'} · ${preset}`; - } - - function setChoicePressed(selector, dataKey, selected) { - all(selector).forEach(control => { - const active = control.dataset[dataKey] === selected; - control.classList.toggle('active', active); - control.setAttribute('aria-pressed', String(active)); - }); - } - - function syncGraphChoices() { - const preset = byId('graph-preset').value; - const style = byId('graph-style').value; - const color = byId('graph-color').value; - const palette = byId('graph-palette').value; - setChoicePressed('[data-graph-preset-choice]', 'graphPresetChoice', preset); - setChoicePressed('[data-graph-style-choice]', 'graphStyleChoice', style); - setChoicePressed('[data-graph-color-choice]', 'graphColorChoice', color); - setChoicePressed('[data-graph-palette-choice]', 'graphPaletteChoice', palette); - byId('graph-style-note').textContent = GRAPH_STYLE_NOTES[style] || GRAPH_STYLE_NOTES.classic; - syncGraphSavedViews(); - } - - function setGraphSwitch(id, on) { - const control = byId(id); - control.classList.toggle('on', on); - control.setAttribute('aria-checked', String(on)); - } - - function graphValueInRange(id, value, fallback) { - const control = byId(id); - const raw = Number(value); - const safe = Number.isFinite(raw) ? raw : fallback; - const min = Number(control.min); - const max = Number(control.max); - return Math.min(Number.isFinite(max) ? max : safe, Math.max(Number.isFinite(min) ? min : safe, safe)); - } - - function graphPresetTuning(preset) { - const available = window.EngraphisGraph && window.EngraphisGraph.PRESETS; - const source = (available && available[preset]) || GRAPH_PRESET_TUNING[preset] || GRAPH_PRESET_TUNING.communities; - return GRAPH_TUNING.reduce((settings, item) => { - settings[item.key] = source && Number.isFinite(Number(source[item.key])) - ? Number(source[item.key]) : item.fallback; - return settings; - }, {}); - } - - function setGraphTuningControl(item, value) { - const control = byId(item.id); - const next = graphValueInRange(item.id, value, item.fallback); - control.value = String(next); - const rendered = item.precision ? next.toFixed(item.precision) : String(Math.round(next)); - const output = byId(`${item.id}-output`); - output.value = rendered; - output.textContent = rendered; - return next; - } - - function graphTuningSettings() { - return GRAPH_TUNING.reduce((settings, item) => { - settings[item.key] = number(byId(item.id).value); - return settings; - }, { flowSpeed: number(byId('graph-flow-speed').value) }); - } - - function syncGraphTuning(settings) { - GRAPH_TUNING.forEach(item => setGraphTuningControl(item, settings && settings[item.key])); - const flowSpeed = graphValueInRange('graph-flow-speed', settings && settings.flowSpeed, 45); - byId('graph-flow-speed').value = String(flowSpeed); - byId('graph-flow-speed-output').value = String(Math.round(flowSpeed)); - byId('graph-flow-speed-output').textContent = String(Math.round(flowSpeed)); - } - - function graphScope() { - const full = state.graphMode === 'full'; - return { - minDegree: full ? 0 : number(byId('graph-min-degree').value), - showUnlinked: full || state.graphShowUnlinked, - depth: number(byId('graph-depth').value), - }; - } - - function applyGraphScope() { - if (state.graphEngine) state.graphEngine.setScope(graphScope()); - } - - function setGraphMinDegree(value, apply = true) { - const next = graphValueInRange('graph-min-degree', value, 1); - byId('graph-min-degree').value = String(next); - byId('graph-min-degree-output').value = String(Math.round(next)); - byId('graph-min-degree-output').textContent = String(Math.round(next)); - byId('graph-tune-min-degree').value = String(next); - byId('graph-tune-min-degree-output').value = String(Math.round(next)); - byId('graph-tune-min-degree-output').textContent = String(Math.round(next)); - if (apply) applyGraphScope(); - } - - function setGraphDepth(value, apply = true) { - const next = graphValueInRange('graph-depth', value, 2); - byId('graph-depth').value = String(next); - byId('graph-depth-output').value = String(Math.round(next)); - byId('graph-depth-output').textContent = String(Math.round(next)); - if (apply) applyGraphScope(); - } - - function setGraphShowUnlinked(on, apply = true) { - const next = on === true; - state.graphShowUnlinked = next; - const control = byId('graph-show-unlinked'); - control.textContent = next ? 'Hide unlinked nodes' : 'Show unlinked nodes'; - control.setAttribute('aria-pressed', String(next)); - control.title = next - ? 'Hide entities that have no relations in this graph view' - : 'Show entities that have no relations in this graph view'; - if (apply) applyGraphScope(); - } - - function graphLayerState() { - return all('[data-graph-layer]').reduce((layers, control) => { - layers[control.dataset.graphLayer] = control.getAttribute('aria-pressed') === 'true'; - return layers; - }, {}); - } - - function setGraphLayers(layers) { - const source = layers && typeof layers === 'object' ? layers : GRAPH_DEFAULT_LAYERS; - all('[data-graph-layer]').forEach(control => { - const active = source[control.dataset.graphLayer] !== false; - control.classList.toggle('active', active); - control.setAttribute('aria-pressed', String(active)); - }); - } - - function updateGraphLayerCounts(data, supplied) { - const counts = GRAPH_LAYERS.reduce((result, layer) => { result[layer] = 0; return result; }, {}); - if (Array.isArray(supplied)) supplied.forEach(item => { - if (item && GRAPH_LAYERS.includes(item.layer)) counts[item.layer] = number(item.count); - }); - else (data.links || []).forEach(link => { - if (GRAPH_LAYERS.includes(link.layer)) counts[link.layer] += 1; - }); - GRAPH_LAYERS.forEach(layer => { byId(`graph-layer-${layer}-count`).textContent = counts[layer].toLocaleString(); }); - } - - function syncGraphSavedViews() { - all('[data-graph-saved-view]').forEach(control => { - const active = control.dataset.graphSavedView === state.graphSavedView; - control.classList.toggle('active', active); - control.setAttribute('aria-pressed', String(active)); - }); - } - - function clearGraphSavedView() { - if (!state.graphSavedView) return; - state.graphSavedView = ''; - syncGraphSavedViews(); - } - - function graphPreference(name, fallback, allowed) { - try { - const saved = JSON.parse(localStorage.getItem(GRAPH_PREFERENCES_KEY) || '{}'); - const value = saved && typeof saved === 'object' ? saved[name] : undefined; - return allowed && !allowed.includes(value) ? fallback : value === undefined ? fallback : value; - } catch (_) { - return fallback; - } - } - - function graphPreferenceSnapshot() { - return { - preset: byId('graph-preset').value, - style: byId('graph-style').value, - color: byId('graph-color').value, - palette: byId('graph-palette').value, - flow: byId('graph-flow').getAttribute('aria-checked') === 'true', - labels: byId('graph-labels').getAttribute('aria-checked') === 'true', - tuning: graphTuningSettings(), - minDegree: number(byId('graph-min-degree').value), - depth: number(byId('graph-depth').value), - showUnlinked: state.graphShowUnlinked, - layers: graphLayerState(), - includeCode: state.graphIncludeCode, - savedView: state.graphSavedView, - bridges: byId('graph-bridges').checked, - collapse: byId('graph-collapse').checked, - asOf: byId('graph-as-of').value, - ghosts: byId('graph-ghosts').checked, - size: byId('graph-size').value, - repoFilter: byId('graph-repo-filter').value.slice(0, 200), - }; - } - - function saveGraphPreferences() { - try { - localStorage.setItem(GRAPH_PREFERENCES_KEY, JSON.stringify(graphPreferenceSnapshot())); - } catch (_) {} - } - - function restoreGraphPreferences() { - const preset = graphPreference('preset', byId('graph-preset').value, - ['original', 'compact', 'communities', 'radial', 'constellation']); - const style = graphPreference('style', byId('graph-style').value, - ['classic', 'galaxy', 'solar', 'cyber']); - const color = graphPreference('color', byId('graph-color').value, - ['community', 'connections', 'type']); - const palette = graphPreference('palette', byId('graph-palette').value, - ['theme', 'aurora', 'ocean', 'ember', 'contrast', 'custom']); - byId('graph-preset').value = preset; - byId('graph-style').value = style; - byId('graph-color').value = color; - byId('graph-palette').value = palette; - - const savedTuning = graphPreference('tuning', {}); - syncGraphTuning({ - ...graphPresetTuning(preset), - ...(savedTuning && typeof savedTuning === 'object' ? savedTuning : {}), - }); - - const savedMin = Number(graphPreference('minDegree', number(byId('graph-min-degree').value))); - const minDegree = Number.isFinite(savedMin) ? Math.max(0, Math.min(12, Math.round(savedMin))) : 1; - setGraphMinDegree(minDegree); - setGraphDepth(graphPreference('depth', 2)); - const savedRepo = graphPreference('repoFilter', ''); - byId('graph-repo-filter').value = typeof savedRepo === 'string' ? savedRepo.slice(0, 200) : ''; - const savedAsOf = graphPreference('asOf', ''); - byId('graph-as-of').value = typeof savedAsOf === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(savedAsOf) - ? savedAsOf : ''; - setGraphShowUnlinked(graphPreference('showUnlinked', state.graphShowUnlinked) === true); - byId('graph-bridges').checked = graphPreference('bridges', byId('graph-bridges').checked) === true; - byId('graph-collapse').checked = graphPreference('collapse', byId('graph-collapse').checked) === true; - byId('graph-ghosts').checked = graphPreference('ghosts', byId('graph-ghosts').checked) !== false; - byId('graph-size').value = graphPreference('size', byId('graph-size').value, - ['degree', 'betweenness']); - // Freeze is deliberately session-only. A previously frozen arrangement must not make a - // freshly opened graph look broken; physics starts live until the person clicks Freeze. - state.graphFrozen = false; - setGraphSwitch('graph-freeze', state.graphFrozen); - setGraphSwitch('graph-flow', graphPreference('flow', true) !== false); - setGraphSwitch('graph-labels', graphPreference('labels', false) === true); - const savedLayers = graphPreference('layers', GRAPH_DEFAULT_LAYERS); - setGraphLayers(GRAPH_LAYERS.reduce((layers, layer) => { - layers[layer] = !savedLayers || typeof savedLayers !== 'object' || savedLayers[layer] !== false; - return layers; - }, {})); - state.graphIncludeCode = graphPreference('includeCode', false) === true; - state.graphSavedView = graphPreference('savedView', 'schema', ['', ...Object.keys(GRAPH_SAVED_VIEWS)]); - syncGraphSavedViews(); - } - - function savedGraphView(id) { - if (id === 'custom') { - try { - const custom = JSON.parse(localStorage.getItem(GRAPH_CUSTOM_VIEW_KEY) || 'null'); - return custom && typeof custom === 'object' ? custom : null; - } catch (_) { - return null; - } - } - return GRAPH_SAVED_VIEWS[id] || null; - } - - function applyGraphView(id) { - const view = savedGraphView(id); - if (!view) { - showNotice(id === 'custom' ? 'No locally saved graph view yet.' : 'That saved graph view is unavailable.'); - return; - } - const preset = Object.prototype.hasOwnProperty.call(GRAPH_PRESET_LABELS, view.preset) - ? view.preset : byId('graph-preset').value; - const style = ['classic', 'galaxy', 'solar', 'cyber'].includes(view.style) ? view.style : byId('graph-style').value; - const color = ['community', 'connections', 'type'].includes(view.color) ? view.color : byId('graph-color').value; - const palette = ['theme', 'aurora', 'ocean', 'ember', 'contrast', 'custom'].includes(view.palette) - ? view.palette : byId('graph-palette').value; - const previousIncludeCode = state.graphIncludeCode; - const previousShowUnlinked = state.graphShowUnlinked; - const previousAsOf = byId('graph-as-of').value; - const asOf = typeof view.asOf === 'string' ? view.asOf : previousAsOf; - const repoFilter = typeof view.repoFilter === 'string' - ? view.repoFilter.slice(0, 200) : byId('graph-repo-filter').value; - state.graphIncludeCode = typeof view.includeCode === 'boolean' - ? view.includeCode : state.graphIncludeCode; - byId('graph-preset').value = preset; - byId('graph-style').value = style; - byId('graph-color').value = color; - byId('graph-palette').value = palette; - byId('graph-as-of').value = asOf; - byId('graph-repo-filter').value = repoFilter; - if (typeof view.ghosts === 'boolean') byId('graph-ghosts').checked = view.ghosts; - if (['degree', 'betweenness'].includes(view.size)) byId('graph-size').value = view.size; - if (typeof view.bridges === 'boolean') byId('graph-bridges').checked = view.bridges; - if (typeof view.collapse === 'boolean') byId('graph-collapse').checked = view.collapse; - if (typeof view.flow === 'boolean') setGraphSwitch('graph-flow', view.flow); - if (typeof view.labels === 'boolean') setGraphSwitch('graph-labels', view.labels); - setGraphSwitch('graph-freeze', state.graphFrozen); - syncGraphTuning({ - ...graphPresetTuning(preset), - ...(view.tuning && typeof view.tuning === 'object' ? view.tuning : {}), - }); - setGraphMinDegree(view.minDegree == null ? 1 : view.minDegree, false); - setGraphDepth(view.depth == null ? 2 : view.depth, false); - setGraphShowUnlinked(view.showUnlinked === true, false); - setGraphLayers(view.layers); - state.graphSavedView = id === 'custom' ? '' : id; - syncGraphChoices(); - if (state.graphEngine) { - state.graphEngine.apply(graph => { - graph.setPreset(preset); - graph.setStyle(style); - graph.setColorBy(color); - applyGraphPalette(palette); - graph.setSettings({ - ...graphTuningSettings(), - flow: byId('graph-flow').getAttribute('aria-checked') === 'true', - labels: byId('graph-labels').getAttribute('aria-checked') === 'true', - frozen: state.graphFrozen, - }); - graph.setScope(graphScope()); - graph.setLayers(graphLayerState()); - graph.setRepoFilter(repoFilter); - graph.setAsOf(graphAsOfTimestamp()); - graph.setSizeBy(byId('graph-size').value); - graph.setBridges(byId('graph-bridges').checked); - graph.setCollapse(byId('graph-collapse').checked ? 'auto' : false); - graph.setGhosts(byId('graph-ghosts').checked); - }, false, !state.graphFrozen); - state.graphEngine.freeze(state.graphFrozen); - } - saveGraphPreferences(); - if (previousIncludeCode !== state.graphIncludeCode - || previousShowUnlinked !== state.graphShowUnlinked || previousAsOf !== asOf) { - loadGraph({ force: true }); - } - const label = all('[data-graph-saved-view]').find(control => control.dataset.graphSavedView === id); - showNotice(`${id === 'custom' ? 'Saved' : (label ? label.textContent : 'Saved')} graph view applied.`); - } - - function saveCurrentGraphView() { - try { - localStorage.setItem(GRAPH_CUSTOM_VIEW_KEY, JSON.stringify(graphPreferenceSnapshot())); - byId('graph-saved-view-status').textContent = 'Current graph view saved locally.'; - showNotice('Current graph view saved locally.'); - } catch (_) { - showNotice('Could not save this graph view in local storage.'); - } - } - - function resetGraphTuning() { - const preset = byId('graph-preset').value; - const previousIncludeCode = state.graphIncludeCode; - const previousShowUnlinked = state.graphShowUnlinked; - state.graphIncludeCode = false; - syncGraphTuning({ ...graphPresetTuning(preset), flowSpeed: 45 }); - setGraphMinDegree(1, false); - setGraphDepth(2, false); - setGraphShowUnlinked(false, false); - setGraphLayers(GRAPH_DEFAULT_LAYERS); - clearGraphSavedView(); - if (state.graphEngine) { - state.graphEngine.apply(graph => { - graph.setPreset(preset); - graph.setSettings({ ...graphTuningSettings(), frozen: state.graphFrozen }); - graph.setScope(graphScope()); - graph.setLayers(graphLayerState()); - }, false, !state.graphFrozen); - state.graphEngine.freeze(state.graphFrozen); - } - saveGraphPreferences(); - if (previousIncludeCode || previousShowUnlinked) loadGraph({ force: true }); - showNotice('Graph tuning reset to the selected layout defaults.'); - } - - function applyGraphPalette(name) { - const graph = state.graphEngine; - if (!graph) return; - graph.setPalette(name); - if (name === 'custom') graph.setTypeColors(GRAPH_CUSTOM_PALETTE); - } - - function graphThemeColors() { - const css = getComputedStyle(document.body); - return { - accent: css.getPropertyValue('--c-acc').trim() || '#a39bf1', - surface: css.getPropertyValue('--c-surface').trim() || '#16191f', - canvas: css.getPropertyValue('--c-bg').trim() || '#0e1014', - label: css.getPropertyValue('--c-fg').trim() || '#e7e9ee', - relation_label: css.getPropertyValue('--c-dim').trim() || '#929baa', - }; - } - - function setGraphTab(tab) { - all('[data-graph-tab]').forEach(control => { - const active = control.dataset.graphTab === tab; - control.classList.toggle('active', active); - control.setAttribute('aria-selected', String(active)); - }); - all('[data-graph-tab-panel]').forEach(panel => { - panel.hidden = panel.dataset.graphTabPanel !== tab; - }); - } - - function downloadGraphFile(blob, name) { - const href = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = href; - link.download = name; - document.body.append(link); - link.click(); - link.remove(); - window.setTimeout(() => URL.revokeObjectURL(href), 0); - } - - function exportGraphJson() { - const graph = state.graphEngine && state.graphEngine.exportData - ? state.graphEngine.exportData() - : state.graphData || { nodes: [], links: [] }; - const payload = { - workspace: state.workspace, - exported_at: new Date().toISOString(), - nodes: graph.nodes, - links: graph.links, - }; - downloadGraphFile(new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }), 'engraphis-graph.json'); - showNotice('Graph data exported as JSON.'); - } - - function exportGraphPng() { - const canvas = byId('graph-canvas').querySelector('canvas'); - if (!canvas || !canvas.toBlob) { - showNotice('The graph image is not ready yet. Export JSON data instead.'); - return; - } - canvas.toBlob(blob => { - if (!blob) { - showNotice('Could not capture the graph image. Export JSON data instead.'); - return; - } - downloadGraphFile(blob, 'engraphis-graph.png'); - showNotice('Graph image exported as PNG.'); - }, 'image/png'); - } - - function graphCountText(nodes, links) { - const available = number(state.graphMeta && state.graphMeta.nodes_available) || nodes; - const prefix = state.graphMode === 'full' && state.graphMeta && state.graphMeta.nodes_complete - ? 'Full graph' - : 'Overview'; - const entityText = available > nodes - ? `${number(nodes).toLocaleString()} of ${available.toLocaleString()} entities` - : `${number(nodes).toLocaleString()} entities`; - return `${prefix} · ${entityText} · ${number(links).toLocaleString()} relations`; - } - - function graphStatsChanged(stats) { - if (!stats) return; - const nodes = stats.nodes == null ? state.graphData.nodes.length : stats.nodes; - const links = stats.links == null ? state.graphData.links.length : stats.links; - byId('graph-count').textContent = graphCountText(nodes, links); - } - - function graphMetricsChanged(metrics) { - state.graphMetrics = metrics || {}; - byId('graph-bridge-count').textContent = metrics && metrics.bridges != null - ? `${metrics.bridges} bridge ${metrics.bridges === 1 ? 'edge' : 'edges'}` - : ''; - } - - function graphAsOfTimestamp() { - const value = byId('graph-as-of').value; - if (!value) return null; - // A date picker represents the complete selected day, not midnight at its start. - const timestamp = Date.parse(`${value}T23:59:59.999Z`); - return Number.isFinite(timestamp) ? timestamp : null; - } - - function graphAsOfQuery() { - const timestamp = graphAsOfTimestamp(); - return timestamp === null ? '' : `&as_of=${encodeURIComponent(timestamp / 1000)}`; - } - - async function loadGraph({ force = false } = {}) { - if (!state.workspace) return; - if (!force && state.graphWorkspace === state.workspace - && state.graphDataMode === state.graphMode - && state.graphDataIncludeCode === state.graphIncludeCode - && state.graphDataShowUnlinked === state.graphShowUnlinked - && state.graphDataAsOf === graphAsOfTimestamp() && state.graphData) { - if (state.graphEngine) state.graphEngine.resize(); - return; - } - const targetWorkspace = state.workspace; - const targetMode = state.graphMode; - const targetIncludeCode = state.graphIncludeCode; - const targetShowUnlinked = state.graphShowUnlinked; - const targetAsOf = graphAsOfTimestamp(); - const fullGraph = targetMode === 'full'; - if (state.graphLoadPromise && state.graphLoadWorkspace === targetWorkspace - && state.graphLoadMode === targetMode && state.graphLoadIncludeCode === targetIncludeCode - && state.graphLoadShowUnlinked === targetShowUnlinked - && state.graphLoadAsOf === targetAsOf) { - return state.graphLoadPromise; - } - if (state.graphLoadPromise && state.graphLoadController) state.graphLoadController.abort(); - byId('graph-empty').hidden = false; - byId('graph-empty').textContent = fullGraph - ? 'Loading every available graph node…' - : 'Loading the responsive evidence graph…'; - const task = (async () => { - const controller = new AbortController(); - state.graphLoadController = controller; - const timeout = window.setTimeout( - () => controller.abort(), - fullGraph ? GRAPH_FULL_LOAD_TIMEOUT_MS : GRAPH_LOAD_TIMEOUT_MS, - ); - try { - const limit = fullGraph ? GRAPH_FULL_NODE_LIMIT : GRAPH_INITIAL_NODE_LIMIT; - const complete = fullGraph ? '&full=true' : ''; - const connectedOnly = !fullGraph && !targetShowUnlinked ? '&connected_only=true' : ''; - const includeCode = targetIncludeCode ? '&include_code=true' : ''; - const asOf = targetAsOf === null ? '' : `&as_of=${encodeURIComponent(targetAsOf / 1000)}`; - const [payload] = await Promise.all([ - api(`/graph?${query(targetWorkspace)}&limit=${limit}${complete}${connectedOnly}${includeCode}${asOf}`, { signal: controller.signal }), - ensureGraphAssets(), - ]); - if (state.workspace !== targetWorkspace || state.graphMode !== targetMode - || state.graphIncludeCode !== targetIncludeCode - || state.graphShowUnlinked !== targetShowUnlinked - || graphAsOfTimestamp() !== targetAsOf) return; - const data = { nodes: graphNodes(payload), links: graphLinks(payload), suggestions: payload.suggestions || [] }; - state.graphData = data; - state.graphWorkspace = targetWorkspace; - state.graphDataMode = targetMode; - state.graphDataIncludeCode = targetIncludeCode; - state.graphDataShowUnlinked = targetShowUnlinked; - state.graphDataAsOf = targetAsOf; - state.graphMeta = payload.meta || { - nodes_available: data.nodes.length, - nodes_complete: fullGraph, - }; - if (state.graphEngine) state.graphEngine.destroy(); - if (typeof window.EngraphisGraph === 'undefined') throw new Error('graph engine asset is unavailable'); - state.graphEngine = window.EngraphisGraph.create(byId('graph-canvas'), { - renderMode: targetMode, - onNodeClick: item => openGraphConnections(item), - onBackgroundClick: () => state.graphEngine && state.graphEngine.clearFocus(), - onStats: graphStatsChanged, - onMetrics: graphMetricsChanged, - onCollapseChange: collapsed => { - if (targetMode === 'overview') showNotice(collapsed ? 'Clusters collapsed for overview.' : ''); - }, - }); - state.graphEngine.apply(graph => { - graph.setPreset(byId('graph-preset').value); - graph.setStyle(byId('graph-style').value); - graph.setColorBy(byId('graph-color').value); - graph.setThemeColors(graphThemeColors()); - applyGraphPalette(byId('graph-palette').value); - graph.setSettings({ - ...graphTuningSettings(), - flow: byId('graph-flow').getAttribute('aria-checked') === 'true', - labels: byId('graph-labels').getAttribute('aria-checked') === 'true', - frozen: state.graphFrozen, - }); - graph.setScope(graphScope()); - graph.setLayers(graphLayerState()); - graph.setRepoFilter(byId('graph-repo-filter').value); - graph.setAsOf(graphAsOfTimestamp()); - graph.setSizeBy(byId('graph-size').value); - graph.setBridges(byId('graph-bridges').checked); - graph.setCollapse(fullGraph ? false : (byId('graph-collapse').checked ? 'auto' : false)); - graph.setGhosts(byId('graph-ghosts').checked); - }, false, false); - state.graphEngine.setData(data); - state.graphEngine.freeze(state.graphFrozen); - byId('graph-empty').hidden = Boolean(data.nodes.length); - if (!data.nodes.length) byId('graph-empty').textContent = 'No entities exist in this workspace yet.'; - updateGraphModeControls(); - updateGraphFacts(data); - updateGraphLayerCounts(data, payload.layers); - } catch (error) { - if (state.workspace !== targetWorkspace || state.graphMode !== targetMode) return; - byId('graph-empty').hidden = false; - byId('graph-empty').textContent = error && error.name === 'AbortError' - ? `${fullGraph ? 'Full graph' : 'Graph'} loading timed out. Choose Retry to try again.` - : `Graph unavailable: ${error.message}`; - } finally { - window.clearTimeout(timeout); - if (state.graphLoadController === controller) state.graphLoadController = null; - } - })(); - state.graphLoadWorkspace = targetWorkspace; - state.graphLoadMode = targetMode; - state.graphLoadIncludeCode = targetIncludeCode; - state.graphLoadShowUnlinked = targetShowUnlinked; - state.graphLoadAsOf = targetAsOf; - state.graphLoadPromise = task; - try { - return await task; - } finally { - if (state.graphLoadPromise === task) { - state.graphLoadPromise = null; - state.graphLoadWorkspace = ''; - state.graphLoadMode = ''; - state.graphLoadIncludeCode = false; - state.graphLoadShowUnlinked = false; - state.graphLoadAsOf = null; - } - } - } - - function searchGraph(value) { - const target = byId('graph-search-results'); - target.replaceChildren(); - const needle = value.trim().toLowerCase(); - if (!needle || !state.graphData) return; - state.graphData.nodes - .filter(item => item.name.toLowerCase().includes(needle)) - .slice(0, 8) - .forEach(item => { - target.append(button(`${item.name} · ${item.degree}`, 'search-result', () => { - revealGraphNode(item.id, item.name); - target.replaceChildren(); - openGraphConnections(item); - })); - }); - } - - function renderMemoryCollection(target, memories, message) { - target.replaceChildren(); - if (!memories.length) { - target.append(empty(message)); - return; - } - memories.forEach(memory => target.append(simpleMemoryCard(memory))); - } - - function switchProvenanceTab(tab) { - state.provenanceTab = tab; - all('[data-provenance-tab]').forEach(control => { - const active = control.dataset.provenanceTab === tab; - control.classList.toggle('active', active); - control.setAttribute('aria-selected', String(active)); - }); - all('[data-provenance-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.provenancePanel === tab)); - if (tab === 'audit') loadAudit(); - } - - async function whySearch(event) { - event.preventDefault(); - const question = byId('why-input').value.trim(); - if (!question) { - showNotice('Enter a claim or topic before tracing belief.'); - byId('why-input').focus(); - return; - } - const request = beginScopedRequest('why'); - showNotice(''); - const target = byId('why-result'); - target.replaceChildren(empty('Tracing the live belief and supersession chain…')); - try { - const payload = await api(`/why?q=${encodeURIComponent(question)}&${query(request.workspace)}&k=8`); - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(); - const live = payload.answer || []; - const superseded = payload.supersedes || []; - target.append(node('h2', '', 'Live support')); - if (!live.length) target.append(empty('No live supporting memory was found.')); - else live.forEach(memory => target.append(simpleMemoryCard(memory))); - target.append(node('h2', '', 'Superseded history')); - if (!superseded.length) target.append(empty('No superseded versions were found.')); - else superseded.forEach(memory => target.append(simpleMemoryCard(memory, 'timeline-card'))); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(empty(`Could not trace belief: ${error.message}`)); - } - } - - async function timelineSearch(event, supersessionsOnly = false) { - event.preventDefault(); - const input = byId(supersessionsOnly ? 'supersession-input' : 'timeline-input'); - const target = byId(supersessionsOnly ? 'supersession-list' : 'timeline-result'); - const question = input.value.trim(); - if (!question) { - showNotice(`Enter a topic before ${supersessionsOnly ? 'finding supersessions' : 'showing history'}.`); - input.focus(); - return; - } - const request = beginScopedRequest(supersessionsOnly ? 'supersessions' : 'timeline'); - showNotice(''); - target.replaceChildren(empty('Loading temporal history…')); - try { - const payload = await api(`/timeline?q=${encodeURIComponent(question)}&${query(request.workspace)}&limit=50`); - if (!isCurrentScopedRequest(request)) return; - let history = payload.history || []; - if (supersessionsOnly) history = history.filter(item => item.valid_to || item.expired_at); - renderMemoryCollection(target, history, supersessionsOnly ? 'No closed versions were found for this topic.' : 'No temporal history was found.'); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(empty(`Could not load history: ${error.message}`)); - } - } - - function renderAuditCards(audit, receipts) { - const target = byId('audit-list'); - target.replaceChildren(); - const combined = [ - ...audit.map(item => ({ ...item, _kind: 'audit' })), - ...receipts.map(item => ({ ...item, _kind: 'receipt' })), - ].sort((a, b) => provenanceTimestampMs(b) - provenanceTimestampMs(a)); - if (!combined.length) { - target.append(empty('No audit records or receipts yet.')); - return; - } - combined.slice(0, 120).forEach(item => { - const card = node('article', 'audit-card'); - card.append( - node('span', '', relative(provenanceTimestampMs(item))), - node('strong', '', item.actor || item.source || 'local operator'), - node('span', 'tag', item.operation || item.action || item.event || item._kind), - node('span', '', item.scope || item.workspace || item.status || state.workspace), - node('code', '', truncate(item.hash || item.id || item.receipt_id, 24) || '—'), - ); - target.append(card); - }); - } - - async function loadAudit() { - const request = beginScopedRequest('audit'); - const target = byId('audit-list'); - target.replaceChildren(empty('Loading audit records and receipts…')); - try { - const [audit, receipts, savings] = await Promise.all([ - api(`/audit?${query(request.workspace)}&limit=100`), - api(`/receipts?${query(request.workspace)}&limit=100`), - api(`/context-savings?${savingsQuery(request.workspace, state.savingsPreset)}`), - ]); - if (!isCurrentScopedRequest(request)) return; - renderSavingsDetail(savings); - renderAuditCards(auditItems(audit), receiptItems(receipts)); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(empty(`Could not load provenance records: ${error.message}`)); - } - } - - async function verifyReceipts() { - try { - const result = await api(`/receipts/verify?${query()}`); - const valid = result.valid != null ? result.valid : result.verified; - showNotice(valid === false ? 'Receipt verification found a broken chain.' : 'Receipt chain verified.'); - } catch (error) { - showNotice(`Could not verify receipts: ${error.message}`); - } - } - - async function exportReceipts() { - try { - const receipts = await api(`/receipts/export?${query()}`); - const blob = new Blob([JSON.stringify(receipts, null, 2)], { type: 'application/json' }); - const link = document.createElement('a'); - const url = URL.createObjectURL(blob); - link.href = url; - link.download = `engraphis-receipts-${state.workspace || 'workspace'}.json`; - document.body.append(link); - link.click(); - link.remove(); - URL.revokeObjectURL(url); - showNotice('Privacy-safe receipts exported.'); - } catch (error) { - showNotice(`Could not export receipts: ${error.message}`); - } - } - - function switchManageTab(tab) { - state.manageTab = tab; - all('[data-manage-tab]').forEach(control => { - const active = control.dataset.manageTab === tab; - control.classList.toggle('active', active); - control.setAttribute('aria-selected', String(active)); - }); - all('[data-manage-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.managePanel === tab)); - loadManageTab(tab); - } - - async function loadManageTab(tab) { - if (tab === 'workspaces') renderWorkspaceList(); - if (tab === 'settings') await loadSettings(); - if (tab === 'plans') await loadPlans(); - if (tab === 'analytics') await loadHosted('analytics'); - if (tab === 'automation') await loadHosted('automation'); - if (tab === 'team') await loadHosted('team'); - if (tab === 'sync') await loadSync(); - } - - function renderWorkspaceList() { - const target = byId('workspace-list'); - target.replaceChildren(); - if (!state.workspaces.length) { - target.append(empty('Create the first workspace to begin.')); - return; - } - state.workspaces.forEach(item => { - const name = workspaceName(item); - const card = node('article', `workspace-card${name === state.workspace ? ' active' : ''}`); - const copy = node('div'); - copy.append( - node('h3', '', name), - node('p', '', item.description || `${number(item.memories).toLocaleString()} memories · ${item.visibility || 'local'}`), - ); - const actions = node('div', 'workspace-card-actions'); - if (name !== state.workspace) actions.append(button('Switch to', 'secondary-button', () => selectWorkspace(name))); - actions.append( - button('Rename', 'secondary-button', () => renameWorkspace(name)), - button('Copy', 'secondary-button', () => copyWorkspace(name)), - ); - if (name !== state.workspace) actions.append(button('Delete', 'danger-button', () => deleteWorkspace(name))); - card.append(copy, actions); - target.append(card); - }); - } - - async function createWorkspace(event) { - event.preventDefault(); - const name = byId('new-workspace-name').value.trim(); - const description = byId('new-workspace-description').value.trim(); - if (!name) { - showNotice('Enter a workspace name before creating it.'); - byId('new-workspace-name').focus(); - return; - } - showNotice(''); - try { - await api('/workspaces/create', { - method: 'POST', - body: { workspace: name, description, visibility: 'personal', confirmed: false }, - }); - showNotice(`Workspace ${name} created.`); - byId('create-workspace-form').reset(); - byId('create-workspace-form').hidden = true; - await refreshBootstrap(name); - } catch (error) { - showNotice(`Could not create workspace: ${error.message}`); - } - } - - async function renameWorkspace(name) { - const next = window.prompt(`Rename ${name} to:`, name); - if (!next || next === name) return; - try { - await api('/workspaces/rename', { method: 'POST', body: { workspace: name, new_name: next } }); - showNotice(`Workspace renamed to ${next}.`); - await refreshBootstrap(name === state.workspace ? next : state.workspace); - } catch (error) { - showNotice(`Could not rename workspace: ${error.message}`); - } - } - - async function copyWorkspace(name) { - try { - const result = await api('/workspaces/copy', { method: 'POST', body: { workspace: name } }); - showNotice(`Workspace copied${result.name ? ` to ${result.name}` : ''}.`); - await refreshBootstrap(state.workspace); - } catch (error) { - showNotice(`Could not copy workspace: ${error.message}`); - } - } - - async function deleteWorkspace(name) { - if (!window.confirm(`Delete workspace “${name}”? Its memories are retired through the governed workspace operation.`)) return; - try { - await api('/workspaces/delete', { method: 'POST', body: { workspace: name } }); - showNotice(`Workspace ${name} deleted.`); - await refreshBootstrap(state.workspace); - } catch (error) { - showNotice(`Could not delete workspace: ${error.message}`); - } - } - - function renderObject(target, payload, title = 'Result') { - target.replaceChildren(); - target.append(node('h3', '', title)); - const entries = Object.entries(payload || {}).filter(([, value]) => ['string', 'number', 'boolean'].includes(typeof value)).slice(0, 12); - if (entries.length) target.append(definitionList(entries.map(([key, value]) => [key.replaceAll('_', ' '), text(value)]))); - else target.append(node('p', '', 'The operation completed.')); - } - - function consolidationOptions() { - return { - workspace: state.workspace, - infer: false, - structured: byId('consolidate-structured').checked, - }; - } - - function sameConsolidationOptions(left, right) { - return Boolean(left && right) - && left.workspace === right.workspace - && left.infer === right.infer - && left.structured === right.structured; - } - - function invalidateConsolidationReview() { - state.consolidationReview = null; - byId('consolidate-commit').disabled = true; - } - - async function previewConsolidation(event) { - event.preventDefault(); - const options = consolidationOptions(); - invalidateConsolidationReview(); - const target = byId('consolidate-result'); - target.replaceChildren(empty('Scanning local memory without writing changes…')); - try { - const result = await api('/consolidate', { - method: 'POST', - body: { - ...options, - dry_run: true, - }, - }); - // The preview is an approval only for the exact workspace and choices that - // produced it; never let a late response authorize a changed form. - if (!sameConsolidationOptions(options, consolidationOptions())) return; - state.consolidationReview = options; - byId('consolidate-commit').disabled = false; - renderObject(target, result, 'Dry preview complete · nothing written'); - } catch (error) { - invalidateConsolidationReview(); - target.replaceChildren(empty(`Preview failed: ${error.message}`)); - } - } - - async function commitConsolidation() { - const options = consolidationOptions(); - if (!sameConsolidationOptions(state.consolidationReview, options)) { - invalidateConsolidationReview(); - showNotice('Run a new dry preview after changing the workspace or consolidation options.'); - return; - } - if (!window.confirm(`Commit the reviewed consolidation result for ${state.workspace}? Original records remain in temporal history.`)) return; - const target = byId('consolidate-result'); - target.replaceChildren(empty('Committing the reviewed local consolidation…')); - try { - const result = await api('/consolidate', { - method: 'POST', - body: { - ...options, - dry_run: false, - }, - }); - invalidateConsolidationReview(); - renderObject(target, result, 'Consolidation committed'); - await selectWorkspace(state.workspace); - } catch (error) { - target.replaceChildren(empty(`Commit failed: ${error.message}`)); - } - } - - function automationCheckbox(id, label, checked) { - const field = node('label', 'check-row'); - const input = node('input'); - input.id = id; - input.type = 'checkbox'; - input.checked = Boolean(checked); - field.htmlFor = id; - field.append(input, document.createTextNode(label)); - return field; - } - - function automationNumber(id, label, value, min, max) { - const field = node('label', '', label); - const input = node('input'); - input.id = id; - input.type = 'number'; - input.min = String(min); - input.max = String(max); - input.value = String(value); - field.htmlFor = id; - field.append(input); - return field; - } - - function renderAutomationPolicy(policy, workspace = state.workspace) { - const target = byId('automation-result'); - if (!target) return; - target.replaceChildren(); - const form = node('form', 'automation-policy-form'); - form.dataset.workspace = workspace; - form.dataset.lastRun = String(policy.last_run || ''); - if (policy.bootstrap_required) { - form.append( - node('p', 'automation-policy-note', 'Hosted automation is not initialized for this workspace. Initializing it uploads one bounded workspace snapshot and saves the default Cloud policy. No upload occurs until you choose this action.'), - ); - const actions = node('div', 'automation-policy-actions'); - const bootstrap = node('button', 'primary-button', 'Initialize hosted automation'); - bootstrap.type = 'button'; - bootstrap.addEventListener('click', () => bootstrapAutomation(workspace, bootstrap)); - actions.append(bootstrap); - form.append(actions); - target.append(form); - return; - } - const enabled = Boolean(policy.enabled); - const dreamEnabled = policy.dream_enabled != null ? policy.dream_enabled : policy.dream; - const lastRun = policy.last_run ? ` Last managed run: ${relative(policy.last_run)}.` : ''; - form.append( - node('p', 'automation-policy-note', enabled - ? `This workspace has an active hosted maintenance policy.${lastRun}` - : 'Hosted maintenance is paused for this workspace.'), - automationCheckbox('automation-enabled', 'Enable hosted maintenance', enabled), - automationNumber('automation-cadence', 'Run every (hours)', Math.max(1, Number(policy.cadence_hours) || 24), 1, 8760), - automationCheckbox('automation-dream', 'Enable Auto Dreaming after accumulation and idle time', dreamEnabled), - automationNumber('automation-dream-min', 'Minimum new memories', Math.max(1, Number(policy.dream_min_new) || 25), 1, 100000), - automationNumber('automation-dream-idle', 'Idle minutes before Dreaming', Math.max(0, Number(policy.dream_idle_minutes) || 0), 0, 10080), - automationCheckbox('automation-infer', 'Allow hosted relationship inference proposals', policy.infer), - node('p', 'automation-policy-note', `Cloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE} Managed compute: saving an enabled policy submits a bounded snapshot of this workspace’s normal and sensitive memory content to Engraphis Cloud. Cloud work returns proposals and never silently changes the local database.`), - ); - const actions = node('div', 'automation-policy-actions'); - const save = node('button', 'primary-button', enabled ? 'Save & send policy to Cloud' : 'Save hosted policy'); - save.type = 'submit'; - actions.append(save); - form.append(actions); - form.addEventListener('submit', saveAutomationPolicy); - target.append(form); - } - - async function bootstrapAutomation(workspace, control) { - if (!workspace || workspace !== state.workspace) return; - if (!window.confirm( - `Initialize hosted automation for ${workspace}? Engraphis will upload one bounded snapshot of that workspace's normal and sensitive memory content and save the default Cloud policy.`, - )) return; - const request = beginScopedRequest('automation-bootstrap'); - control.disabled = true; - control.textContent = 'Initializing…'; - try { - const policy = await api(`/automation/bootstrap?${query(workspace)}`, { method: 'POST' }); - if (!isCurrentScopedRequest(request) || !control.isConnected) return; - state.hostedLoaded.add(`automation:${workspace}`); - renderAutomationPolicy(policy, workspace); - showNotice('Hosted automation initialized.'); - } catch (error) { - if (!isCurrentScopedRequest(request) || !control.isConnected) return; - control.disabled = false; - control.textContent = 'Initialize hosted automation'; - showNotice(`Could not initialize hosted automation: ${error.message}`); - } - } - - async function saveAutomationPolicy(event) { - event.preventDefault(); - const form = event.currentTarget; - const workspace = form.dataset.workspace || ''; - if (!workspace || workspace !== state.workspace) { - showNotice('This policy belongs to a different workspace. Reloading the active workspace policy.'); - state.hostedLoaded.delete(`automation:${state.workspace}`); - await loadHosted('automation'); - return; - } - const request = beginScopedRequest('automation-save'); - const policy = { - enabled: byId('automation-enabled').checked, - cadence_hours: Math.max(1, Number(byId('automation-cadence').value) || 1), - dream_enabled: byId('automation-dream').checked, - dream_min_new: Math.max(1, Number(byId('automation-dream-min').value) || 1), - dream_idle_minutes: Math.max(0, Number(byId('automation-dream-idle').value) || 0), - infer: byId('automation-infer').checked, - }; - if (policy.enabled && !window.confirm( - `Save this hosted policy for ${workspace}? Engraphis will submit a bounded snapshot of that workspace’s normal and sensitive memory content to Cloud for managed compute.\n\nCloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE}`, - )) return; - const save = form.querySelector('button[type="submit"]'); - if (save) { - save.disabled = true; - save.textContent = 'Saving…'; - } - try { - const saved = await api(`/automation?${query(workspace)}`, { method: 'POST', body: policy }); - if (!isCurrentScopedRequest(request) || !form.isConnected) return; - state.hostedLoaded.add(`automation:${workspace}`); - renderAutomationPolicy({ ...saved, last_run: form.dataset.lastRun }, workspace); - showNotice('Hosted maintenance policy saved to Engraphis Cloud.'); - } catch (error) { - if (!isCurrentScopedRequest(request) || !form.isConnected) return; - if (save) { - save.disabled = false; - save.textContent = policy.enabled ? 'Save & send policy to Cloud' : 'Save hosted policy'; - } - showNotice(`Could not save the hosted policy: ${error.message}`); - } - } - - async function loadHosted(kind) { - const request = beginScopedRequest(`hosted-${kind}`); - const workspace = request.workspace; - const cacheKey = `${kind}:${workspace}`; - const target = byId(`${kind}-result`); - if (state.hostedLoaded.has(cacheKey)) return; - target.replaceChildren(empty(`Checking ${kind} availability…`)); - try { - if (kind === 'team') { - const [auth, license] = await Promise.all([api('/auth/state'), api('/license')]); - if (!isCurrentScopedRequest(request)) return; - state.license = license; - updatePlanBadge(); - renderSidebarCta(); - renderObject(target, { - local_mode: auth.mode || 'open', - hosted_team: Boolean(auth.hosted_team), - cloud_access: Boolean(license.cloud_access_active), - plan: license.plan || 'local', - }, 'Connection state'); - } else { - const result = await api(`/${kind}?${query(workspace)}`); - if (!isCurrentScopedRequest(request)) return; - if (kind === 'automation') renderAutomationPolicy(result, workspace); - else renderObject(target, result, `${kind[0].toUpperCase()}${kind.slice(1)} status`); - } - if (isCurrentScopedRequest(request)) state.hostedLoaded.add(cacheKey); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(empty(`${kind[0].toUpperCase()}${kind.slice(1)} is not active: ${error.message}`)); - } - } - function syncSummaryMessage(summary) { - if (!summary) return 'No sync has run in this dashboard process.'; - const attempted = number(summary.attempted); - const succeeded = number(summary.succeeded); - const errors = Array.isArray(summary.errors) ? summary.errors : []; - const complete = summary.complete === true - || (summary.complete !== false && errors.length === 0 && succeeded >= attempted); - const counts = `${succeeded}/${attempted} eligible workspaces completed`; - const changes = `${number(summary.added)} added · ${number(summary.updated)} updated · ${number(summary.exported)} exported`; - return `${complete ? 'Last sync complete' : 'Last sync incomplete'} · ${counts} · ${changes}${errors.length ? ` · ${errors.length} ${errors.length === 1 ? 'error' : 'errors'}` : ''}.`; - } - - function renderSyncStatus(status, message = '') { - state.syncStatus = status || {}; - const target = byId('sync-result'); - if (!target) return; - target.replaceChildren(); - if (message) target.append(empty(message, 'form-error')); - target.append( - node('p', 'automation-policy-note', syncSummaryMessage(state.syncStatus.last)), - definitionList([ - ['Connection', state.syncStatus.available ? 'Connected' : 'Not connected'], - ['Mode', state.syncStatus.read_only ? 'Read only · pull without upload' : 'Push and pull'], - ['Credential', state.syncStatus.has_cloud_session - ? 'Managed Cloud session' - : (state.syncStatus.has_user_token ? 'Local sync token' : 'None')], - ]), - node('p', 'automation-policy-note', CLOUD_SYNC_PRIVACY_NOTICE), - ); - const actions = node('div', 'automation-policy-actions'); - const run = button('Sync now', 'primary-button', runCloudSync); - run.id = 'sync-now'; - run.disabled = !state.syncStatus.available; - actions.append(run); - if (!state.syncStatus.available) { - const url = safeUrl(state.syncStatus.upgrade_url) || hostedAccountUrl('sync'); - if (url) { - const connect = node('a', 'secondary-button', 'Connect Engraphis Cloud'); - connect.href = url; - connect.target = '_blank'; - connect.rel = 'noopener'; - actions.append(connect); - } - } - target.append(actions); - } - - async function loadSync() { - const request = beginScopedRequest('sync-status'); - const target = byId('sync-result'); - if (!target) return; - target.replaceChildren(empty('Checking Cloud Sync connection…')); - try { - const status = await api('/sync/status'); - if (!isCurrentScopedRequest(request)) return; - renderSyncStatus(status); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - target.replaceChildren(empty(`Could not load Cloud Sync status: ${error.message}`, 'form-error')); - } - } - - async function runCloudSync() { - const request = beginScopedRequest('sync-run'); - const buttonNode = byId('sync-now'); - if (buttonNode) { - buttonNode.disabled = true; - buttonNode.textContent = 'Syncing…'; - } - try { - const result = await api('/sync/run', { method: 'POST' }); - if (!isCurrentScopedRequest(request)) return; - const summary = result && result.summary ? result.summary : {}; - const responseOk = Boolean(result) && result.ok !== false; - const displayedSummary = responseOk ? summary : { ...summary, complete: false }; - renderSyncStatus({ ...(state.syncStatus || {}), last: displayedSummary }); - const errors = Array.isArray(summary.errors) ? summary.errors : []; - const complete = responseOk && (summary.complete === true - || (summary.complete !== false && errors.length === 0 - && number(summary.succeeded) >= number(summary.attempted))); - showNotice(complete - ? 'Cloud Sync completed for every eligible workspace.' - : 'Cloud Sync is incomplete. Review the status before retrying.'); - } catch (error) { - if (!isCurrentScopedRequest(request)) return; - renderSyncStatus(state.syncStatus || {}, `Cloud Sync failed: ${error.message}`); - showNotice(`Cloud Sync failed: ${error.message}`); - } - } - - function planPrices() { - const annual = byId('billing-select').value === 'annual'; - return annual - ? { free: '$0', pro: '$100 / owner / year', team: '$200 / seat / year' } - : { free: '$0', pro: '$10 / owner / month', team: '$20 / seat / month' }; - } - - function renderPlans() { - const target = byId('plan-cards'); - target.replaceChildren(); - const prices = planPrices(); - const plans = [ - { id: 'free', name: 'Free', price: prices.free, note: 'The complete local memory engine and every core operation.', action: 'Current local plan' }, - { id: 'pro', name: 'Pro', price: prices.pro, note: 'Cloud sync, managed automation and portfolio analytics.' }, - { id: 'team', name: 'Team', price: prices.team, note: 'Shared workspaces, member roles, seats and remote agents.' }, - ]; - plans.forEach(plan => { - const card = node('article', `plan-card${plan.id === 'pro' ? ' featured' : ''}`); - card.append( - node('p', 'eyebrow', plan.id === (state.license && state.license.plan) ? 'Current plan' : plan.id), - node('h2', '', plan.name), - node('div', 'price', plan.price), - node('p', '', plan.note), - ); - if (plan.id === 'pro') { - card.append( - node('p', 'plan-support', 'Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development.'), - node('p', 'plan-benefits', 'Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming across your installations.'), - ); - } - if (plan.id === 'free') { - const status = node('span', 'secondary-button', plan.action); - card.append(status); - } else { - const interval = byId('billing-select').value === 'annual' ? 'annual' : 'monthly'; - const cta = hostedCta(plan.id, 'plans', interval); - const action = node('a', 'primary-button', cta.label); - const url = cta.href; - action.dataset.proCta = plan.id; - action.href = url || '#'; - if (url) { - action.target = '_blank'; - action.rel = 'noopener'; - } else { - action.addEventListener('click', event => { - event.preventDefault(); - showNotice('Connect this installation to Engraphis Cloud to open hosted plan options.'); - }); - } - card.append(action); - } - target.append(card); - }); - } - - async function loadPlans() { - const request = beginScopedRequest('plans'); - try { - const license = await api(`/license?${query(request.workspace)}`); - if (!isCurrentScopedRequest(request)) return; - state.license = license; - } catch (_) { - if (!isCurrentScopedRequest(request)) return; - state.license = { plan: 'free' }; - } - updatePlanBadge(); - renderSidebarCta(); - renderPlans(); - } - - function llmSnippet(provider, model, keySet) { - return [ - `ENGRAPHIS_LLM_PROVIDER=${provider}`, - `ENGRAPHIS_LLM_MODEL=${model}`, - 'ENGRAPHIS_LLM_API_KEY=', - keySet ? 'ENGRAPHIS_EXTRACTOR=llm_structured' : '# set ENGRAPHIS_EXTRACTOR=llm_structured to use it', - 'ENGRAPHIS_LLM_AUTO_EXTRACT=1', - ].join('\n'); - } - - function setLlmTestResult(message, tone = '') { - const target = byId('llm-test-result'); - if (!target) return; - target.textContent = message; - target.dataset.tone = tone; - } - - function updateLlmSnippet(status) { - const provider = byId('llm-provider').value; - const model = byId('llm-model').value; - byId('llm-env-snippet').value = llmSnippet(provider, model, Boolean(status.key_set)); - } - - function renderLlmSettings(status) { - const target = byId('llm-connection'); - target.replaceChildren(); - const defaults = status.default_models || {}; - const provider = status.provider || 'openai'; - const model = status.model || defaults[provider] || ''; - const providers = [...new Set([...Object.keys(defaults), provider])]; - const models = [...new Set([model, ...Object.values(defaults)].filter(Boolean))]; - const configured = Boolean(status.configured); - const extractionEnabled = Boolean(status.extractor_enabled); - const stateLabel = status.working ? 'verified' : (configured ? 'configured' : 'not configured'); - - const overview = node('div', 'llm-status-line'); - overview.append( - node('span', '', 'Provider · Model'), - node('span', `llm-status-badge ${configured ? 'ready' : 'muted'}`, stateLabel), - ); - - const pickerGrid = node('div', 'llm-picker-grid'); - const providerLabel = node('label', '', 'Provider'); - const providerSelect = node('select'); - providerSelect.id = 'llm-provider'; - providers.forEach(value => providerSelect.append(option(value, value, value === provider))); - providerLabel.htmlFor = providerSelect.id; - providerLabel.append(providerSelect); - const modelLabel = node('label', '', 'Model'); - const modelSelect = node('select'); - modelSelect.id = 'llm-model'; - models.forEach(value => modelSelect.append(option(value, value, value === model))); - modelLabel.htmlFor = modelSelect.id; - modelLabel.append(modelSelect); - pickerGrid.append(providerLabel, modelLabel); - - const keyState = node('p', 'llm-key-state', status.key_set ? 'API key set' : 'No API key set'); - keyState.append(node('span', '', ` · extractor: ${status.extractor || 'none'}`)); - const setupNote = node('p', 'llm-setup-note', 'Choose a provider and model for the copyable .env snippet. Update it locally, then restart Engraphis to apply the change.'); - const snippetLabel = node('label', 'llm-snippet-label', 'Local .env setup'); - const snippet = node('textarea', 'llm-env-snippet'); - snippet.id = 'llm-env-snippet'; - snippet.readOnly = true; - snippet.rows = 5; - snippet.value = llmSnippet(provider, model, Boolean(status.key_set)); - snippetLabel.htmlFor = snippet.id; - snippetLabel.append(snippet); - const copy = button('Copy', 'secondary-button', copyLlmSnippet); - copy.classList.add('llm-copy-button'); - const snippetWrap = node('div', 'llm-snippet-wrap'); - snippetWrap.append(snippetLabel, copy); - - const extraction = node('div', 'llm-status-line'); - extraction.append( - node('span', '', 'LLM extraction'), - node('span', `llm-status-badge ${extractionEnabled ? 'ready' : 'muted'}`, extractionEnabled ? 'ON' : 'OFF'), - ); - const extractionNote = node('p', 'llm-extraction-note', 'While ON, ingested memory content is sent to your configured provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.'); - const retentionUsesLlm = text(status.retention_supervisor).toLowerCase() === 'llm'; - const retentionNote = node( - 'p', - 'llm-extraction-note', - retentionUsesLlm - ? 'Retention supervision is ON. New memories may send their title and a bounded excerpt to the configured provider.' - : 'Retention supervision is OFF.', - ); - const extractionActions = node('div', 'llm-actions'); - const turnOn = button('Turn on', 'primary-button', () => setLlmExtractor(true)); - turnOn.disabled = extractionEnabled || !configured; - const turnOff = button('Turn off', 'secondary-button', () => setLlmExtractor(false)); - turnOff.disabled = !extractionEnabled; - extractionActions.append(turnOn, turnOff); - - const testActions = node('div', 'llm-actions'); - testActions.append(button('Test connection', 'secondary-button', testLlm)); - const testResult = node('p', 'llm-test-result'); - testResult.id = 'llm-test-result'; - testResult.setAttribute('role', 'status'); - testResult.setAttribute('aria-live', 'polite'); - testActions.append(testResult); - - providerSelect.addEventListener('change', () => { - const defaultModel = defaults[providerSelect.value]; - if (defaultModel && models.includes(defaultModel)) modelSelect.value = defaultModel; - updateLlmSnippet(status); - }); - modelSelect.addEventListener('change', () => updateLlmSnippet(status)); - target.append(overview, pickerGrid, keyState, setupNote, snippetWrap, extraction, extractionNote, retentionNote, extractionActions, testActions); - } - - async function copyLlmSnippet() { - const snippet = byId('llm-env-snippet'); - try { - await navigator.clipboard.writeText(snippet.value); - showNotice('Copied the local .env setup snippet.'); - } catch (_) { - snippet.focus(); - snippet.select(); - if (document.execCommand('copy')) showNotice('Copied the local .env setup snippet.'); - else showNotice('Select the snippet and copy it manually.'); - } - } - - async function loadSettings() { - try { - state.license = await api('/license'); - updatePlanBadge(); - renderSidebarCta(); - } catch (_) {} - renderCloudAccountSettings(); - try { - renderLlmSettings(await api('/llm/status')); - } catch (error) { - byId('llm-connection').replaceChildren(empty(`Model status unavailable: ${error.message}`)); - } - } - - async function setLlmExtractor(enabled) { - if (enabled && !window.confirm(`Turn on LLM extraction? ${EXTERNAL_LLM_PRIVACY_NOTICE}`)) return; - setLlmTestResult(enabled ? 'Verifying the configured provider…' : 'Turning extraction off…'); - try { - const result = await api('/llm/extractor', { method: 'POST', body: { enabled } }); - await loadSettings(); - const state = result.extractor_enabled ? 'LLM extraction is on for new ingested memories.' : 'LLM extraction is off for new ingested memories.'; - setLlmTestResult(`${state}${result.persisted === false ? ' The restart setting could not be saved.' : ''}`, result.extractor_enabled ? 'ready' : 'muted'); - } catch (error) { - setLlmTestResult(`Could not change extraction: ${error.message}`, 'error'); - } - } - - async function testLlm() { - setLlmTestResult('Testing the configured model…'); - try { - const result = await api('/llm/test', { method: 'POST' }); - await loadSettings(); - if (result.ok) { - const suffix = result.auto_enabled ? ' Extraction is active for new ingested memories.' : ''; - setLlmTestResult(`Connected — ${result.provider}/${result.model}.${suffix}`, 'ready'); - } else { - setLlmTestResult(`Could not connect: ${result.error || 'Check the provider, model, API key, and network.'}`, 'error'); - } - } catch (error) { - setLlmTestResult(`Model connection failed: ${error.message}`, 'error'); - } - } - - function switchView(view) { - state.view = view; - all('[data-view-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.viewPanel === view)); - all('[data-view]').forEach(control => { - const active = control.dataset.view === view; - control.classList.toggle('active', active); - if (active) control.setAttribute('aria-current', 'page'); - else control.removeAttribute('aria-current'); - }); - try { - localStorage.setItem('engraphis-ledger-view', view); - } catch (_) {} - if (view === 'relations') loadGraph(); - if (view === 'provenance' && state.provenanceTab === 'audit') loadAudit(); - if (view === 'manage') loadManageTab(state.manageTab); - window.scrollTo({ top: 0, behavior: 'instant' }); - } - - function applyTheme(theme) { - const valid = ['slate', 'midnight', 'paper', 'matrix']; - const selected = valid.includes(theme) ? theme : 'slate'; - document.body.dataset.theme = selected; - byId('theme-select').value = selected; - byId('sidebar-theme-select').value = selected; - try { - localStorage.setItem('engraphis-ledger-theme', selected); - localStorage.setItem('engraphis-theme', ({ slate: 'dark', paper: 'light', midnight: 'midnight', matrix: 'matrix' })[selected]); - } catch (_) {} - if (state.graphEngine) state.graphEngine.setThemeColors(graphThemeColors()); - } - - async function refreshBootstrap(preferred = '') { - const bootstrap = await api('/bootstrap'); - renderUpdateBanner(bootstrap.update); - state.workspaces = bootstrap.workspaces || []; - state.license = bootstrap.license || state.license; - updatePlanBadge(); - renderSidebarCta(); - const select = byId('workspace-select'); - select.replaceChildren(); - state.workspaces.forEach(item => { - const name = workspaceName(item); - select.append(option(name, name)); - }); - if (!state.workspaces.length) { - select.append(option('', 'No workspace')); - select.disabled = true; - setConnection('Local engine connected · no workspace'); - state.workspace = ''; - renderWorkspaceNames(); - renderWorkspaceList(); - return; - } - select.disabled = false; - let saved = preferred; - try { - saved = preferred || localStorage.getItem('engraphis-workspace') || ''; - } catch (_) {} - const names = state.workspaces.map(workspaceName); - const selected = names.includes(saved) - ? saved - : workspaceName([...state.workspaces].sort((a, b) => number(b.memories) - number(a.memories))[0]); - await selectWorkspace(selected); - setConnection('Local engine connected'); - } - - async function boot() { - byId('today-date').textContent = new Intl.DateTimeFormat(undefined, { dateStyle: 'long' }).format(new Date()); - let theme = 'slate'; - try { - theme = localStorage.getItem('engraphis-ledger-theme') || theme; - } catch (_) {} - applyTheme(theme); - try { - await refreshBootstrap(); - let view = 'today'; - try { - const saved = localStorage.getItem('engraphis-ledger-view'); - if (['today', 'ask', 'library', 'relations', 'provenance', 'manage'].includes(saved)) view = saved; - } catch (_) {} - switchView(view); - } catch (error) { - if (error.status === 401 && await authenticateBrowser()) { - location.reload(); - return; - } - setConnection('Local engine unavailable', false); - showNotice(`Ledger could not connect: ${error.message}`); - } - } - - all('[data-view]').forEach(control => control.addEventListener('click', () => switchView(control.dataset.view))); - all('[data-go]').forEach(control => control.addEventListener('click', () => switchView(control.dataset.go))); - all('[data-manage]').forEach(control => control.addEventListener('click', () => { - switchView('manage'); - switchManageTab(control.dataset.manage); - })); - all('[data-provenance]').forEach(control => control.addEventListener('click', () => { - switchView('provenance'); - switchProvenanceTab(control.dataset.provenance); - })); - all('[data-provenance-tab]').forEach(control => control.addEventListener('click', () => switchProvenanceTab(control.dataset.provenanceTab))); - all('[data-manage-tab]').forEach(control => control.addEventListener('click', () => switchManageTab(control.dataset.manageTab))); - - byId('workspace-select').addEventListener('change', event => selectWorkspace(event.target.value)); - byId('ask-form').addEventListener('submit', askMemory); - byId('library-filter').addEventListener('input', renderLibrary); - byId('library-type').addEventListener('change', renderLibrary); - byId('new-memory-button').addEventListener('click', () => openEditor()); - byId('editor-close').addEventListener('click', closeEditor); - byId('editor-cancel').addEventListener('click', closeEditor); - byId('memory-editor').addEventListener('submit', saveMemory); - byId('import-button').addEventListener('click', () => byId('import-files').click()); - byId('import-files').addEventListener('change', event => importFiles(event.target.files)); - - all('[data-graph-tab]').forEach(control => control.addEventListener('click', () => setGraphTab(control.dataset.graphTab))); - byId('graph-fit').addEventListener('click', () => state.graphEngine && state.graphEngine.fit()); - byId('graph-reheat').addEventListener('click', () => state.graphEngine && state.graphEngine.reheat()); - byId('graph-clear-focus').addEventListener('click', () => { - if (state.graphEngine) state.graphEngine.clearFocus(); - }); - byId('graph-freeze').addEventListener('click', () => { - state.graphFrozen = !state.graphFrozen; - setGraphSwitch('graph-freeze', state.graphFrozen); - if (state.graphEngine) state.graphEngine.freeze(state.graphFrozen); - saveGraphPreferences(); - }); - byId('graph-flow').addEventListener('click', event => { - const on = event.currentTarget.getAttribute('aria-checked') !== 'true'; - setGraphSwitch('graph-flow', on); - if (state.graphEngine) state.graphEngine.setSettings({ flow: on }); - clearGraphSavedView(); - saveGraphPreferences(); - }); - byId('graph-labels').addEventListener('click', event => { - const on = event.currentTarget.getAttribute('aria-checked') !== 'true'; - setGraphSwitch('graph-labels', on); - if (state.graphEngine) state.graphEngine.setSettings({ labels: on }); - clearGraphSavedView(); - saveGraphPreferences(); - }); - byId('graph-flow-speed').addEventListener('input', event => { - const speed = graphValueInRange('graph-flow-speed', event.target.value, 45); - byId('graph-flow-speed').value = String(speed); - byId('graph-flow-speed-output').value = String(Math.round(speed)); - byId('graph-flow-speed-output').textContent = String(Math.round(speed)); - if (state.graphEngine) state.graphEngine.setSettings({ flowSpeed: speed }); - clearGraphSavedView(); - saveGraphPreferences(); - }); - byId('graph-search').addEventListener('input', event => searchGraph(event.target.value)); - byId('graph-repo-filter').addEventListener('input', event => { - if (state.graphEngine) state.graphEngine.setRepoFilter(event.target.value); - clearGraphSavedView(); - saveGraphPreferences(); - }); - all('[data-graph-preset-choice]').forEach(control => control.addEventListener('click', () => { - const preset = control.dataset.graphPresetChoice; - const resumeLayout = state.graphFrozen; - byId('graph-preset').value = preset; - if (state.graphEngine && resumeLayout) { - // Freeze is the safe default for arranging nodes by hand. Selecting a named layout is an - // explicit request to run physics, so make that transition visible and leave the switch - // truthful; the person can freeze the settled arrangement again when they are happy. - state.graphFrozen = false; - setGraphSwitch('graph-freeze', false); - state.graphEngine.freeze(false); - } - let settings = graphPresetTuning(preset); - if (state.graphEngine) settings = state.graphEngine.setPreset(preset); - syncGraphTuning(settings); - updateGraphModeControls(); - clearGraphSavedView(); - syncGraphChoices(); - saveGraphPreferences(); - if (resumeLayout) showNotice('Layout applied. Simulation resumed — freeze it to lock node positions.'); - })); - all('[data-graph-style-choice]').forEach(control => control.addEventListener('click', () => { - byId('graph-style').value = control.dataset.graphStyleChoice; - if (state.graphEngine) state.graphEngine.setStyle(control.dataset.graphStyleChoice); - clearGraphSavedView(); - syncGraphChoices(); - saveGraphPreferences(); - })); - all('[data-graph-color-choice]').forEach(control => control.addEventListener('click', () => { - byId('graph-color').value = control.dataset.graphColorChoice; - if (state.graphEngine) state.graphEngine.setColorBy(control.dataset.graphColorChoice); - clearGraphSavedView(); - syncGraphChoices(); - saveGraphPreferences(); - })); - all('[data-graph-palette-choice]').forEach(control => control.addEventListener('click', () => { - const palette = control.dataset.graphPaletteChoice; - byId('graph-palette').value = palette; - applyGraphPalette(palette); - clearGraphSavedView(); - syncGraphChoices(); - saveGraphPreferences(); - showNotice(`${control.textContent.trim()} palette applied to the graph.`); - })); - byId('graph-min-degree').addEventListener('input', event => { - setGraphMinDegree(event.target.value); - clearGraphSavedView(); - saveGraphPreferences(); - }); - byId('graph-show-unlinked').addEventListener('click', event => { - setGraphShowUnlinked(event.currentTarget.getAttribute('aria-pressed') !== 'true'); - clearGraphSavedView(); - saveGraphPreferences(); - loadGraph({ force: true }); - }); - byId('graph-tune-min-degree').addEventListener('input', event => { - setGraphMinDegree(event.target.value); - clearGraphSavedView(); - saveGraphPreferences(); - }); - byId('graph-depth').addEventListener('input', event => { - setGraphDepth(event.target.value); - clearGraphSavedView(); - saveGraphPreferences(); - }); - GRAPH_TUNING.forEach(item => byId(item.id).addEventListener('input', event => { - const value = setGraphTuningControl(item, event.target.value); - if (state.graphEngine) state.graphEngine.setSettings({ [item.key]: value }); - clearGraphSavedView(); - saveGraphPreferences(); - })); - all('[data-graph-layer]').forEach(control => control.addEventListener('click', () => { - const layers = graphLayerState(); - const layer = control.dataset.graphLayer; - layers[layer] = !layers[layer]; - const previousIncludeCode = state.graphIncludeCode; - state.graphIncludeCode = layers.code === true; - setGraphLayers(layers); - if (state.graphEngine) state.graphEngine.setLayers(layers); - clearGraphSavedView(); - saveGraphPreferences(); - if (previousIncludeCode !== state.graphIncludeCode) loadGraph({ force: true }); - })); - all('[data-graph-saved-view]').forEach(control => control.addEventListener('click', () => applyGraphView(control.dataset.graphSavedView))); - byId('graph-save-view').addEventListener('click', saveCurrentGraphView); - byId('graph-reset-tuning').addEventListener('click', resetGraphTuning); - byId('graph-retry').addEventListener('click', () => loadGraph({ force: true })); - byId('graph-bridges').addEventListener('change', event => { - if (state.graphEngine) state.graphEngine.setBridges(event.target.checked); - saveGraphPreferences(); - }); - byId('graph-collapse').addEventListener('change', event => { - if (state.graphEngine) state.graphEngine.setCollapse(event.target.checked ? 'auto' : false); - saveGraphPreferences(); - }); - byId('graph-as-of').addEventListener('change', event => { - if (state.graphEngine) state.graphEngine.setAsOf(graphAsOfTimestamp()); - saveGraphPreferences(); - loadGraph({ force: true }); - }); - byId('graph-ghosts').addEventListener('change', event => { - if (state.graphEngine) state.graphEngine.setGhosts(event.target.checked); - saveGraphPreferences(); - }); - byId('graph-size').addEventListener('change', event => { - if (state.graphEngine) state.graphEngine.setSizeBy(event.target.value); - saveGraphPreferences(); - }); - byId('graph-export').addEventListener('click', () => { - const menu = byId('graph-export-menu'); - const open = menu.hidden; - menu.hidden = !open; - byId('graph-export').setAttribute('aria-expanded', String(open)); - }); - byId('graph-export-png').addEventListener('click', () => { - byId('graph-export-menu').hidden = true; - byId('graph-export').setAttribute('aria-expanded', 'false'); - exportGraphPng(); - }); - byId('graph-export-json').addEventListener('click', () => { - byId('graph-export-menu').hidden = true; - byId('graph-export').setAttribute('aria-expanded', 'false'); - exportGraphJson(); - }); - byId('graph-connections-close').addEventListener('click', closeGraphConnections); - byId('graph-connections-dialog').addEventListener('click', event => { - if (event.target === event.currentTarget) closeGraphConnections(); - }); - restoreGraphPreferences(); - syncGraphChoices(); - - byId('why-form').addEventListener('submit', whySearch); - byId('timeline-form').addEventListener('submit', event => timelineSearch(event, false)); - byId('supersession-form').addEventListener('submit', event => timelineSearch(event, true)); - byId('verify-receipts').addEventListener('click', verifyReceipts); - byId('export-receipts').addEventListener('click', exportReceipts); - - byId('create-workspace-toggle').addEventListener('click', () => { - byId('create-workspace-form').hidden = !byId('create-workspace-form').hidden; - if (!byId('create-workspace-form').hidden) byId('new-workspace-name').focus(); - }); - byId('create-workspace-form').addEventListener('submit', createWorkspace); - byId('consolidate-form').addEventListener('submit', previewConsolidation); - byId('consolidate-commit').addEventListener('click', commitConsolidation); - ['consolidate-structured'].forEach(id => { - byId(id).addEventListener('change', invalidateConsolidationReview); - }); - byId('billing-select').addEventListener('change', renderPlans); - byId('dashboard-select').addEventListener('change', event => { - location.assign(event.target.value === 'classic' ? '/classic' : '/'); - }); - byId('theme-select').addEventListener('change', event => applyTheme(event.target.value)); - byId('sidebar-theme-select').addEventListener('change', event => applyTheme(event.target.value)); - boot(); -})(); + if (eligible) { + target.append(node('p', 'field-note', `Baseline ${formatSavingsTokens(estimate.baseline_tokens)} → emitted ${formatSavingsTokens(estimate.emitted_tokens)} · confidence: ${text(estimate.confidence || 'unknown')}`)); + target.append(node('p', 'field-note', 'Packed context is packing savings; adaptive history is estimated avoided prompt context.')); + const basisTitle = node('h3', '', 'Savings basis'); + const basisRows = node('div', 'savings-breakdown'); + (estimate.by_basis || []).forEach(row => { + const item = node('div', 'savings-breakdown-row'); + item.append( + node('span', '', `${text(row.basis || 'unclassified').replaceAll('_', ' ')} · ${text(row.confidence || 'unknown')}`), + node('span', '', `${formatSavingsTokens(row.baseline_tokens)} → ${formatSavingsTokens(row.emitted_tokens)} · ${formatSavingsTokens(row.saved_tokens)} saved`), + ); + basisRows.append(item); + }); + target.append(basisTitle, basisRows); + if ((estimate.by_token_counter || []).length) { + target.append(node('h3', '', 'Token counters')); + const counterRows = node('div', 'savings-breakdown'); + (estimate.by_token_counter || []).forEach(row => { + const item = node('div', 'savings-breakdown-row'); + item.append( + node('span', '', text(row.token_counter || 'unknown')), + node('span', '', `${formatSavingsTokens(row.saved_tokens)} saved · ${row.receipt_count || 0} eligible deliver${number(row.receipt_count) === 1 ? 'y' : 'ies'}`), + ); + counterRows.append(item); + }); + target.append(counterRows); + } + } + target.append(node('p', 'savings-note', `${excluded} excluded or unclassified deliver${excluded === 1 ? 'y' : 'ies'}. Measures estimated prompt-context reduction; it does not measure provider billing.`)); + } + + function renderDecisions(memories) { + const target = byId('decision-list'); + target.replaceChildren(); + const candidates = memories.slice(0, 3); + if (!candidates.length) { + target.append(empty('No high-signal memories need review.')); + return; + } + candidates.forEach(memory => { + const card = node(memory.id ? 'button' : 'article', 'decision-card memory-link-card'); + if (memory.id) { + card.type = 'button'; + card.dataset.memoryId = memory.id; + card.addEventListener('click', () => openMemory(memory)); + } + const header = node('div', 'decision-card-header'); + header.append( + node('span', 'tag', memory.pinned ? 'Pinned' : memoryType(memory)), + node('h3', '', memory.title || memory.id || 'Untitled memory'), + ); + card.append(header, node('p', '', truncate(memory.content || memory.summary, 360))); + target.append(card); + }); + } + + function auditItems(payload) { + if (Array.isArray(payload)) return payload; + return payload.audit || payload.entries || payload.records || payload.events || []; + } + + function receiptItems(payload) { + if (Array.isArray(payload)) return payload; + return payload.receipts || payload.entries || payload.records || []; + } + + function provenanceTimestampMs(item) { + // Audit rows use seconds (`ts`), while receipts use milliseconds (`ts_ms`). + // Normalize before merging so both the newest-first order and 120-row cap are + // chronological across the two independently paginated feeds. + const raw = item && (item.ts_ms ?? item.ts ?? item.timestamp ?? item.created_at); + const numeric = Number(raw); + if (Number.isFinite(numeric)) return numeric < 1e12 ? numeric * 1000 : numeric; + const parsed = Date.parse(raw); + return Number.isFinite(parsed) ? parsed : 0; + } + + function auditField(item, ...names) { + for (const name of names) { + if (item && item[name] != null && item[name] !== '') return item[name]; + } + return ''; + } + + function renderActivity(items) { + const target = byId('activity-body'); + target.replaceChildren(); + if (!items.length) { + const row = node('tr'); + const cell = node('td', '', 'No audit entries yet.'); + cell.colSpan = 5; + row.append(cell); + target.append(row); + return; + } + items.slice(0, 8).forEach(item => { + const row = node('tr'); + const timestamp = auditField(item, 'ts', 'timestamp', 'created_at', 'valid_from'); + const values = [ + relative(timestamp), + auditField(item, 'actor', 'source') || 'local operator', + auditField(item, 'action', 'operation', 'event') || 'recorded', + auditField(item, 'scope', 'workspace', 'target') || state.workspace, + truncate(auditField(item, 'hash', 'id', 'receipt_id'), 14) || '—', + ]; + values.forEach(value => row.append(node('td', '', value))); + target.append(row); + }); + } + + function renderProactive(memories, unavailableMessage = '') { + const target = byId('proactive-list'); + target.replaceChildren(); + if (!memories.length) { + target.append(empty(unavailableMessage || 'No proactive context is available.')); + return; + } + memories.slice(0, 5).forEach(memory => { + const row = node('button', 'compact-row'); + row.type = 'button'; + if (memory.id) row.dataset.memoryId = memory.id; + row.append( + node('strong', '', memory.title || memory.id || 'Memory'), + node('span', '', truncate(memory.summary || memory.content, 140)), + ); + row.addEventListener('click', () => openMemory(memory)); + target.append(row); + }); + } + + async function loadStats(workspace, epoch) { + const stats = await api(`/stats?${query(workspace)}`); + if (epoch !== state.refreshEpoch) return; + state.stats = stats; + renderMetricValues(stats); + renderTypeBars(stats); + } + + async function loadSavings(workspace, epoch) { + try { + const payload = await api(`/context-savings?${savingsQuery(workspace)}`); + if (epoch !== state.refreshEpoch) return; + renderSavingsOverview(payload); + } catch (error) { + if (epoch !== state.refreshEpoch) return; + const message = `Could not load savings: ${error.message}`; + const target = byId('context-savings-summary-body'); + if (target) target.replaceChildren(empty(message)); + const persistentValue = byId('context-savings-persistent-value'); + const persistentMeta = byId('context-savings-persistent-meta'); + const persistentRate = byId('context-savings-persistent-rate'); + if (persistentValue) persistentValue.textContent = 'Unavailable'; + if (persistentMeta) persistentMeta.textContent = 'Receipt-backed estimate could not be loaded.'; + if (persistentRate) persistentRate.textContent = '—'; + } + } + + async function loadMemories(workspace, epoch) { + const payload = await api(`/memories?${query(workspace)}&limit=500`); + if (epoch !== state.refreshEpoch) return; + state.memories = payload.memories || []; + renderLibrary(); + } + + async function loadToday(workspace, epoch) { + const [proactiveResult, auditResult] = await Promise.allSettled([ + api(`/proactive?${query(workspace)}&k=8`), + api(`/audit?${query(workspace)}&limit=12`), + ]); + if (epoch !== state.refreshEpoch) return; + const proactive = proactiveResult.status === 'fulfilled' + ? (proactiveResult.value.memories || proactiveResult.value.results || []) + : []; + renderProactive(proactive, proactiveResult.status === 'rejected' + ? 'Strongest memories are unavailable. Try refreshing this workspace.' : ''); + renderDecisions(proactive); + renderActivity(auditResult.status === 'fulfilled' ? auditItems(auditResult.value) : []); + if (auditResult.status === 'rejected') { + const cell = byId('activity-body').querySelector('td'); + if (cell) cell.textContent = 'Activity is unavailable. Try refreshing this workspace.'; + } + } + + function renderWorkspaceNames() { + all('[data-workspace-name]').forEach(element => { + element.textContent = state.workspace || 'this workspace'; + }); + } + + function workspaceName(item) { + return typeof item === 'string' ? item : item.name; + } + function resetScopedPanels() { + const messages = { + 'answer-panel': 'Ask a question to receive a grounded answer with citations.', + 'retrieval-list': 'Retrieved memories will appear here.', + 'why-result': 'Trace a claim to inspect live and superseded support.', + 'timeline-result': 'Search a topic to inspect its temporal history.', + 'supersession-list': 'Search a topic to compare closed and current records.', + 'audit-list': 'Open Audit to load this workspace’s records and receipts.', + 'savings-detail': 'Open Audit to load this workspace’s receipt-backed estimate.', + 'analytics-result': 'Open this tab to check availability.', + 'automation-result': 'Open this tab to check availability.', + 'team-result': 'Open this tab to check connection state.', + }; + Object.entries(messages).forEach(([id, message]) => { + const target = byId(id); + if (target) target.replaceChildren(empty(message)); + }); + } + + async function selectWorkspace(name) { + if (!name) return; + invalidateConsolidationReview(); + const epoch = ++state.refreshEpoch; + invalidateScopedRequests(); + closeGraphConnections(); + state.workspace = name; + state.graphWorkspace = ''; + state.graphData = null; + state.graphDataIncludeCode = false; + state.graphDataShowUnlinked = false; + state.selectedMemory = ''; + // Detail/editor handlers close over a memory record. Clear both before the + // workspace fetches begin so a stale form cannot write that record into the + // newly selected workspace. + state.editorMemory = null; + byId('memory-editor').hidden = true; + const memoryDetail = byId('memory-detail'); + memoryDetail.replaceChildren(); + memoryDetail.hidden = true; + resetScopedPanels(); + state.syncStatus = null; + if (state.graphEngine) { + state.graphEngine.destroy(); + state.graphEngine = null; + } + byId('workspace-select').value = name; + renderWorkspaceNames(); + try { + localStorage.setItem('engraphis-workspace', name); + } catch (_) {} + showNotice(''); + try { + const results = await Promise.allSettled([ + loadStats(name, epoch), + loadSavings(name, epoch), + loadMemories(name, epoch), + loadToday(name, epoch), + ]); + if (epoch !== state.refreshEpoch) return; + const failed = results.find(result => result.status === 'rejected'); + if (failed) showNotice(`Some workspace panels could not refresh: ${failed.reason.message}`); + renderWorkspaceList(); + if (state.view === 'relations') await loadGraph(); + if (state.view === 'provenance' && state.provenanceTab === 'audit') await loadAudit(); + if (state.view === 'manage') await loadManageTab(state.manageTab); + } catch (error) { + if (epoch === state.refreshEpoch) showNotice(`Could not refresh ${name}: ${error.message}`); + } + } + + function memoryCard(memory) { + const card = node('button', 'memory-card'); + card.type = 'button'; + card.setAttribute('role', 'option'); + card.dataset.memoryId = memory.id; + card.setAttribute('aria-selected', String(state.selectedMemory === memory.id)); + if (state.selectedMemory === memory.id) card.classList.add('selected'); + card.append( + node('h2', '', memory.title || memory.id || 'Untitled memory'), + node('p', '', truncate(memory.content || memory.summary, 240)), + memoryMeta(memory), + ); + card.addEventListener('click', () => openMemory(memory)); + return card; + } + + function filteredMemories() { + const filter = byId('library-filter').value.trim().toLowerCase(); + const type = byId('library-type').value; + return state.memories.filter(memory => { + const matchesText = !filter || `${memory.title || ''} ${memory.content || ''} ${memory.summary || ''}` + .toLowerCase().includes(filter); + return matchesText && (!type || memoryType(memory) === type); + }); + } + + function renderLibrary() { + const target = byId('library-list'); + if (!target.dataset.keyboardBound) { + target.dataset.keyboardBound = 'true'; + target.addEventListener('keydown', event => { + const cards = [...target.querySelectorAll('[role="option"]')]; + const current = event.target.closest('[role="option"]'); + if (!current || !cards.length) return; + let index = cards.indexOf(current); + if (event.key === 'Home') index = 0; + else if (event.key === 'End') index = cards.length - 1; + else if (event.key === 'ArrowDown' || event.key === 'ArrowRight') index = Math.min(cards.length - 1, index + 1); + else if (event.key === 'ArrowUp' || event.key === 'ArrowLeft') index = Math.max(0, index - 1); + else return; + event.preventDefault(); + cards.forEach((card, cardIndex) => { card.tabIndex = cardIndex === index ? 0 : -1; }); + cards[index].focus(); + }); + } + target.replaceChildren(); + const memories = filteredMemories(); + byId('library-count').textContent = `${memories.length.toLocaleString()} ${memories.length === 1 ? 'memory' : 'memories'}`; + if (!memories.length) { + target.append(empty(state.memories.length ? 'No memories match these filters.' : 'No active memories in this workspace.')); + return; + } + memories.forEach(memory => target.append(memoryCard(memory))); + const cards = [...target.querySelectorAll('[role="option"]')]; + const selectedIndex = cards.findIndex(card => card.getAttribute('aria-selected') === 'true'); + cards.forEach((card, index) => { card.tabIndex = index === (selectedIndex >= 0 ? selectedIndex : 0) ? 0 : -1; }); + } + + function definitionList(entries) { + const list = node('dl', 'definition-list'); + entries.forEach(([term, value]) => { + const row = node('div'); + row.append(node('dt', '', term), node('dd', '', value || '—')); + list.append(row); + }); + return list; + } + + async function selectMemory(id) { + state.selectedMemory = id; + renderLibrary(); + const target = byId('memory-detail'); + target.hidden = false; + byId('memory-editor').hidden = true; + target.replaceChildren(empty('Loading memory…')); + try { + const payload = await api(`/memory/${encodeURIComponent(id)}?${query()}`); + const memory = payload.memory || state.memories.find(item => item.id === id); + if (!memory || state.selectedMemory !== id) return; + state.editorMemory = memory; + target.replaceChildren(); + target.append( + node('p', 'eyebrow', `${memoryType(memory)} · ${memory.scope || 'workspace'}`), + node('h2', '', memory.title || memory.id || 'Untitled memory'), + node('p', '', memory.content || memory.summary || 'No content.'), + memoryMeta(memory), + definitionList([ + ['Memory id', memory.id], + ['Importance', memory.importance == null ? '—' : number(memory.importance).toFixed(2)], + ['Valid from', relative(memory.valid_from)], + ['Valid to', memory.valid_to ? relative(memory.valid_to) : 'current'], + ['Source', memory.provenance && (memory.provenance.source || memory.provenance.kind)], + ['Review', memory.provenance && (memory.provenance.review_state || 'pending')], + ]), + ); + const actions = node('div', 'detail-actions'); + const provenance = memory.provenance || {}; + if (provenance.review_state !== 'approved' || provenance.trusted !== true) { + actions.append(button('Approve for prompt…', 'primary-button', () => approveForPrompt(memory))); + } + actions.append( + button('Edit', 'secondary-button', () => openEditor(memory)), + button(memory.pinned ? 'Unpin' : 'Pin', 'secondary-button', () => togglePin(memory)), + button('View timeline', 'secondary-button', () => openMemoryTimeline(memory)), + button('Retire', 'danger-button', () => retireMemory(memory)), + button('Secure erase leak', 'danger-button', () => secureEraseMemory(memory)), + ); + target.append(actions); + const chain = payload.chain || []; + if (chain.length) { + target.append(node('h3', '', 'Supersession chain')); + const list = node('div', 'timeline-list'); + chain.forEach(item => list.append(simpleMemoryCard(item, 'timeline-card'))); + target.append(list); + } + } catch (error) { + if (state.selectedMemory === id) target.replaceChildren(empty(`Could not inspect memory: ${error.message}`)); + } + } + + function openMemory(memory) { + if (!memory || !memory.id) { + showNotice('This result no longer identifies a memory to inspect.'); + return; + } + switchView('library'); + selectMemory(memory.id); + } + + function simpleMemoryCard(memory, className = 'memory-card') { + const interactive = Boolean(memory && memory.id); + const card = node(interactive ? 'button' : 'article', `${className}${interactive ? ' memory-link-card' : ''}`); + if (interactive) { + card.type = 'button'; + card.dataset.memoryId = memory.id; + card.addEventListener('click', () => openMemory(memory)); + } + card.append( + node('h3', '', memory.title || memory.id || 'Memory'), + node('p', '', truncate(memory.content || memory.summary, 500)), + memoryMeta(memory), + ); + return card; + } + + function openEditor(memory = null) { + state.editorMemory = memory; + state.editorReturnFocus = document.activeElement instanceof HTMLElement + ? document.activeElement : byId('new-memory-button'); + byId('memory-detail').hidden = true; + const editor = byId('memory-editor'); + editor.hidden = false; + byId('editor-title').textContent = memory ? 'Revise memory' : 'New memory'; + byId('editor-memory-title').value = memory ? (memory.title || '') : ''; + byId('editor-memory-type').value = memory ? memoryType(memory) : 'semantic'; + byId('editor-memory-content').value = memory ? (memory.content || memory.summary || '') : ''; + byId('editor-memory-content').removeAttribute('aria-invalid'); + byId('editor-error').hidden = true; + byId('editor-error').textContent = ''; + byId('editor-memory-importance').value = memory && memory.importance != null ? memory.importance : 0.5; + byId('editor-memory-title').focus(); + } + + function closeEditor() { + const returnFocus = state.editorReturnFocus; + byId('memory-editor').hidden = true; + byId('memory-detail').hidden = false; + state.editorMemory = null; + state.editorReturnFocus = null; + if (returnFocus && document.contains(returnFocus) && !returnFocus.hidden + && !returnFocus.disabled) returnFocus.focus(); + else byId('new-memory-button').focus(); + } + + async function saveMemory(event) { + event.preventDefault(); + const current = state.editorMemory; + const title = byId('editor-memory-title').value.trim(); + const memoryTypeValue = byId('editor-memory-type').value; + const content = byId('editor-memory-content').value.trim(); + const importance = number(byId('editor-memory-importance').value); + const currentImportance = current && current.importance != null + ? number(current.importance) : 0.5; + const contentField = byId('editor-memory-content'); + const editorError = byId('editor-error'); + contentField.removeAttribute('aria-invalid'); + editorError.hidden = true; + editorError.textContent = ''; + if (!content) { + contentField.setAttribute('aria-invalid', 'true'); + editorError.textContent = 'Enter memory content before saving.'; + editorError.hidden = false; + showNotice('Enter memory content before saving.'); + contentField.focus(); + return; + } + try { + if (current) { + if (content !== (current.content || current.summary || '')) { + const corrected = await api('/correct', { + method: 'POST', + body: { id: current.id, workspace: state.workspace, content, reason: 'revised in Ledger' }, + }); + // A correction intentionally creates a replacement. The core inherits the + // source importance; carry any label edits to that replacement rather than + // accidentally applying them to the historical source record. + if (title !== (current.title || '') || memoryTypeValue !== memoryType(current) + || importance !== currentImportance) { + await api('/memory/update', { + method: 'POST', + body: { + id: corrected.id, + workspace: state.workspace, + title, + memory_type: memoryTypeValue, + importance, + }, + }); + } + } else if (title !== (current.title || '') || memoryTypeValue !== memoryType(current) + || importance !== currentImportance) { + await api('/memory/update', { + method: 'POST', + body: { + id: current.id, + workspace: state.workspace, + title, + memory_type: memoryTypeValue, + importance, + }, + }); + } + showNotice('Memory revision recorded with temporal history preserved.'); + } else { + await api('/remember', { + method: 'POST', + body: { + workspace: state.workspace, + content, + title, + mtype: memoryTypeValue, + scope: 'workspace', + importance, + source: 'human:ledger', + trusted: true, + }, + }); + showNotice('Memory saved locally.'); + } + closeEditor(); + await selectWorkspace(state.workspace); + } catch (error) { + showNotice(`Could not save memory: ${error.message}`); + } + } + + async function togglePin(memory) { + try { + await api('/pin', { + method: 'POST', + body: { id: memory.id, workspace: state.workspace, pinned: !memory.pinned }, + }); + showNotice(memory.pinned ? 'Memory unpinned.' : 'Memory pinned against decay.'); + await selectWorkspace(state.workspace); + selectMemory(memory.id); + } catch (error) { + showNotice(`Could not change pin: ${error.message}`); + } + } + + async function retireMemory(memory) { + if (!window.confirm(`Retire “${memory.title || memory.id}”? The record stays in temporal history but leaves live recall.`)) return; + try { + await api('/retire', { + method: 'POST', + body: { id: memory.id, workspace: state.workspace, reason: 'retired in Ledger' }, + }); + state.selectedMemory = ''; + byId('memory-detail').replaceChildren(empty('Memory moved out of live recall. Its history is retained.')); + showNotice('Memory retired without hard deletion.'); + await selectWorkspace(state.workspace); + } catch (error) { + showNotice(`Could not retire memory: ${error.message}`); + } + } + + async function secureEraseMemory(memory) { + const name = memory.title || memory.id; + if (!window.confirm(`Securely erase “${name}”? This destroys temporal history and local index copies. Rotate the leaked credential; copied exports, snapshots, remote peers, and an already-compromised agent cannot be erased here.`)) return; + try { + const result = await api('/secure-erase', { + method: 'POST', body: { id: memory.id, workspace: state.workspace }, + }); + state.selectedMemory = ''; + byId('memory-detail').replaceChildren(empty('Memory securely erased from this local store. Review the reported backup limitations and rotate the credential.')); + showNotice(result.vector_index_cleanup === 'failed' + ? 'Memory removed locally; configured vector index needs separate remediation.' + : 'Memory securely erased from local persistence.'); + await selectWorkspace(state.workspace); + } catch (error) { + showNotice(`Could not securely erase memory: ${error.message}`); + } + } + + function openMemoryTimeline(memory) { + switchView('provenance'); + switchProvenanceTab('timeline'); + byId('timeline-input').value = memory.title || truncate(memory.content, 80); + byId('timeline-form').requestSubmit(); + } + + async function importFiles(files) { + if (!files.length) return; + const form = new FormData(); + form.append('workspace', state.workspace); + form.append('memory_type', 'semantic'); + form.append('derive_facts', 'false'); + [...files].forEach(file => form.append('files', file)); + try { + showNotice(`Importing ${files.length} ${files.length === 1 ? 'file' : 'files'} locally…`); + const result = await api('/workspaces/import-files', { method: 'POST', body: form }); + showNotice(`Import complete${result.count != null ? ` · ${result.count} memories` : ''}.`); + await selectWorkspace(state.workspace); + } catch (error) { + showNotice(`Import failed: ${error.message}`); + } finally { + byId('import-files').value = ''; + } + } + + const obsidianImport = { + preview: null, job: null, poll: null, selection: null, sources: [], + jobWorkspace: '', running: false, reviewGeneration: 0, + }; + let documentExtensions = null; + + async function obsidianApi(path, options = {}) { + const csrf = await reviewCsrfToken(); + return api(path, { + ...options, + headers: { ...(options.headers || {}), 'X-Engraphis-Review-CSRF': csrf }, + }); + } + + function obsidianSelection() { + const files = [ + ...byId('obsidian-import-files').files, + ...byId('obsidian-import-folder').files, + ]; + const sourceMode = byId('obsidian-source-mode').value; + const markdown = files.filter(file => /\.md$/i.test(file.name)); + const documents = files.filter(file => { + const suffix = (file.name.split('.').pop() || '').toLowerCase(); + // The format endpoint is an owner-only convenience hint. The server still + // enforces its registry for every byte if the hint is temporarily unavailable. + return !documentExtensions || documentExtensions.has(suffix); + }); + const uploadFiles = sourceMode === 'obsidian' ? markdown : documents; + const attachments = sourceMode === 'obsidian' + ? files.filter(file => !/\.md$/i.test(file.name)).map(file => ({ + path: file.webkitRelativePath || file.name, size: file.size, + })) : []; + const unsupported = sourceMode === 'obsidian' + ? 0 : files.length - uploadFiles.length; + const fields = { + workspace: byId('obsidian-workspace').value.trim(), + repo: byId('obsidian-repo').value.trim(), + session_id: byId('obsidian-session').value.trim(), + scope: byId('obsidian-scope').value.trim(), + memory_type: byId('obsidian-memory-type').value, + source_id: byId('obsidian-vault-id').value, + source_label: byId('obsidian-vault-label').value.trim(), + on_conflict: byId('obsidian-conflict').value, + source_mode: sourceMode, + }; + return { uploadFiles, attachments, unsupported, sourceMode, fields }; + } + + function obsidianFormData(selection, { confirmed = false, reviewToken = '' } = {}) { + const form = new FormData(); + Object.entries(selection.fields).forEach(([name, value]) => form.append(name, value)); + form.append('confirmed', confirmed ? 'true' : 'false'); + if (reviewToken) form.append('review_token', reviewToken); + form.append('attachment_manifest', JSON.stringify(selection.attachments)); + selection.uploadFiles.forEach(file => ( + form.append('files', file, file.webkitRelativePath || file.name) + )); + return form; + } + + function invalidateDocumentImportPreview(message = 'Selection changed. Preview again before importing.') { + obsidianImport.reviewGeneration += 1; + obsidianImport.preview = null; + obsidianImport.selection = null; + byId('obsidian-confirmed').checked = false; + byId('obsidian-run').disabled = true; + if (obsidianImport.running) return; + obsidianImport.job = null; + obsidianImport.jobWorkspace = ''; + byId('obsidian-cancel').hidden = true; + delete byId('obsidian-cancel').dataset.jobId; + renderObsidianReport(null); + if (message) byId('obsidian-import-progress').textContent = message; + } + + function updateDocumentImportMode() { + const obsidian = byId('obsidian-source-mode').value === 'obsidian'; + byId('obsidian-files-label').textContent = obsidian ? 'Individual Markdown notes' : 'Individual documents'; + byId('obsidian-folder-label').textContent = obsidian ? 'Obsidian vault folder' : 'Document folder'; + byId('obsidian-import-description').textContent = obsidian + ? 'Choose an Obsidian vault folder. Engraphis previews Markdown note bytes and attachment metadata before it writes anything; attachment bytes are never uploaded.' + : 'Choose individual files or a folder. Engraphis previews supported document formats before it writes anything; uploaded bytes are processed locally and are not kept as dashboard upload copies.'; + byId('obsidian-run').textContent = obsidian ? 'Import vault notes' : 'Import documents'; + byId('obsidian-import-files').value = ''; + byId('obsidian-import-folder').value = ''; + invalidateDocumentImportPreview('Choose files or a folder to preview its import.'); + } + + function updateSourceLabelRequirement() { + const label = byId('obsidian-vault-label'); + const isNewSource = !byId('obsidian-vault-id').value; + label.required = isNewSource; + label.setAttribute('aria-required', isNewSource ? 'true' : 'false'); + label.placeholder = isNewSource ? 'Required for a new source' : 'Saved source label'; + } + + function prefillNewSourceLabelFromFolder() { + if (byId('obsidian-vault-id').value || byId('obsidian-vault-label').value.trim()) return; + const firstFolderFile = [...byId('obsidian-import-folder').files] + .find(file => file.webkitRelativePath && file.webkitRelativePath.includes('/')); + if (!firstFolderFile) return; + const folderName = firstFolderFile.webkitRelativePath.split('/')[0].trim(); + if (folderName) byId('obsidian-vault-label').value = folderName; + } + + function requireNewSourceLabel() { + if (byId('obsidian-vault-id').value || byId('obsidian-vault-label').value.trim()) return true; + byId('obsidian-import-progress').textContent = 'Enter a Source label before creating a new source.'; + byId('obsidian-vault-label').focus(); + return false; + } + + function obsidianRows(result) { + const rows = result && (result.files || result.details || result.entries || []); + return Array.isArray(rows) ? rows : []; + } + + function renderObsidianReport(result) { + const target = byId('obsidian-import-report'); + const wanted = byId('obsidian-report-filter').value; + target.replaceChildren(); + const rows = obsidianRows(result).filter(row => { + const status = String(row.status || row.action || row.result || '').toLowerCase(); + if (wanted === 'all') return true; + if (wanted === 'reject') return /reject|error|warn|conflict/.test(status) || Boolean(row.warning || row.error); + return status.includes(wanted); + }); + if (!rows.length) { + target.append(empty(wanted === 'all' ? 'No per-file details were returned.' : 'No files match this filter.')); + return; + } + const list = node('ul'); + rows.forEach(row => { + const status = String(row.status || row.action || row.result || 'reported').toLowerCase(); + const action = row.action && String(row.action).toLowerCase() !== status + ? ` · action: ${row.action}` : ''; + const format = row.format || row.format_name ? ` · format: ${row.format || row.format_name}` : ''; + const warning = row.warning || row.error || row.reason + || (Number(row.warning_count) ? `${row.warning_count} warning(s)` : ''); + const item = node('li', '', `${status.toUpperCase()} · ${row.path || row.file || row.relative_path || 'unnamed document'}${format}${action}${warning ? ` · ${warning}` : ''}`); + item.dataset.status = /reject|error/.test(status) || row.error || row.reason ? 'reject' : status; + list.append(item); + }); + target.append(list); + } + + function obsidianSummary(result, prefix = 'Preview') { + const counts = result && (result.counts || result); + const keys = ['documents', 'markdown', 'formats', 'imported', 'updated', 'renamed', 'skipped', 'rejected', 'conflict', 'missing', 'error']; + const summary = keys.filter(key => Number.isFinite(Number(counts && counts[key]))) + .map(key => `${key.replace('_', ' ')}: ${counts[key]}`); + const unsupported = obsidianImport.selection && obsidianImport.selection.unsupported; + const warning = unsupported ? ` · warning: ${unsupported} unsupported files were not uploaded` : ''; + byId('obsidian-import-progress').textContent = summary.length ? `${prefix} · ${summary.join(' · ')}${warning}` : `${prefix} ready.${warning}`; + } + + async function loadObsidianVaults() { + const select = byId('obsidian-vault-id'); + try { + const result = await obsidianApi(`/workspaces/import-documents/sources?${query(state.workspace)}`); + const vaults = result.sources || result.vaults || result || []; + obsidianImport.sources = Array.isArray(vaults) ? vaults : []; + select.replaceChildren(option('', 'New source')); + obsidianImport.sources.forEach(vault => select.append(option(vault.id, vault.label || vault.name || vault.id))); + } catch (_) { + // A first-run vault list is optional; preview/import still present a useful error. + select.replaceChildren(option('', 'New source')); + obsidianImport.sources = []; + } + } + + async function loadDocumentFormats() { + try { + const result = await obsidianApi('/workspaces/import-documents/formats'); + const extensions = Array.isArray(result.extensions) ? result.extensions : []; + documentExtensions = new Set(extensions.map(extension => String(extension).replace(/^\./, '').toLowerCase())); + } catch (_) { + // Server-side validation remains authoritative; do not invent a stale client registry. + documentExtensions = null; + } + } + + function applySelectedDocumentSource() { + const source = obsidianImport.sources.find(item => item.id === byId('obsidian-vault-id').value); + if (!source) { + byId('obsidian-vault-label').value = ''; + updateSourceLabelRequirement(); + invalidateDocumentImportPreview(); + return; + } + byId('obsidian-vault-label').value = source.label || source.name || ''; + if (source.repo != null) byId('obsidian-repo').value = source.repo; + if (source.session_id != null) byId('obsidian-session').value = source.session_id; + if (source.scope) byId('obsidian-scope').value = source.scope; + if (source.memory_type) byId('obsidian-memory-type').value = source.memory_type; + byId('obsidian-source-mode').value = source.adapter === 'obsidian' || source.kind === 'obsidian' + ? 'obsidian' : 'documents'; + updateSourceLabelRequirement(); + updateDocumentImportMode(); + } + + async function previewObsidianImport() { + if (obsidianImport.running) return; + if (!requireNewSourceLabel()) return; + const selection = obsidianSelection(); + if (!selection.uploadFiles.length) { + byId('obsidian-import-progress').textContent = selection.sourceMode === 'obsidian' + ? 'Choose a folder containing Markdown notes.' + : 'Choose supported documents to import.'; + return; + } + invalidateDocumentImportPreview(''); + const generation = obsidianImport.reviewGeneration; + const type = selection.sourceMode === 'obsidian' ? 'Markdown notes' : 'supported documents'; + const ignored = selection.unsupported ? ` · ${selection.unsupported} unsupported files will not be uploaded` : ''; + byId('obsidian-import-progress').textContent = `Previewing ${selection.uploadFiles.length} ${type}${selection.attachments.length ? ` and ${selection.attachments.length} attachment manifests` : ''}${ignored}…`; + byId('obsidian-preview').disabled = true; + try { + const preview = await obsidianApi('/workspaces/import-documents/preview', { + method: 'POST', body: obsidianFormData(selection), + }); + if (generation !== obsidianImport.reviewGeneration) return; + if (!preview || typeof preview.review_token !== 'string' || !preview.review_token) { + throw new Error('The server did not bind this preview. Preview again.'); + } + selection.reviewToken = preview.review_token; + obsidianImport.selection = selection; + obsidianImport.preview = preview; + byId('obsidian-confirmed').checked = false; + renderObsidianReport(obsidianImport.preview); + obsidianSummary(obsidianImport.preview); + byId('obsidian-run').disabled = false; + } catch (error) { + if (generation !== obsidianImport.reviewGeneration) return; + obsidianImport.selection = null; + obsidianImport.preview = null; + byId('obsidian-import-progress').textContent = `Preview failed: ${error.message}`; + byId('obsidian-run').disabled = true; + } finally { + byId('obsidian-preview').disabled = false; + } + } + + async function pollObsidianImport(jobId, workspace) { + try { + const result = await obsidianApi(`/workspaces/import-documents/jobs/${encodeURIComponent(jobId)}?${query(workspace)}`); + obsidianImport.job = result; + renderObsidianReport(result); + obsidianSummary(result, 'Import'); + if (!['complete', 'completed', 'partial', 'failed', 'cancelled'].includes(String(result.state || result.status || '').toLowerCase())) { + obsidianImport.poll = window.setTimeout(() => pollObsidianImport(jobId, workspace), 750); + return; + } + obsidianImport.running = false; + obsidianImport.poll = null; + obsidianImport.selection = null; + obsidianImport.preview = null; + byId('obsidian-confirmed').checked = false; + byId('obsidian-cancel').hidden = true; + byId('obsidian-run').disabled = true; + byId('obsidian-preview').disabled = false; + showNotice('Document import finished.'); + await selectWorkspace(state.workspace); + } catch (error) { + byId('obsidian-import-progress').textContent = `Could not read import progress: ${error.message}`; + byId('obsidian-run').disabled = true; + } + } + + async function runObsidianImport(event) { + event.preventDefault(); + if (!requireNewSourceLabel()) return; + if (!byId('obsidian-confirmed').checked) { + byId('obsidian-import-progress').textContent = 'Confirm the selected scope before importing.'; + byId('obsidian-confirmed').focus(); + return; + } + const selection = obsidianImport.selection; + if (!selection || !selection.reviewToken) { + byId('obsidian-import-progress').textContent = 'Preview this exact selection before importing.'; + byId('obsidian-run').disabled = true; + return; + } + const workspace = selection.fields.workspace; + const runBody = obsidianFormData(selection, { + confirmed: true, reviewToken: selection.reviewToken, + }); + // The server token is one-time. Clear the client copy before the request so + // a double submit or ambiguous network failure cannot reuse it. + selection.reviewToken = ''; + byId('obsidian-run').disabled = true; + byId('obsidian-preview').disabled = true; + byId('obsidian-import-progress').textContent = 'Starting local document import…'; + obsidianImport.running = true; + obsidianImport.jobWorkspace = workspace; + try { + const result = await obsidianApi('/workspaces/import-documents/run', { + method: 'POST', + body: runBody, + }); + obsidianImport.job = result; + renderObsidianReport(result); + obsidianSummary(result, 'Import'); + const jobId = result.job_id || result.id; + if (jobId) { + byId('obsidian-cancel').hidden = false; + byId('obsidian-cancel').dataset.jobId = jobId; + byId('obsidian-cancel').dataset.workspace = workspace; + await pollObsidianImport(jobId, workspace); + } + else { + obsidianImport.running = false; + obsidianImport.selection = null; + obsidianImport.preview = null; + byId('obsidian-confirmed').checked = false; + byId('obsidian-run').disabled = true; + byId('obsidian-preview').disabled = false; + showNotice('Document import finished.'); + await selectWorkspace(state.workspace); + } + } catch (error) { + obsidianImport.running = false; + obsidianImport.selection = null; + obsidianImport.preview = null; + byId('obsidian-confirmed').checked = false; + byId('obsidian-import-progress').textContent = `Import failed: ${error.message} Preview again before retrying.`; + byId('obsidian-run').disabled = true; + byId('obsidian-preview').disabled = false; + } + } + + async function cancelObsidianImport() { + const button = byId('obsidian-cancel'); + const jobId = button.dataset.jobId; + const workspace = button.dataset.workspace || obsidianImport.jobWorkspace; + if (!jobId || !workspace) return; + button.disabled = true; + const form = new FormData(); + form.append('workspace', workspace); + try { + await obsidianApi(`/workspaces/import-documents/jobs/${encodeURIComponent(jobId)}/cancel`, { method: 'POST', body: form }); + byId('obsidian-import-progress').textContent = 'Cancellation requested; finishing the current document safely…'; + } catch (error) { + byId('obsidian-import-progress').textContent = `Could not cancel import: ${error.message}`; + } finally { + button.disabled = false; + } + } + + async function openObsidianImport() { + const dialog = byId('obsidian-import-dialog'); + byId('obsidian-confirmed').checked = false; + if (!obsidianImport.running) { + if (obsidianImport.poll) window.clearTimeout(obsidianImport.poll); + obsidianImport.preview = null; + obsidianImport.job = null; + obsidianImport.poll = null; + obsidianImport.selection = null; + obsidianImport.jobWorkspace = ''; + delete byId('obsidian-cancel').dataset.jobId; + delete byId('obsidian-cancel').dataset.workspace; + } + byId('obsidian-workspace').value = state.workspace; + byId('obsidian-repo').value = ''; + byId('obsidian-session').value = ''; + byId('obsidian-vault-label').value = ''; + if (!obsidianImport.running) { + byId('obsidian-import-progress').textContent = 'Choose individual files or a folder to preview its import.'; + } + byId('obsidian-run').disabled = true; + byId('obsidian-preview').disabled = obsidianImport.running; + byId('obsidian-cancel').hidden = !obsidianImport.running; + if (!obsidianImport.running) renderObsidianReport(null); + await Promise.all([loadObsidianVaults(), loadDocumentFormats()]); + byId('obsidian-vault-id').value = ''; + updateSourceLabelRequirement(); + updateDocumentImportMode(); + dialog.showModal(); + byId('obsidian-import-files').focus(); + } + + function renderAnswer(result) { + const target = byId('answer-panel'); + target.replaceChildren(); + const meta = node('div', 'answer-meta'); + const grounded = Boolean(result.grounded); + meta.append( + node('span', `support-pill ${grounded ? 'grounded' : 'abstained'}`, grounded ? 'Grounded' : 'Abstained'), + node('span', 'support-pill', `Support ${number(result.support).toFixed(2)}`), + node('span', 'support-pill', `${(result.citations || []).length} citations`), + ); + target.append(meta); + if (!grounded) { + target.append( + node('h2', '', 'Insufficient evidence'), + node('p', 'answer-copy', result.reason || 'The active workspace does not support a grounded answer.'), + ); + return; + } + target.append(node('p', 'answer-copy', result.answer || 'The cited memories support this answer.')); + const citations = node('div', 'citation-list'); + (result.citations || []).forEach(citation => { + const card = node(citation.id ? 'button' : 'article', 'citation-card memory-link-card'); + if (citation.id) { + card.type = 'button'; + card.dataset.memoryId = citation.id; + card.addEventListener('click', () => openMemory(citation)); + } + card.append( + node('h3', '', `[${citation.n || citation.number || '•'}] ${citation.title || citation.id || 'Memory'}`), + node('p', '', citation.content || citation.summary || ''), + node('div', 'memory-meta', `support ${number(citation.support || citation.score).toFixed(2)} · ${citation.id || ''}`), + ); + citations.append(card); + }); + target.append(citations); + } + + async function askMemory(event) { + event.preventDefault(); + const input = byId('ask-input'); + const question = input.value.trim(); + if (!question) { + showNotice('Enter a question before requesting a grounded answer.'); + input.focus(); + return; + } + if (!state.workspace) { + showNotice('Choose a workspace before requesting a grounded answer.'); + return; + } + const request = beginScopedRequest('ask'); + const workspace = request.workspace; + showNotice(''); + const k = number(byId('ask-k').value) || 5; + byId('answer-panel').replaceChildren(empty('Searching, checking support and building citations…')); + byId('retrieval-list').replaceChildren(empty('Retrieving candidate memories…')); + try { + const [answer, retrieval] = await Promise.all([ + api('/answer', { + method: 'POST', + body: { query: question, workspace, k: Math.max(8, k), max_citations: k }, + }), + // The dashboard /recall route is deliberately read-only (reinforce=False). + // Keep it alongside /answer for uncited raw candidates without a second + // reinforcement of the memories that answer already cited. + api(`/recall?q=${encodeURIComponent(question)}&${query(workspace)}&k=${Math.max(8, k)}`), + ]); + if (!isCurrentScopedRequest(request)) return; + renderAnswer(answer); + const target = byId('retrieval-list'); + target.replaceChildren(); + const memories = retrieval.memories || []; + if (!memories.length) target.append(empty('No raw candidates were returned.')); + else memories.forEach(memory => target.append(simpleMemoryCard(memory))); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + byId('answer-panel').replaceChildren(empty(`Grounded Ask is unavailable: ${error.message}`)); + byId('retrieval-list').replaceChildren(empty('Raw retrieval did not complete.')); + } + } + + function graphNodes(payload) { + const source = payload.nodes || payload.entities || []; + return source.map(item => ({ + id: item.id, + name: item.label || item.name || item.id, + label: item.label || item.name || item.id, + etype: item.etype || item.type || 'person_or_concept', + nodeKind: item.node_kind || item.kind || '', + degree: number(item.degree), + community: Number.isFinite(Number(item.community)) ? Number(item.community) : undefined, + repo: item.repo || '', + topic: item.topic || '', + valid_from: item.valid_from, + valid_to: item.valid_to, + })); + } + + function graphLinks(payload) { + const source = payload.edges || payload.links || []; + return source.map((item, index) => ({ + id: item.id || `edge-${index}`, + source: item.from || (item.source && (item.source.id || item.source)), + target: item.to || (item.target && (item.target.id || item.target)), + label: item.label || item.relation || 'related', + layer: item.layer || 'semantic', + valid_from: item.valid_from, + valid_to: item.valid_to, + })).filter(item => item.source && item.target); + } + + function revealGraphNode(id, label = 'Selected entity') { + const engine = state.graphEngine; + if (!engine) return; + let attempts = 0; + const reveal = () => { + if (state.graphEngine !== engine) return; + if (engine.reveal(id)) return; + attempts += 1; + if (attempts < 8) { + window.requestAnimationFrame(reveal); + return; + } + showNotice(`${label} is outside the current graph scope.`); + }; + reveal(); + } + + function cancelGraphConnectionMemoryLoad() { + state.graphConnectionsRequest += 1; + if (state.graphConnectionsController) state.graphConnectionsController.abort(); + state.graphConnectionsController = null; + } + + function closeGraphConnections() { + cancelGraphConnectionMemoryLoad(); + const dialog = byId('graph-connections-dialog'); + if (dialog.open) dialog.close(); + } + + function graphMemoryCard(evidence) { + return { + id: evidence.memory_id || evidence.id, + title: evidence.title || evidence.label || evidence.memory_id || evidence.id, + content: evidence.excerpt || evidence.content || evidence.summary || '', + mtype: evidence.memory_type || evidence.mtype, + valid_from: evidence.valid_from, + valid_to: evidence.valid_to, + ingested_at: evidence.ingested_at, + provenance: evidence.provenance, + }; + } + + function graphMemoryEvidenceCard(memory) { + const card = node('article', 'graph-memory-evidence'); + card.append( + node('h4', '', memory.title || memory.id || 'Memory'), + node('p', '', truncate(memory.content || memory.summary, 500)), + memoryMeta(memory), + ); + if (memory.id) { + card.append(button('Open in Library', 'secondary-button', () => { + closeGraphConnections(); + openMemory(memory); + })); + } + return card; + } + + function renderGraphConnectionMemories(memories, message) { + const target = byId('graph-connection-memory-list'); + target.replaceChildren(); + if (!memories.length) { + const placeholder = empty(message); + placeholder.setAttribute('role', 'listitem'); + target.append(placeholder); + return; + } + memories.forEach(memory => { + const card = graphMemoryEvidenceCard(memory); + card.setAttribute('role', 'listitem'); + target.append(card); + }); + } + + function isGraphMemoryNode(item) { + const kind = String(item.nodeKind || '').toLowerCase(); + const type = String(item.etype || '').toLowerCase(); + return kind === 'memory' || type === 'memory' || type.startsWith('memory_'); + } + + function graphConnectionEntries(item) { + const graph = state.graphEngine && state.graphEngine.exportData + ? state.graphEngine.exportData() : state.graphData; + if (!graph) return []; + const nodes = new Map(graph.nodes.map(candidate => [candidate.id, candidate])); + const connections = new Map(); + graph.links.forEach(link => { + const source = link.source; + const target = link.target; + if (source !== item.id && target !== item.id) return; + const otherId = source === item.id ? target : source; + const other = nodes.get(otherId); + if (!other || other.id === item.id) return; + const entry = connections.get(other.id) || { item: other, relations: new Set() }; + if (link.label) entry.relations.add(link.label); + connections.set(other.id, entry); + }); + return [...connections.values()].sort((left, right) => { + const degree = number(right.item.degree) - number(left.item.degree); + return degree || left.item.name.localeCompare(right.item.name); + }); + } + + async function showGraphConnectionMemories(item) { + if (!item || !item.id || !state.workspace) return; + cancelGraphConnectionMemoryLoad(); + const request = ++state.graphConnectionsRequest; + const workspace = state.workspace; + const title = item.name || item.label || item.id; + byId('graph-connection-memory-title').textContent = `Memories for ${title}`; + renderGraphConnectionMemories([], 'Loading memory evidence…'); + if (isGraphMemoryNode(item)) { + const known = state.memories.find(memory => memory.id === item.id); + if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; + renderGraphConnectionMemories( + [known || graphMemoryCard(item)], 'No memory details are available for this node.', + ); + return; + } + const controller = new AbortController(); + state.graphConnectionsController = controller; + const timeout = window.setTimeout(() => controller.abort(), GRAPH_CONNECTION_MEMORIES_TIMEOUT_MS); + try { + const detail = await api( + `/graph/entities/${encodeURIComponent(item.id)}/memories?${query(workspace)}${graphAsOfQuery()}`, + { signal: controller.signal }, + ); + if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; + const evidence = detail.evidence || []; + const total = number(detail.totals && detail.totals.evidence) || evidence.length; + byId('graph-connection-memory-title').textContent = `${total} ${total === 1 ? 'memory' : 'memories'} for ${title}`; + renderGraphConnectionMemories( + evidence.map(graphMemoryCard), + 'No active memories support this connected node.', + ); + } catch (error) { + if (request !== state.graphConnectionsRequest || workspace !== state.workspace) return; + byId('graph-connection-memory-title').textContent = `Memories for ${title}`; + renderGraphConnectionMemories([], error && error.name === 'AbortError' + ? 'Memory evidence loading timed out. Choose this node again to retry.' + : `Could not load memory evidence: ${error.message}`); + } finally { + window.clearTimeout(timeout); + if (state.graphConnectionsController === controller) state.graphConnectionsController = null; + } + } + + function graphConnectionRow(entry) { + const item = entry.item; + const row = node('article', 'graph-connection-row'); + row.setAttribute('role', 'listitem'); + const details = node('div'); + const relations = [...entry.relations]; + const relationLabel = relations.length ? ` · ${relations.join(', ')}` : ''; + details.append( + node('h3', '', item.name), + node('p', '', `${number(item.degree)} connections · ${item.etype}${relationLabel}`), + ); + const actions = node('div', 'graph-connection-actions'); + actions.append( + button('Focus graph', 'secondary-button', () => { + closeGraphConnections(); + revealGraphNode(item.id, item.name); + }), + button('Memories', 'secondary-button', () => showGraphConnectionMemories(item)), + ); + row.append(details, actions); + return row; + } + + function openGraphConnections(item) { + if (!item || !item.id) return; + cancelGraphConnectionMemoryLoad(); + const dialog = byId('graph-connections-dialog'); + const entries = graphConnectionEntries(item); + const title = item.name || item.label || item.id; + byId('graph-connections-title').textContent = `Connected to ${title}`; + byId('graph-connections-meta').textContent = `${entries.length} direct ${entries.length === 1 ? 'connection' : 'connections'} visible in this graph view`; + const target = byId('graph-connections-list'); + target.replaceChildren(); + if (!entries.length) target.append(empty('No connected nodes are visible in this graph view.')); + else entries.forEach(entry => target.append(graphConnectionRow(entry))); + byId('graph-connection-memory-title').textContent = 'Memories'; + renderGraphConnectionMemories([], 'Choose a connected node to inspect its memory evidence.'); + if (!dialog.open) dialog.showModal(); + } + + function updateGraphFacts(data) { + const stats = byId('graph-stats'); + stats.replaceChildren(); + const degrees = data.nodes.map(item => number(item.degree)).sort((a, b) => a - b); + const values = [ + ['Entities', data.nodes.length], + ['Relations', data.links.length], + ['Unlinked', data.nodes.filter(item => !number(item.degree)).length], + ['Median links', degrees.length ? degrees[Math.floor(degrees.length / 2)] : 0], + ]; + values.forEach(([label, value]) => { + const item = node('div', 'stat-item'); + item.append(node('span', '', label), node('strong', '', number(value).toLocaleString())); + stats.append(item); + }); + const top = byId('graph-top'); + top.replaceChildren(); + [...data.nodes].sort((a, b) => number(b.degree) - number(a.degree)).slice(0, 7).forEach(item => { + const control = node('button', 'compact-row'); + control.type = 'button'; + control.append(node('strong', '', item.name), node('span', '', `${number(item.degree)} connections · ${item.etype}`)); + control.addEventListener('click', () => openGraphConnections(item)); + top.append(control); + }); + } + + function updateGraphModeControls() { + const full = state.graphMode === 'full'; + ['graph-min-degree', 'graph-tune-min-degree', 'graph-collapse'].forEach(id => { + const scopeControl = byId(id); + scopeControl.disabled = full; + scopeControl.title = full + ? 'Full node graph always includes unlinked nodes and never collapses clusters.' + : ''; + }); + const preset = GRAPH_PRESET_LABELS[byId('graph-preset').value] || 'Islands'; + byId('graph-mode').textContent = `${full ? 'Full node graph' : 'Responsive overview'} · ${preset}`; + } + + function setChoicePressed(selector, dataKey, selected) { + all(selector).forEach(control => { + const active = control.dataset[dataKey] === selected; + control.classList.toggle('active', active); + control.setAttribute('aria-pressed', String(active)); + }); + } + + function syncGraphChoices() { + const preset = byId('graph-preset').value; + const style = byId('graph-style').value; + const color = byId('graph-color').value; + const palette = byId('graph-palette').value; + setChoicePressed('[data-graph-preset-choice]', 'graphPresetChoice', preset); + setChoicePressed('[data-graph-style-choice]', 'graphStyleChoice', style); + setChoicePressed('[data-graph-color-choice]', 'graphColorChoice', color); + setChoicePressed('[data-graph-palette-choice]', 'graphPaletteChoice', palette); + byId('graph-style-note').textContent = GRAPH_STYLE_NOTES[style] || GRAPH_STYLE_NOTES.classic; + syncGraphSavedViews(); + } + + function setGraphSwitch(id, on) { + const control = byId(id); + control.classList.toggle('on', on); + control.setAttribute('aria-checked', String(on)); + } + + function graphValueInRange(id, value, fallback) { + const control = byId(id); + const raw = Number(value); + const safe = Number.isFinite(raw) ? raw : fallback; + const min = Number(control.min); + const max = Number(control.max); + return Math.min(Number.isFinite(max) ? max : safe, Math.max(Number.isFinite(min) ? min : safe, safe)); + } + + function graphPresetTuning(preset) { + const available = window.EngraphisGraph && window.EngraphisGraph.PRESETS; + const source = (available && available[preset]) || GRAPH_PRESET_TUNING[preset] || GRAPH_PRESET_TUNING.communities; + return GRAPH_TUNING.reduce((settings, item) => { + settings[item.key] = source && Number.isFinite(Number(source[item.key])) + ? Number(source[item.key]) : item.fallback; + return settings; + }, {}); + } + + function setGraphTuningControl(item, value) { + const control = byId(item.id); + const next = graphValueInRange(item.id, value, item.fallback); + control.value = String(next); + const rendered = item.precision ? next.toFixed(item.precision) : String(Math.round(next)); + const output = byId(`${item.id}-output`); + output.value = rendered; + output.textContent = rendered; + return next; + } + + function graphTuningSettings() { + return GRAPH_TUNING.reduce((settings, item) => { + settings[item.key] = number(byId(item.id).value); + return settings; + }, { flowSpeed: number(byId('graph-flow-speed').value) }); + } + + function syncGraphTuning(settings) { + GRAPH_TUNING.forEach(item => setGraphTuningControl(item, settings && settings[item.key])); + const flowSpeed = graphValueInRange('graph-flow-speed', settings && settings.flowSpeed, 45); + byId('graph-flow-speed').value = String(flowSpeed); + byId('graph-flow-speed-output').value = String(Math.round(flowSpeed)); + byId('graph-flow-speed-output').textContent = String(Math.round(flowSpeed)); + } + + function graphScope() { + const full = state.graphMode === 'full'; + return { + minDegree: full ? 0 : number(byId('graph-min-degree').value), + showUnlinked: full || state.graphShowUnlinked, + depth: number(byId('graph-depth').value), + }; + } + + function applyGraphScope() { + if (state.graphEngine) state.graphEngine.setScope(graphScope()); + } + + function setGraphMinDegree(value, apply = true) { + const next = graphValueInRange('graph-min-degree', value, 1); + byId('graph-min-degree').value = String(next); + byId('graph-min-degree-output').value = String(Math.round(next)); + byId('graph-min-degree-output').textContent = String(Math.round(next)); + byId('graph-tune-min-degree').value = String(next); + byId('graph-tune-min-degree-output').value = String(Math.round(next)); + byId('graph-tune-min-degree-output').textContent = String(Math.round(next)); + if (apply) applyGraphScope(); + } + + function setGraphDepth(value, apply = true) { + const next = graphValueInRange('graph-depth', value, 2); + byId('graph-depth').value = String(next); + byId('graph-depth-output').value = String(Math.round(next)); + byId('graph-depth-output').textContent = String(Math.round(next)); + if (apply) applyGraphScope(); + } + + function setGraphShowUnlinked(on, apply = true) { + const next = on === true; + state.graphShowUnlinked = next; + const control = byId('graph-show-unlinked'); + control.textContent = next ? 'Hide unlinked nodes' : 'Show unlinked nodes'; + control.setAttribute('aria-pressed', String(next)); + control.title = next + ? 'Hide entities that have no relations in this graph view' + : 'Show entities that have no relations in this graph view'; + if (apply) applyGraphScope(); + } + + function graphLayerState() { + return all('[data-graph-layer]').reduce((layers, control) => { + layers[control.dataset.graphLayer] = control.getAttribute('aria-pressed') === 'true'; + return layers; + }, {}); + } + + function setGraphLayers(layers) { + const source = layers && typeof layers === 'object' ? layers : GRAPH_DEFAULT_LAYERS; + all('[data-graph-layer]').forEach(control => { + const active = source[control.dataset.graphLayer] !== false; + control.classList.toggle('active', active); + control.setAttribute('aria-pressed', String(active)); + }); + } + + function updateGraphLayerCounts(data, supplied) { + const counts = GRAPH_LAYERS.reduce((result, layer) => { result[layer] = 0; return result; }, {}); + if (Array.isArray(supplied)) supplied.forEach(item => { + if (item && GRAPH_LAYERS.includes(item.layer)) counts[item.layer] = number(item.count); + }); + else (data.links || []).forEach(link => { + if (GRAPH_LAYERS.includes(link.layer)) counts[link.layer] += 1; + }); + GRAPH_LAYERS.forEach(layer => { byId(`graph-layer-${layer}-count`).textContent = counts[layer].toLocaleString(); }); + } + + function syncGraphSavedViews() { + all('[data-graph-saved-view]').forEach(control => { + const active = control.dataset.graphSavedView === state.graphSavedView; + control.classList.toggle('active', active); + control.setAttribute('aria-pressed', String(active)); + }); + } + + function clearGraphSavedView() { + if (!state.graphSavedView) return; + state.graphSavedView = ''; + syncGraphSavedViews(); + } + + function graphPreference(name, fallback, allowed) { + try { + const saved = JSON.parse(localStorage.getItem(GRAPH_PREFERENCES_KEY) || '{}'); + const value = saved && typeof saved === 'object' ? saved[name] : undefined; + return allowed && !allowed.includes(value) ? fallback : value === undefined ? fallback : value; + } catch (_) { + return fallback; + } + } + + function graphPreferenceSnapshot() { + return { + preset: byId('graph-preset').value, + style: byId('graph-style').value, + color: byId('graph-color').value, + palette: byId('graph-palette').value, + flow: byId('graph-flow').getAttribute('aria-checked') === 'true', + labels: byId('graph-labels').getAttribute('aria-checked') === 'true', + tuning: graphTuningSettings(), + minDegree: number(byId('graph-min-degree').value), + depth: number(byId('graph-depth').value), + showUnlinked: state.graphShowUnlinked, + layers: graphLayerState(), + includeCode: state.graphIncludeCode, + savedView: state.graphSavedView, + bridges: byId('graph-bridges').checked, + collapse: byId('graph-collapse').checked, + asOf: byId('graph-as-of').value, + ghosts: byId('graph-ghosts').checked, + size: byId('graph-size').value, + repoFilter: byId('graph-repo-filter').value.slice(0, 200), + }; + } + + function saveGraphPreferences() { + try { + localStorage.setItem(GRAPH_PREFERENCES_KEY, JSON.stringify(graphPreferenceSnapshot())); + } catch (_) {} + } + + function restoreGraphPreferences() { + const preset = graphPreference('preset', byId('graph-preset').value, + ['original', 'compact', 'communities', 'radial', 'constellation']); + const style = graphPreference('style', byId('graph-style').value, + ['classic', 'galaxy', 'solar', 'cyber']); + const color = graphPreference('color', byId('graph-color').value, + ['community', 'connections', 'type']); + const palette = graphPreference('palette', byId('graph-palette').value, + ['theme', 'aurora', 'ocean', 'ember', 'contrast', 'custom']); + byId('graph-preset').value = preset; + byId('graph-style').value = style; + byId('graph-color').value = color; + byId('graph-palette').value = palette; + + const savedTuning = graphPreference('tuning', {}); + syncGraphTuning({ + ...graphPresetTuning(preset), + ...(savedTuning && typeof savedTuning === 'object' ? savedTuning : {}), + }); + + const savedMin = Number(graphPreference('minDegree', number(byId('graph-min-degree').value))); + const minDegree = Number.isFinite(savedMin) ? Math.max(0, Math.min(12, Math.round(savedMin))) : 1; + setGraphMinDegree(minDegree); + setGraphDepth(graphPreference('depth', 2)); + const savedRepo = graphPreference('repoFilter', ''); + byId('graph-repo-filter').value = typeof savedRepo === 'string' ? savedRepo.slice(0, 200) : ''; + const savedAsOf = graphPreference('asOf', ''); + byId('graph-as-of').value = typeof savedAsOf === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(savedAsOf) + ? savedAsOf : ''; + setGraphShowUnlinked(graphPreference('showUnlinked', state.graphShowUnlinked) === true); + byId('graph-bridges').checked = graphPreference('bridges', byId('graph-bridges').checked) === true; + byId('graph-collapse').checked = graphPreference('collapse', byId('graph-collapse').checked) === true; + byId('graph-ghosts').checked = graphPreference('ghosts', byId('graph-ghosts').checked) !== false; + byId('graph-size').value = graphPreference('size', byId('graph-size').value, + ['degree', 'betweenness']); + // Freeze is deliberately session-only. A previously frozen arrangement must not make a + // freshly opened graph look broken; physics starts live until the person clicks Freeze. + state.graphFrozen = false; + setGraphSwitch('graph-freeze', state.graphFrozen); + setGraphSwitch('graph-flow', graphPreference('flow', true) !== false); + setGraphSwitch('graph-labels', graphPreference('labels', false) === true); + const savedLayers = graphPreference('layers', GRAPH_DEFAULT_LAYERS); + setGraphLayers(GRAPH_LAYERS.reduce((layers, layer) => { + layers[layer] = !savedLayers || typeof savedLayers !== 'object' || savedLayers[layer] !== false; + return layers; + }, {})); + state.graphIncludeCode = graphPreference('includeCode', false) === true; + state.graphSavedView = graphPreference('savedView', 'schema', ['', ...Object.keys(GRAPH_SAVED_VIEWS)]); + syncGraphSavedViews(); + } + + function savedGraphView(id) { + if (id === 'custom') { + try { + const custom = JSON.parse(localStorage.getItem(GRAPH_CUSTOM_VIEW_KEY) || 'null'); + return custom && typeof custom === 'object' ? custom : null; + } catch (_) { + return null; + } + } + return GRAPH_SAVED_VIEWS[id] || null; + } + + function applyGraphView(id) { + const view = savedGraphView(id); + if (!view) { + showNotice(id === 'custom' ? 'No locally saved graph view yet.' : 'That saved graph view is unavailable.'); + return; + } + const preset = Object.prototype.hasOwnProperty.call(GRAPH_PRESET_LABELS, view.preset) + ? view.preset : byId('graph-preset').value; + const style = ['classic', 'galaxy', 'solar', 'cyber'].includes(view.style) ? view.style : byId('graph-style').value; + const color = ['community', 'connections', 'type'].includes(view.color) ? view.color : byId('graph-color').value; + const palette = ['theme', 'aurora', 'ocean', 'ember', 'contrast', 'custom'].includes(view.palette) + ? view.palette : byId('graph-palette').value; + const previousIncludeCode = state.graphIncludeCode; + const previousShowUnlinked = state.graphShowUnlinked; + const previousAsOf = byId('graph-as-of').value; + const asOf = typeof view.asOf === 'string' ? view.asOf : previousAsOf; + const repoFilter = typeof view.repoFilter === 'string' + ? view.repoFilter.slice(0, 200) : byId('graph-repo-filter').value; + state.graphIncludeCode = typeof view.includeCode === 'boolean' + ? view.includeCode : state.graphIncludeCode; + byId('graph-preset').value = preset; + byId('graph-style').value = style; + byId('graph-color').value = color; + byId('graph-palette').value = palette; + byId('graph-as-of').value = asOf; + byId('graph-repo-filter').value = repoFilter; + if (typeof view.ghosts === 'boolean') byId('graph-ghosts').checked = view.ghosts; + if (['degree', 'betweenness'].includes(view.size)) byId('graph-size').value = view.size; + if (typeof view.bridges === 'boolean') byId('graph-bridges').checked = view.bridges; + if (typeof view.collapse === 'boolean') byId('graph-collapse').checked = view.collapse; + if (typeof view.flow === 'boolean') setGraphSwitch('graph-flow', view.flow); + if (typeof view.labels === 'boolean') setGraphSwitch('graph-labels', view.labels); + setGraphSwitch('graph-freeze', state.graphFrozen); + syncGraphTuning({ + ...graphPresetTuning(preset), + ...(view.tuning && typeof view.tuning === 'object' ? view.tuning : {}), + }); + setGraphMinDegree(view.minDegree == null ? 1 : view.minDegree, false); + setGraphDepth(view.depth == null ? 2 : view.depth, false); + setGraphShowUnlinked(view.showUnlinked === true, false); + setGraphLayers(view.layers); + state.graphSavedView = id === 'custom' ? '' : id; + syncGraphChoices(); + if (state.graphEngine) { + state.graphEngine.apply(graph => { + graph.setPreset(preset); + graph.setStyle(style); + graph.setColorBy(color); + applyGraphPalette(palette); + graph.setSettings({ + ...graphTuningSettings(), + flow: byId('graph-flow').getAttribute('aria-checked') === 'true', + labels: byId('graph-labels').getAttribute('aria-checked') === 'true', + frozen: state.graphFrozen, + }); + graph.setScope(graphScope()); + graph.setLayers(graphLayerState()); + graph.setRepoFilter(repoFilter); + graph.setAsOf(graphAsOfTimestamp()); + graph.setSizeBy(byId('graph-size').value); + graph.setBridges(byId('graph-bridges').checked); + graph.setCollapse(byId('graph-collapse').checked ? 'auto' : false); + graph.setGhosts(byId('graph-ghosts').checked); + }, false, !state.graphFrozen); + state.graphEngine.freeze(state.graphFrozen); + } + saveGraphPreferences(); + if (previousIncludeCode !== state.graphIncludeCode + || previousShowUnlinked !== state.graphShowUnlinked || previousAsOf !== asOf) { + loadGraph({ force: true }); + } + const label = all('[data-graph-saved-view]').find(control => control.dataset.graphSavedView === id); + showNotice(`${id === 'custom' ? 'Saved' : (label ? label.textContent : 'Saved')} graph view applied.`); + } + + function saveCurrentGraphView() { + try { + localStorage.setItem(GRAPH_CUSTOM_VIEW_KEY, JSON.stringify(graphPreferenceSnapshot())); + byId('graph-saved-view-status').textContent = 'Current graph view saved locally.'; + showNotice('Current graph view saved locally.'); + } catch (_) { + showNotice('Could not save this graph view in local storage.'); + } + } + + function resetGraphTuning() { + const preset = byId('graph-preset').value; + const previousIncludeCode = state.graphIncludeCode; + const previousShowUnlinked = state.graphShowUnlinked; + state.graphIncludeCode = false; + syncGraphTuning({ ...graphPresetTuning(preset), flowSpeed: 45 }); + setGraphMinDegree(1, false); + setGraphDepth(2, false); + setGraphShowUnlinked(false, false); + setGraphLayers(GRAPH_DEFAULT_LAYERS); + clearGraphSavedView(); + if (state.graphEngine) { + state.graphEngine.apply(graph => { + graph.setPreset(preset); + graph.setSettings({ ...graphTuningSettings(), frozen: state.graphFrozen }); + graph.setScope(graphScope()); + graph.setLayers(graphLayerState()); + }, false, !state.graphFrozen); + state.graphEngine.freeze(state.graphFrozen); + } + saveGraphPreferences(); + if (previousIncludeCode || previousShowUnlinked) loadGraph({ force: true }); + showNotice('Graph tuning reset to the selected layout defaults.'); + } + + function applyGraphPalette(name) { + const graph = state.graphEngine; + if (!graph) return; + graph.setPalette(name); + if (name === 'custom') graph.setTypeColors(GRAPH_CUSTOM_PALETTE); + } + + function graphThemeColors() { + const css = getComputedStyle(document.body); + return { + accent: css.getPropertyValue('--c-acc').trim() || '#a39bf1', + surface: css.getPropertyValue('--c-surface').trim() || '#16191f', + canvas: css.getPropertyValue('--c-bg').trim() || '#0e1014', + label: css.getPropertyValue('--c-fg').trim() || '#e7e9ee', + relation_label: css.getPropertyValue('--c-dim').trim() || '#929baa', + }; + } + + function setGraphTab(tab) { + all('[data-graph-tab]').forEach(control => { + const active = control.dataset.graphTab === tab; + control.classList.toggle('active', active); + control.setAttribute('aria-selected', String(active)); + control.tabIndex = active ? 0 : -1; + }); + all('[data-graph-tab-panel]').forEach(panel => { + panel.hidden = panel.dataset.graphTabPanel !== tab; + }); + } + + function downloadGraphFile(blob, name) { + const href = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = href; + link.download = name; + document.body.append(link); + link.click(); + link.remove(); + window.setTimeout(() => URL.revokeObjectURL(href), 0); + } + + function exportGraphJson() { + const graph = state.graphEngine && state.graphEngine.exportData + ? state.graphEngine.exportData() + : state.graphData || { nodes: [], links: [] }; + const payload = { + workspace: state.workspace, + exported_at: new Date().toISOString(), + nodes: graph.nodes, + links: graph.links, + }; + downloadGraphFile(new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }), 'engraphis-graph.json'); + showNotice('Graph data exported as JSON.'); + } + + function exportGraphPng() { + const canvas = byId('graph-canvas').querySelector('canvas'); + if (!canvas || !canvas.toBlob) { + showNotice('The graph image is not ready yet. Export JSON data instead.'); + return; + } + canvas.toBlob(blob => { + if (!blob) { + showNotice('Could not capture the graph image. Export JSON data instead.'); + return; + } + downloadGraphFile(blob, 'engraphis-graph.png'); + showNotice('Graph image exported as PNG.'); + }, 'image/png'); + } + + function graphCountText(nodes, links) { + const available = number(state.graphMeta && state.graphMeta.nodes_available) || nodes; + const prefix = state.graphMode === 'full' && state.graphMeta && state.graphMeta.nodes_complete + ? 'Full graph' + : 'Overview'; + const entityText = available > nodes + ? `${number(nodes).toLocaleString()} of ${available.toLocaleString()} entities` + : `${number(nodes).toLocaleString()} entities`; + return `${prefix} · ${entityText} · ${number(links).toLocaleString()} relations`; + } + + function graphStatsChanged(stats) { + if (!stats) return; + const nodes = stats.nodes == null ? state.graphData.nodes.length : stats.nodes; + const links = stats.links == null ? state.graphData.links.length : stats.links; + byId('graph-count').textContent = graphCountText(nodes, links); + } + + function graphMetricsChanged(metrics) { + state.graphMetrics = metrics || {}; + byId('graph-bridge-count').textContent = metrics && metrics.bridges != null + ? `${metrics.bridges} bridge ${metrics.bridges === 1 ? 'edge' : 'edges'}` + : ''; + } + + function graphAsOfTimestamp() { + const value = byId('graph-as-of').value; + if (!value) return null; + // A date picker represents the complete selected day, not midnight at its start. + const timestamp = Date.parse(`${value}T23:59:59.999Z`); + return Number.isFinite(timestamp) ? timestamp : null; + } + + function graphAsOfQuery() { + const timestamp = graphAsOfTimestamp(); + return timestamp === null ? '' : `&as_of=${encodeURIComponent(timestamp / 1000)}`; + } + + async function loadGraph({ force = false } = {}) { + if (!state.workspace) return; + if (!force && state.graphWorkspace === state.workspace + && state.graphDataMode === state.graphMode + && state.graphDataIncludeCode === state.graphIncludeCode + && state.graphDataShowUnlinked === state.graphShowUnlinked + && state.graphDataAsOf === graphAsOfTimestamp() && state.graphData) { + if (state.graphEngine) state.graphEngine.resize(); + return; + } + const targetWorkspace = state.workspace; + const targetMode = state.graphMode; + const targetIncludeCode = state.graphIncludeCode; + const targetShowUnlinked = state.graphShowUnlinked; + const targetAsOf = graphAsOfTimestamp(); + const fullGraph = targetMode === 'full'; + if (state.graphLoadPromise && state.graphLoadWorkspace === targetWorkspace + && state.graphLoadMode === targetMode && state.graphLoadIncludeCode === targetIncludeCode + && state.graphLoadShowUnlinked === targetShowUnlinked + && state.graphLoadAsOf === targetAsOf) { + return state.graphLoadPromise; + } + if (state.graphLoadPromise && state.graphLoadController) state.graphLoadController.abort(); + byId('graph-empty').hidden = false; + byId('graph-empty').textContent = fullGraph + ? 'Loading every available graph node…' + : 'Loading the responsive evidence graph…'; + const task = (async () => { + const controller = new AbortController(); + state.graphLoadController = controller; + const timeout = window.setTimeout( + () => controller.abort(), + fullGraph ? GRAPH_FULL_LOAD_TIMEOUT_MS : GRAPH_LOAD_TIMEOUT_MS, + ); + try { + const limit = fullGraph ? GRAPH_FULL_NODE_LIMIT : GRAPH_INITIAL_NODE_LIMIT; + const complete = fullGraph ? '&full=true' : ''; + const connectedOnly = !fullGraph && !targetShowUnlinked ? '&connected_only=true' : ''; + const includeCode = targetIncludeCode ? '&include_code=true' : ''; + const asOf = targetAsOf === null ? '' : `&as_of=${encodeURIComponent(targetAsOf / 1000)}`; + const [payload] = await Promise.all([ + api(`/graph?${query(targetWorkspace)}&limit=${limit}${complete}${connectedOnly}${includeCode}${asOf}`, { signal: controller.signal }), + ensureGraphAssets(), + ]); + if (state.workspace !== targetWorkspace || state.graphMode !== targetMode + || state.graphIncludeCode !== targetIncludeCode + || state.graphShowUnlinked !== targetShowUnlinked + || graphAsOfTimestamp() !== targetAsOf) return; + const data = { nodes: graphNodes(payload), links: graphLinks(payload), suggestions: payload.suggestions || [] }; + state.graphData = data; + state.graphWorkspace = targetWorkspace; + state.graphDataMode = targetMode; + state.graphDataIncludeCode = targetIncludeCode; + state.graphDataShowUnlinked = targetShowUnlinked; + state.graphDataAsOf = targetAsOf; + state.graphMeta = payload.meta || { + nodes_available: data.nodes.length, + nodes_complete: fullGraph, + }; + if (state.graphEngine) state.graphEngine.destroy(); + if (typeof window.EngraphisGraph === 'undefined') throw new Error('graph engine asset is unavailable'); + state.graphEngine = window.EngraphisGraph.create(byId('graph-canvas'), { + renderMode: targetMode, + onNodeClick: item => openGraphConnections(item), + onBackgroundClick: () => state.graphEngine && state.graphEngine.clearFocus(), + onStats: graphStatsChanged, + onMetrics: graphMetricsChanged, + onCollapseChange: collapsed => { + if (targetMode === 'overview') showNotice(collapsed ? 'Clusters collapsed for overview.' : ''); + }, + }); + state.graphEngine.apply(graph => { + graph.setPreset(byId('graph-preset').value); + graph.setStyle(byId('graph-style').value); + graph.setColorBy(byId('graph-color').value); + graph.setThemeColors(graphThemeColors()); + applyGraphPalette(byId('graph-palette').value); + graph.setSettings({ + ...graphTuningSettings(), + flow: byId('graph-flow').getAttribute('aria-checked') === 'true', + labels: byId('graph-labels').getAttribute('aria-checked') === 'true', + frozen: state.graphFrozen, + }); + graph.setScope(graphScope()); + graph.setLayers(graphLayerState()); + graph.setRepoFilter(byId('graph-repo-filter').value); + graph.setAsOf(graphAsOfTimestamp()); + graph.setSizeBy(byId('graph-size').value); + graph.setBridges(byId('graph-bridges').checked); + graph.setCollapse(fullGraph ? false : (byId('graph-collapse').checked ? 'auto' : false)); + graph.setGhosts(byId('graph-ghosts').checked); + }, false, false); + state.graphEngine.setData(data); + state.graphEngine.freeze(state.graphFrozen); + byId('graph-empty').hidden = Boolean(data.nodes.length); + if (!data.nodes.length) byId('graph-empty').textContent = 'No entities exist in this workspace yet.'; + updateGraphModeControls(); + updateGraphFacts(data); + updateGraphLayerCounts(data, payload.layers); + } catch (error) { + if (state.workspace !== targetWorkspace || state.graphMode !== targetMode) return; + byId('graph-empty').hidden = false; + byId('graph-empty').textContent = error && error.name === 'AbortError' + ? `${fullGraph ? 'Full graph' : 'Graph'} loading timed out. Choose Retry to try again.` + : `Graph unavailable: ${error.message}`; + } finally { + window.clearTimeout(timeout); + if (state.graphLoadController === controller) state.graphLoadController = null; + } + })(); + state.graphLoadWorkspace = targetWorkspace; + state.graphLoadMode = targetMode; + state.graphLoadIncludeCode = targetIncludeCode; + state.graphLoadShowUnlinked = targetShowUnlinked; + state.graphLoadAsOf = targetAsOf; + state.graphLoadPromise = task; + try { + return await task; + } finally { + if (state.graphLoadPromise === task) { + state.graphLoadPromise = null; + state.graphLoadWorkspace = ''; + state.graphLoadMode = ''; + state.graphLoadIncludeCode = false; + state.graphLoadShowUnlinked = false; + state.graphLoadAsOf = null; + } + } + } + + function searchGraph(value) { + const target = byId('graph-search-results'); + target.replaceChildren(); + const needle = value.trim().toLowerCase(); + if (!needle || !state.graphData) return; + state.graphData.nodes + .filter(item => item.name.toLowerCase().includes(needle)) + .slice(0, 8) + .forEach(item => { + target.append(button(`${item.name} · ${item.degree}`, 'search-result', () => { + revealGraphNode(item.id, item.name); + target.replaceChildren(); + openGraphConnections(item); + })); + }); + } + + function renderMemoryCollection(target, memories, message) { + target.replaceChildren(); + if (!memories.length) { + target.append(empty(message)); + return; + } + memories.forEach(memory => target.append(simpleMemoryCard(memory))); + } + + function switchProvenanceTab(tab) { + state.provenanceTab = tab; + all('[data-provenance-tab]').forEach(control => { + const active = control.dataset.provenanceTab === tab; + control.classList.toggle('active', active); + control.setAttribute('aria-selected', String(active)); + control.tabIndex = active ? 0 : -1; + }); + all('[data-provenance-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.provenancePanel === tab)); + if (tab === 'audit') loadAudit(); + } + + async function whySearch(event) { + event.preventDefault(); + const question = byId('why-input').value.trim(); + if (!question) { + showNotice('Enter a claim or topic before tracing belief.'); + byId('why-input').focus(); + return; + } + const request = beginScopedRequest('why'); + showNotice(''); + const target = byId('why-result'); + target.replaceChildren(empty('Tracing the live belief and supersession chain…')); + try { + const payload = await api(`/why?q=${encodeURIComponent(question)}&${query(request.workspace)}&k=8`); + if (!isCurrentScopedRequest(request)) return; + target.replaceChildren(); + const live = payload.answer || []; + const superseded = payload.supersedes || []; + target.append(node('h2', '', 'Live support')); + if (!live.length) target.append(empty('No live supporting memory was found.')); + else live.forEach(memory => target.append(simpleMemoryCard(memory))); + target.append(node('h2', '', 'Superseded history')); + if (!superseded.length) target.append(empty('No superseded versions were found.')); + else superseded.forEach(memory => target.append(simpleMemoryCard(memory, 'timeline-card'))); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + target.replaceChildren(empty(`Could not trace belief: ${error.message}`)); + } + } + + async function timelineSearch(event, supersessionsOnly = false) { + event.preventDefault(); + const input = byId(supersessionsOnly ? 'supersession-input' : 'timeline-input'); + const target = byId(supersessionsOnly ? 'supersession-list' : 'timeline-result'); + const question = input.value.trim(); + if (!question) { + showNotice(`Enter a topic before ${supersessionsOnly ? 'finding supersessions' : 'showing history'}.`); + input.focus(); + return; + } + const request = beginScopedRequest(supersessionsOnly ? 'supersessions' : 'timeline'); + showNotice(''); + target.replaceChildren(empty('Loading temporal history…')); + try { + const payload = await api(`/timeline?q=${encodeURIComponent(question)}&${query(request.workspace)}&limit=50`); + if (!isCurrentScopedRequest(request)) return; + let history = payload.history || []; + if (supersessionsOnly) history = history.filter(item => item.valid_to || item.expired_at); + renderMemoryCollection(target, history, supersessionsOnly ? 'No closed versions were found for this topic.' : 'No temporal history was found.'); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + target.replaceChildren(empty(`Could not load history: ${error.message}`)); + } + } + + function renderAuditCards(audit, receipts) { + const target = byId('audit-list'); + target.replaceChildren(); + const combined = [ + ...audit.map(item => ({ ...item, _kind: 'audit' })), + ...receipts.map(item => ({ ...item, _kind: 'receipt' })), + ].sort((a, b) => provenanceTimestampMs(b) - provenanceTimestampMs(a)); + if (!combined.length) { + target.append(empty('No audit records or receipts yet.')); + return; + } + combined.slice(0, 120).forEach(item => { + const card = node('article', 'audit-card'); + card.append( + node('span', '', relative(provenanceTimestampMs(item))), + node('strong', '', item.actor || item.source || 'local operator'), + node('span', 'tag', item.operation || item.action || item.event || item._kind), + node('span', '', item.scope || item.workspace || item.status || state.workspace), + node('code', '', truncate(item.hash || item.id || item.receipt_id, 24) || '—'), + ); + target.append(card); + }); + } + + async function loadAudit() { + const request = beginScopedRequest('audit'); + const target = byId('audit-list'); + target.replaceChildren(empty('Loading audit records and receipts…')); + byId('savings-detail').replaceChildren(empty('Loading receipt-backed estimate…')); + const [auditResult, receiptsResult, savingsResult] = await Promise.allSettled([ + api(`/audit?${query(request.workspace)}&limit=100`), + api(`/receipts?${query(request.workspace)}&limit=100`), + api(`/context-savings?${savingsQuery(request.workspace, state.savingsPreset)}`), + ]); + if (!isCurrentScopedRequest(request)) return; + if (savingsResult.status === 'fulfilled') { + renderSavingsDetail(savingsResult.value); + } else { + byId('savings-detail').replaceChildren(empty(`Could not load context savings: ${savingsResult.reason.message}`)); + } + const audit = auditResult.status === 'fulfilled' ? auditItems(auditResult.value) : []; + const receipts = receiptsResult.status === 'fulfilled' ? receiptItems(receiptsResult.value) : []; + if (auditResult.status === 'rejected' && receiptsResult.status === 'rejected') { + target.replaceChildren(empty('Could not load audit records or receipts. Try again.')); + } else { + renderAuditCards(audit, receipts); + } + if (auditResult.status === 'rejected' || receiptsResult.status === 'rejected') { + showNotice('Some provenance data could not be loaded; available records remain visible.'); + } + } + + async function verifyReceipts() { + try { + const result = await api(`/receipts/verify?${query()}`); + const valid = result.valid != null ? result.valid : result.verified; + showNotice(valid === false ? 'Receipt verification found a broken chain.' : 'Receipt chain verified.'); + } catch (error) { + showNotice(`Could not verify receipts: ${error.message}`); + } + } + + async function exportReceipts() { + try { + const receipts = await api(`/receipts/export?${query()}`); + const blob = new Blob([JSON.stringify(receipts, null, 2)], { type: 'application/json' }); + const link = document.createElement('a'); + const url = URL.createObjectURL(blob); + link.href = url; + link.download = `engraphis-receipts-${state.workspace || 'workspace'}.json`; + document.body.append(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); + showNotice('Privacy-safe receipts exported.'); + } catch (error) { + showNotice(`Could not export receipts: ${error.message}`); + } + } + + function switchManageTab(tab) { + state.manageTab = tab; + all('[data-manage-tab]').forEach(control => { + const active = control.dataset.manageTab === tab; + control.classList.toggle('active', active); + control.setAttribute('aria-selected', String(active)); + control.tabIndex = active ? 0 : -1; + }); + all('[data-manage-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.managePanel === tab)); + loadManageTab(tab); + } + + async function loadManageTab(tab) { + if (tab === 'workspaces') renderWorkspaceList(); + if (tab === 'settings') await loadSettings(); + if (tab === 'plans') await loadPlans(); + if (tab === 'analytics') await loadHosted('analytics'); + if (tab === 'automation') await loadHosted('automation'); + if (tab === 'team') await loadHosted('team'); + if (tab === 'sync') await loadSync(); + } + + function renderWorkspaceList() { + const target = byId('workspace-list'); + target.replaceChildren(); + if (!state.workspaces.length) { + target.append(empty('Create the first workspace to begin.')); + return; + } + state.workspaces.forEach(item => { + const name = workspaceName(item); + const card = node('article', `workspace-card${name === state.workspace ? ' active' : ''}`); + const copy = node('div'); + copy.append( + node('h3', '', name), + node('p', '', item.description || `${number(item.memories).toLocaleString()} memories · ${item.visibility || 'local'}`), + ); + const actions = node('div', 'workspace-card-actions'); + if (name !== state.workspace) actions.append(button('Switch to', 'secondary-button', () => selectWorkspace(name))); + actions.append( + button('Rename', 'secondary-button', () => renameWorkspace(name)), + button('Copy', 'secondary-button', () => copyWorkspace(name)), + ); + if (name !== state.workspace) actions.append(button('Delete', 'danger-button', () => deleteWorkspace(name))); + card.append(copy, actions); + target.append(card); + }); + } + + async function createWorkspace(event) { + event.preventDefault(); + const name = byId('new-workspace-name').value.trim(); + const description = byId('new-workspace-description').value.trim(); + if (!name) { + showNotice('Enter a workspace name before creating it.'); + byId('new-workspace-name').focus(); + return; + } + showNotice(''); + try { + await api('/workspaces/create', { + method: 'POST', + body: { workspace: name, description, visibility: 'personal', confirmed: false }, + }); + showNotice(`Workspace ${name} created.`); + byId('create-workspace-form').reset(); + byId('create-workspace-form').hidden = true; + await refreshBootstrap(name); + } catch (error) { + showNotice(`Could not create workspace: ${error.message}`); + } + } + + async function renameWorkspace(name) { + const next = window.prompt(`Rename ${name} to:`, name); + if (!next || next === name) return; + try { + await api('/workspaces/rename', { method: 'POST', body: { workspace: name, new_name: next } }); + showNotice(`Workspace renamed to ${next}.`); + await refreshBootstrap(name === state.workspace ? next : state.workspace); + } catch (error) { + showNotice(`Could not rename workspace: ${error.message}`); + } + } + + async function copyWorkspace(name) { + try { + const result = await api('/workspaces/copy', { method: 'POST', body: { workspace: name } }); + showNotice(`Workspace copied${result.name ? ` to ${result.name}` : ''}.`); + await refreshBootstrap(state.workspace); + } catch (error) { + showNotice(`Could not copy workspace: ${error.message}`); + } + } + + async function deleteWorkspace(name) { + if (!window.confirm(`Delete workspace “${name}”? Its memories are retired through the governed workspace operation.`)) return; + try { + await api('/workspaces/delete', { method: 'POST', body: { workspace: name } }); + showNotice(`Workspace ${name} deleted.`); + await refreshBootstrap(state.workspace); + } catch (error) { + showNotice(`Could not delete workspace: ${error.message}`); + } + } + + function renderObject(target, payload, title = 'Result') { + target.replaceChildren(); + target.append(node('h3', '', title)); + const entries = Object.entries(payload || {}).filter(([, value]) => ['string', 'number', 'boolean'].includes(typeof value)).slice(0, 12); + if (entries.length) target.append(definitionList(entries.map(([key, value]) => [key.replaceAll('_', ' '), text(value)]))); + else target.append(node('p', '', 'The operation completed.')); + } + + function consolidationOptions() { + return { + workspace: state.workspace, + infer: false, + structured: byId('consolidate-structured').checked, + }; + } + + function sameConsolidationOptions(left, right) { + return Boolean(left && right) + && left.workspace === right.workspace + && left.infer === right.infer + && left.structured === right.structured; + } + + function invalidateConsolidationReview() { + state.consolidationReview = null; + byId('consolidate-commit').disabled = true; + } + + async function previewConsolidation(event) { + event.preventDefault(); + const options = consolidationOptions(); + invalidateConsolidationReview(); + const target = byId('consolidate-result'); + target.replaceChildren(empty('Scanning local memory without writing changes…')); + try { + const result = await api('/consolidate', { + method: 'POST', + body: { + ...options, + dry_run: true, + }, + }); + // The preview is an approval only for the exact workspace and choices that + // produced it; never let a late response authorize a changed form. + if (!sameConsolidationOptions(options, consolidationOptions())) return; + state.consolidationReview = options; + byId('consolidate-commit').disabled = false; + renderObject(target, result, 'Dry preview complete · nothing written'); + } catch (error) { + invalidateConsolidationReview(); + target.replaceChildren(empty(`Preview failed: ${error.message}`)); + } + } + + async function commitConsolidation() { + const options = consolidationOptions(); + if (!sameConsolidationOptions(state.consolidationReview, options)) { + invalidateConsolidationReview(); + showNotice('Run a new dry preview after changing the workspace or consolidation options.'); + return; + } + if (!window.confirm(`Commit the reviewed consolidation result for ${state.workspace}? Original records remain in temporal history.`)) return; + const target = byId('consolidate-result'); + target.replaceChildren(empty('Committing the reviewed local consolidation…')); + try { + const result = await api('/consolidate', { + method: 'POST', + body: { + ...options, + dry_run: false, + }, + }); + invalidateConsolidationReview(); + renderObject(target, result, 'Consolidation committed'); + await selectWorkspace(state.workspace); + } catch (error) { + target.replaceChildren(empty(`Commit failed: ${error.message}`)); + } + } + + function automationCheckbox(id, label, checked) { + const field = node('label', 'check-row'); + const input = node('input'); + input.id = id; + input.type = 'checkbox'; + input.checked = Boolean(checked); + field.htmlFor = id; + field.append(input, document.createTextNode(label)); + return field; + } + + function automationNumber(id, label, value, min, max) { + const field = node('label', '', label); + const input = node('input'); + input.id = id; + input.type = 'number'; + input.min = String(min); + input.max = String(max); + input.value = String(value); + field.htmlFor = id; + field.append(input); + return field; + } + + function renderAutomationPolicy(policy, workspace = state.workspace) { + const target = byId('automation-result'); + if (!target) return; + target.replaceChildren(); + const form = node('form', 'automation-policy-form'); + form.dataset.workspace = workspace; + form.dataset.lastRun = String(policy.last_run || ''); + if (policy.bootstrap_required) { + form.append( + node('p', 'automation-policy-note', 'Hosted automation is not initialized for this workspace. Initializing it uploads one bounded workspace snapshot and saves the default Cloud policy. No upload occurs until you choose this action.'), + ); + const actions = node('div', 'automation-policy-actions'); + const bootstrap = node('button', 'primary-button', 'Initialize hosted automation'); + bootstrap.type = 'button'; + bootstrap.addEventListener('click', () => bootstrapAutomation(workspace, bootstrap)); + actions.append(bootstrap); + form.append(actions); + target.append(form); + return; + } + const enabled = Boolean(policy.enabled); + const dreamEnabled = policy.dream_enabled != null ? policy.dream_enabled : policy.dream; + const lastRun = policy.last_run ? ` Last managed run: ${relative(policy.last_run)}.` : ''; + form.append( + node('p', 'automation-policy-note', enabled + ? `This workspace has an active hosted maintenance policy.${lastRun}` + : 'Hosted maintenance is paused for this workspace.'), + automationCheckbox('automation-enabled', 'Enable hosted maintenance', enabled), + automationNumber('automation-cadence', 'Run every (hours)', Math.max(1, Number(policy.cadence_hours) || 24), 1, 8760), + automationCheckbox('automation-dream', 'Enable Auto Dreaming after accumulation and idle time', dreamEnabled), + automationNumber('automation-dream-min', 'Minimum new memories', Math.max(1, Number(policy.dream_min_new) || 25), 1, 100000), + automationNumber('automation-dream-idle', 'Idle minutes before Dreaming', Math.max(0, Number(policy.dream_idle_minutes) || 0), 0, 10080), + automationCheckbox('automation-infer', 'Allow hosted relationship inference proposals', policy.infer), + node('p', 'automation-policy-note', `Cloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE} Managed compute: saving an enabled policy submits a bounded snapshot of this workspace’s normal and sensitive memory content to Engraphis Cloud. Cloud work returns proposals and never silently changes the local database.`), + ); + const actions = node('div', 'automation-policy-actions'); + const save = node('button', 'primary-button', enabled ? 'Save & send policy to Cloud' : 'Save hosted policy'); + save.type = 'submit'; + actions.append(save); + form.append(actions); + form.addEventListener('submit', saveAutomationPolicy); + target.append(form); + } + + async function bootstrapAutomation(workspace, control) { + if (!workspace || workspace !== state.workspace) return; + if (!window.confirm( + `Initialize hosted automation for ${workspace}? Engraphis will upload one bounded snapshot of that workspace's normal and sensitive memory content and save the default Cloud policy.`, + )) return; + const request = beginScopedRequest('automation-bootstrap'); + control.disabled = true; + control.textContent = 'Initializing…'; + try { + const policy = await api(`/automation/bootstrap?${query(workspace)}`, { method: 'POST' }); + if (!isCurrentScopedRequest(request) || !control.isConnected) return; + state.hostedLoaded.add(`automation:${workspace}`); + renderAutomationPolicy(policy, workspace); + showNotice('Hosted automation initialized.'); + } catch (error) { + if (!isCurrentScopedRequest(request) || !control.isConnected) return; + control.disabled = false; + control.textContent = 'Initialize hosted automation'; + showNotice(`Could not initialize hosted automation: ${error.message}`); + } + } + + async function saveAutomationPolicy(event) { + event.preventDefault(); + const form = event.currentTarget; + const workspace = form.dataset.workspace || ''; + if (!workspace || workspace !== state.workspace) { + showNotice('This policy belongs to a different workspace. Reloading the active workspace policy.'); + state.hostedLoaded.delete(`automation:${state.workspace}`); + await loadHosted('automation'); + return; + } + const request = beginScopedRequest('automation-save'); + const policy = { + enabled: byId('automation-enabled').checked, + cadence_hours: Math.max(1, Number(byId('automation-cadence').value) || 1), + dream_enabled: byId('automation-dream').checked, + dream_min_new: Math.max(1, Number(byId('automation-dream-min').value) || 1), + dream_idle_minutes: Math.max(0, Number(byId('automation-dream-idle').value) || 0), + infer: byId('automation-infer').checked, + }; + if (policy.enabled && !window.confirm( + `Save this hosted policy for ${workspace}? Engraphis will submit a bounded snapshot of that workspace’s normal and sensitive memory content to Cloud for managed compute.\n\nCloud Sync: ${CLOUD_SYNC_PRIVACY_NOTICE}`, + )) return; + const save = form.querySelector('button[type="submit"]'); + if (save) { + save.disabled = true; + save.textContent = 'Saving…'; + } + try { + const saved = await api(`/automation?${query(workspace)}`, { method: 'POST', body: policy }); + if (!isCurrentScopedRequest(request) || !form.isConnected) return; + state.hostedLoaded.add(`automation:${workspace}`); + renderAutomationPolicy({ ...saved, last_run: form.dataset.lastRun }, workspace); + showNotice('Hosted maintenance policy saved to Engraphis Cloud.'); + } catch (error) { + if (!isCurrentScopedRequest(request) || !form.isConnected) return; + if (save) { + save.disabled = false; + save.textContent = policy.enabled ? 'Save & send policy to Cloud' : 'Save hosted policy'; + } + showNotice(`Could not save the hosted policy: ${error.message}`); + } + } + + async function loadHosted(kind) { + const request = beginScopedRequest(`hosted-${kind}`); + const workspace = request.workspace; + const cacheKey = `${kind}:${workspace}`; + const target = byId(`${kind}-result`); + if (state.hostedLoaded.has(cacheKey)) return; + target.replaceChildren(empty(`Checking ${kind} availability…`)); + try { + if (kind === 'team') { + const [auth, license] = await Promise.all([api('/auth/state'), api('/license')]); + if (!isCurrentScopedRequest(request)) return; + state.license = license; + updatePlanBadge(); + renderSidebarCta(); + renderObject(target, { + local_mode: auth.mode || 'open', + hosted_team: Boolean(auth.hosted_team), + cloud_access: Boolean(license.cloud_access_active), + plan: license.plan || 'local', + }, 'Connection state'); + } else { + const result = await api(`/${kind}?${query(workspace)}`); + if (!isCurrentScopedRequest(request)) return; + if (kind === 'automation') renderAutomationPolicy(result, workspace); + else renderObject(target, result, `${kind[0].toUpperCase()}${kind.slice(1)} status`); + } + if (isCurrentScopedRequest(request)) state.hostedLoaded.add(cacheKey); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + target.replaceChildren(empty(`${kind[0].toUpperCase()}${kind.slice(1)} is not active: ${error.message}`)); + } + } + function syncSummaryMessage(summary) { + if (!summary) return 'No sync has run in this dashboard process.'; + const attempted = number(summary.attempted); + const succeeded = number(summary.succeeded); + const errors = Array.isArray(summary.errors) ? summary.errors : []; + const complete = summary.complete === true + || (summary.complete !== false && errors.length === 0 && succeeded >= attempted); + const counts = `${succeeded}/${attempted} eligible workspaces completed`; + const changes = `${number(summary.added)} added · ${number(summary.updated)} updated · ${number(summary.exported)} exported`; + return `${complete ? 'Last sync complete' : 'Last sync incomplete'} · ${counts} · ${changes}${errors.length ? ` · ${errors.length} ${errors.length === 1 ? 'error' : 'errors'}` : ''}.`; + } + + function renderSyncStatus(status, message = '') { + state.syncStatus = status || {}; + const target = byId('sync-result'); + if (!target) return; + target.replaceChildren(); + if (message) target.append(empty(message, 'form-error')); + target.append( + node('p', 'automation-policy-note', syncSummaryMessage(state.syncStatus.last)), + definitionList([ + ['Connection', state.syncStatus.available ? 'Connected' : 'Not connected'], + ['Mode', state.syncStatus.read_only ? 'Read only · pull without upload' : 'Push and pull'], + ['Credential', state.syncStatus.has_cloud_session + ? 'Managed Cloud session' + : (state.syncStatus.has_user_token ? 'Local sync token' : 'None')], + ]), + node('p', 'automation-policy-note', CLOUD_SYNC_PRIVACY_NOTICE), + ); + const actions = node('div', 'automation-policy-actions'); + const run = button('Sync now', 'primary-button', runCloudSync); + run.id = 'sync-now'; + run.disabled = !state.syncStatus.available; + actions.append(run); + if (!state.syncStatus.available) { + const url = safeUrl(state.syncStatus.upgrade_url) || hostedAccountUrl('sync'); + if (url) { + const connect = node('a', 'secondary-button', 'Connect Engraphis Cloud'); + connect.href = url; + connect.target = '_blank'; + connect.rel = 'noopener'; + actions.append(connect); + } + } + target.append(actions); + } + + async function loadSync() { + const request = beginScopedRequest('sync-status'); + const target = byId('sync-result'); + if (!target) return; + target.replaceChildren(empty('Checking Cloud Sync connection…')); + try { + const status = await api('/sync/status'); + if (!isCurrentScopedRequest(request)) return; + renderSyncStatus(status); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + target.replaceChildren(empty(`Could not load Cloud Sync status: ${error.message}`, 'form-error')); + } + } + + async function runCloudSync() { + const request = beginScopedRequest('sync-run'); + const buttonNode = byId('sync-now'); + if (buttonNode) { + buttonNode.disabled = true; + buttonNode.textContent = 'Syncing…'; + } + try { + const result = await api('/sync/run', { method: 'POST' }); + if (!isCurrentScopedRequest(request)) return; + const summary = result && result.summary ? result.summary : {}; + const responseOk = Boolean(result) && result.ok !== false; + const displayedSummary = responseOk ? summary : { ...summary, complete: false }; + renderSyncStatus({ ...(state.syncStatus || {}), last: displayedSummary }); + const errors = Array.isArray(summary.errors) ? summary.errors : []; + const complete = responseOk && (summary.complete === true + || (summary.complete !== false && errors.length === 0 + && number(summary.succeeded) >= number(summary.attempted))); + showNotice(complete + ? 'Cloud Sync completed for every eligible workspace.' + : 'Cloud Sync is incomplete. Review the status before retrying.'); + } catch (error) { + if (!isCurrentScopedRequest(request)) return; + renderSyncStatus(state.syncStatus || {}, `Cloud Sync failed: ${error.message}`); + showNotice(`Cloud Sync failed: ${error.message}`); + } + } + + function planPrices() { + const annual = byId('billing-select').value === 'annual'; + return annual + ? { free: '$0', pro: '$100 / owner / year', team: '$200 / seat / year' } + : { free: '$0', pro: '$10 / owner / month', team: '$20 / seat / month' }; + } + + function renderPlans() { + const target = byId('plan-cards'); + target.replaceChildren(); + const prices = planPrices(); + const plans = [ + { id: 'free', name: 'Free', price: prices.free, note: 'The complete local memory engine and every core operation.', action: 'Current local plan' }, + { id: 'pro', name: 'Pro', price: prices.pro, note: 'Cloud sync, managed automation and portfolio analytics.' }, + { id: 'team', name: 'Team', price: prices.team, note: 'Shared workspaces, member roles, seats and remote agents.' }, + ]; + plans.forEach(plan => { + const card = node('article', `plan-card${plan.id === 'pro' ? ' featured' : ''}`); + card.append( + node('p', 'eyebrow', plan.id === (state.license && state.license.plan) ? 'Current plan' : plan.id), + node('h2', '', plan.name), + node('div', 'price', plan.price), + node('p', '', plan.note), + ); + if (plan.id === 'pro') { + card.append( + node('p', 'plan-support', 'Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development.'), + node('p', 'plan-benefits', 'Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming across your installations.'), + ); + } + if (plan.id === 'free') { + const status = node('span', 'secondary-button', plan.action); + card.append(status); + } else { + const interval = byId('billing-select').value === 'annual' ? 'annual' : 'monthly'; + const cta = hostedCta(plan.id, 'plans', interval); + const action = node('a', 'primary-button', cta.label); + const url = cta.href; + action.dataset.proCta = plan.id; + action.href = url || '#'; + if (url) { + action.target = '_blank'; + action.rel = 'noopener'; + } else { + action.addEventListener('click', event => { + event.preventDefault(); + showNotice('Connect this installation to Engraphis Cloud to open hosted plan options.'); + }); + } + card.append(action); + } + target.append(card); + }); + } + + async function loadPlans() { + const request = beginScopedRequest('plans'); + try { + const license = await api(`/license?${query(request.workspace)}`); + if (!isCurrentScopedRequest(request)) return; + state.license = license; + } catch (_) { + if (!isCurrentScopedRequest(request)) return; + state.license = { plan: 'free' }; + } + updatePlanBadge(); + renderSidebarCta(); + renderPlans(); + } + + function llmSnippet(provider, model, keySet) { + return [ + `ENGRAPHIS_LLM_PROVIDER=${provider}`, + `ENGRAPHIS_LLM_MODEL=${model}`, + 'ENGRAPHIS_LLM_API_KEY=', + keySet ? 'ENGRAPHIS_EXTRACTOR=llm_structured' : '# set ENGRAPHIS_EXTRACTOR=llm_structured to use it', + 'ENGRAPHIS_LLM_AUTO_EXTRACT=1', + ].join('\n'); + } + + function setLlmTestResult(message, tone = '') { + const target = byId('llm-test-result'); + if (!target) return; + target.textContent = message; + target.dataset.tone = tone; + } + + function updateLlmSnippet(status) { + const provider = byId('llm-provider').value; + const model = byId('llm-model').value; + byId('llm-env-snippet').value = llmSnippet(provider, model, Boolean(status.key_set)); + } + + function renderLlmSettings(status) { + const target = byId('llm-connection'); + target.replaceChildren(); + const defaults = status.default_models || {}; + const provider = status.provider || 'openai'; + const model = status.model || defaults[provider] || ''; + const providers = [...new Set([...Object.keys(defaults), provider])]; + const models = [...new Set([model, ...Object.values(defaults)].filter(Boolean))]; + const configured = Boolean(status.configured); + const extractionEnabled = Boolean(status.extractor_enabled); + const stateLabel = status.working ? 'verified' : (configured ? 'configured' : 'not configured'); + + const overview = node('div', 'llm-status-line'); + overview.append( + node('span', '', 'Provider · Model'), + node('span', `llm-status-badge ${configured ? 'ready' : 'muted'}`, stateLabel), + ); + + const pickerGrid = node('div', 'llm-picker-grid'); + const providerLabel = node('label', '', 'Provider'); + const providerSelect = node('select'); + providerSelect.id = 'llm-provider'; + providers.forEach(value => providerSelect.append(option(value, value, value === provider))); + providerLabel.htmlFor = providerSelect.id; + providerLabel.append(providerSelect); + const modelLabel = node('label', '', 'Model'); + const modelSelect = node('select'); + modelSelect.id = 'llm-model'; + models.forEach(value => modelSelect.append(option(value, value, value === model))); + modelLabel.htmlFor = modelSelect.id; + modelLabel.append(modelSelect); + pickerGrid.append(providerLabel, modelLabel); + + const keyState = node('p', 'llm-key-state', status.key_set ? 'API key set' : 'No API key set'); + keyState.append(node('span', '', ` · extractor: ${status.extractor || 'none'}`)); + const setupNote = node('p', 'llm-setup-note', 'Choose a provider and model for the copyable .env snippet. Update it locally, then restart Engraphis to apply the change.'); + const snippetLabel = node('label', 'llm-snippet-label', 'Local .env setup'); + const snippet = node('textarea', 'llm-env-snippet'); + snippet.id = 'llm-env-snippet'; + snippet.readOnly = true; + snippet.rows = 5; + snippet.value = llmSnippet(provider, model, Boolean(status.key_set)); + snippetLabel.htmlFor = snippet.id; + snippetLabel.append(snippet); + const copy = button('Copy', 'secondary-button', copyLlmSnippet); + copy.classList.add('llm-copy-button'); + const snippetWrap = node('div', 'llm-snippet-wrap'); + snippetWrap.append(snippetLabel, copy); + + const extraction = node('div', 'llm-status-line'); + extraction.append( + node('span', '', 'LLM extraction'), + node('span', `llm-status-badge ${extractionEnabled ? 'ready' : 'muted'}`, extractionEnabled ? 'ON' : 'OFF'), + ); + const extractionNote = node('p', 'llm-extraction-note', 'While ON, ingested memory content is sent to your configured provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.'); + const retentionUsesLlm = text(status.retention_supervisor).toLowerCase() === 'llm'; + const retentionNote = node( + 'p', + 'llm-extraction-note', + retentionUsesLlm + ? 'Retention supervision is ON. New memories may send their title and a bounded excerpt to the configured provider.' + : 'Retention supervision is OFF.', + ); + const extractionActions = node('div', 'llm-actions'); + const turnOn = button('Turn on', 'primary-button', () => setLlmExtractor(true)); + turnOn.disabled = extractionEnabled || !configured; + const turnOff = button('Turn off', 'secondary-button', () => setLlmExtractor(false)); + turnOff.disabled = !extractionEnabled; + extractionActions.append(turnOn, turnOff); + + const testActions = node('div', 'llm-actions'); + testActions.append(button('Test connection', 'secondary-button', testLlm)); + const testResult = node('p', 'llm-test-result'); + testResult.id = 'llm-test-result'; + testResult.setAttribute('role', 'status'); + testResult.setAttribute('aria-live', 'polite'); + testActions.append(testResult); + + providerSelect.addEventListener('change', () => { + const defaultModel = defaults[providerSelect.value]; + if (defaultModel && models.includes(defaultModel)) modelSelect.value = defaultModel; + updateLlmSnippet(status); + }); + modelSelect.addEventListener('change', () => updateLlmSnippet(status)); + target.append(overview, pickerGrid, keyState, setupNote, snippetWrap, extraction, extractionNote, retentionNote, extractionActions, testActions); + } + + async function copyLlmSnippet() { + const snippet = byId('llm-env-snippet'); + try { + await navigator.clipboard.writeText(snippet.value); + showNotice('Copied the local .env setup snippet.'); + } catch (_) { + snippet.focus(); + snippet.select(); + if (document.execCommand('copy')) showNotice('Copied the local .env setup snippet.'); + else showNotice('Select the snippet and copy it manually.'); + } + } + + async function loadSettings() { + try { + state.license = await api('/license'); + updatePlanBadge(); + renderSidebarCta(); + } catch (_) {} + renderCloudAccountSettings(); + try { + renderLlmSettings(await api('/llm/status')); + } catch (error) { + byId('llm-connection').replaceChildren(empty(`Model status unavailable: ${error.message}`)); + } + } + + async function setLlmExtractor(enabled) { + if (enabled && !window.confirm(`Turn on LLM extraction? ${EXTERNAL_LLM_PRIVACY_NOTICE}`)) return; + setLlmTestResult(enabled ? 'Verifying the configured provider…' : 'Turning extraction off…'); + try { + const result = await api('/llm/extractor', { method: 'POST', body: { enabled } }); + await loadSettings(); + const state = result.extractor_enabled ? 'LLM extraction is on for new ingested memories.' : 'LLM extraction is off for new ingested memories.'; + setLlmTestResult(`${state}${result.persisted === false ? ' The restart setting could not be saved.' : ''}`, result.extractor_enabled ? 'ready' : 'muted'); + } catch (error) { + setLlmTestResult(`Could not change extraction: ${error.message}`, 'error'); + } + } + + async function testLlm() { + setLlmTestResult('Testing the configured model…'); + try { + const result = await api('/llm/test', { method: 'POST' }); + await loadSettings(); + if (result.ok) { + const suffix = result.auto_enabled ? ' Extraction is active for new ingested memories.' : ''; + setLlmTestResult(`Connected — ${result.provider}/${result.model}.${suffix}`, 'ready'); + } else { + setLlmTestResult(`Could not connect: ${result.error || 'Check the provider, model, API key, and network.'}`, 'error'); + } + } catch (error) { + setLlmTestResult(`Model connection failed: ${error.message}`, 'error'); + } + } + + function switchView(view, { pushHistory = true } = {}) { + const validViews = ['today', 'ask', 'library', 'relations', 'provenance', 'manage']; + if (!validViews.includes(view)) view = 'today'; + if (pushHistory && state.view !== view) { + const url = new URL(location.href); + url.searchParams.set('view', view); + window.history.pushState({ view }, '', url); + } + state.view = view; + all('[data-view-panel]').forEach(panel => panel.classList.toggle('active', panel.dataset.viewPanel === view)); + all('[data-view]').forEach(control => { + const active = control.dataset.view === view; + control.classList.toggle('active', active); + if (active) control.setAttribute('aria-current', 'page'); + else control.removeAttribute('aria-current'); + }); + try { + localStorage.setItem('engraphis-ledger-view', view); + } catch (_) {} + if (view === 'relations') loadGraph(); + if (view === 'provenance' && state.provenanceTab === 'audit') loadAudit(); + if (view === 'manage') loadManageTab(state.manageTab); + window.scrollTo({ top: 0, behavior: 'instant' }); + const heading = byId(`${view}-title`); + if (heading) { + heading.setAttribute('tabindex', '-1'); + heading.focus({ preventScroll: true }); + } + } + + function applyTheme(theme) { + const valid = ['slate', 'midnight', 'paper', 'matrix']; + const selected = valid.includes(theme) ? theme : 'slate'; + document.body.dataset.theme = selected; + byId('theme-select').value = selected; + byId('sidebar-theme-select').value = selected; + try { + localStorage.setItem('engraphis-ledger-theme', selected); + localStorage.setItem('engraphis-theme', ({ slate: 'dark', paper: 'light', midnight: 'midnight', matrix: 'matrix' })[selected]); + } catch (_) {} + if (state.graphEngine) state.graphEngine.setThemeColors(graphThemeColors()); + } + + async function refreshBootstrap(preferred = '') { + const bootstrap = await api('/bootstrap'); + renderUpdateBanner(bootstrap.update); + state.workspaces = bootstrap.workspaces || []; + state.license = bootstrap.license || state.license; + updatePlanBadge(); + renderSidebarCta(); + const select = byId('workspace-select'); + select.replaceChildren(); + state.workspaces.forEach(item => { + const name = workspaceName(item); + select.append(option(name, name)); + }); + if (!state.workspaces.length) { + select.append(option('', 'No workspace')); + select.disabled = true; + setConnection('Local engine connected · no workspace'); + state.workspace = ''; + renderWorkspaceNames(); + renderWorkspaceList(); + renderMetricValues({ memories: 0, total_rows: 0, workspaces: 0, sessions: 0 }); + byId('decision-list').replaceChildren(empty('Create a workspace in Manage to start reviewing memory.')); + const emptyActivity = node('tr'); + const emptyActivityCell = node('td', '', 'No workspace selected yet.'); + emptyActivityCell.colSpan = 5; + emptyActivity.append(emptyActivityCell); + byId('activity-body').replaceChildren(emptyActivity); + byId('proactive-list').replaceChildren(empty('Create a workspace to see proactive context.')); + byId('context-savings-summary-body').replaceChildren(empty('Create a workspace to start tracking context savings.')); + byId('context-savings-persistent-value').textContent = '—'; + byId('context-savings-persistent-meta').textContent = 'Create a workspace to start tracking context savings.'; + byId('context-savings-persistent-rate').textContent = '—'; + return; + } + select.disabled = false; + let saved = preferred; + try { + saved = preferred || localStorage.getItem('engraphis-workspace') || ''; + } catch (_) {} + const names = state.workspaces.map(workspaceName); + const selected = names.includes(saved) + ? saved + : workspaceName([...state.workspaces].sort((a, b) => number(b.memories) - number(a.memories))[0]); + await selectWorkspace(selected); + setConnection('Local engine connected'); + } + + async function boot() { + byId('today-date').textContent = new Intl.DateTimeFormat(undefined, { dateStyle: 'long' }).format(new Date()); + let theme = 'slate'; + try { + theme = localStorage.getItem('engraphis-ledger-theme') || theme; + } catch (_) {} + applyTheme(theme); + try { + await refreshBootstrap(); + let view = 'today'; + try { + const saved = localStorage.getItem('engraphis-ledger-view'); + if (['today', 'ask', 'library', 'relations', 'provenance', 'manage'].includes(saved)) view = saved; + } catch (_) {} + const urlView = new URL(location.href).searchParams.get('view'); + switchView(['today', 'ask', 'library', 'relations', 'provenance', 'manage'].includes(urlView) ? urlView : view, { pushHistory: false }); + } catch (error) { + if (error.status === 401 && await authenticateBrowser()) { + location.reload(); + return; + } + setConnection('Local engine unavailable', false); + showNotice(`Ledger could not connect: ${error.message}`); + } + } + + all('[data-view]').forEach(control => control.addEventListener('click', () => switchView(control.dataset.view))); + all('[data-go]').forEach(control => control.addEventListener('click', () => switchView(control.dataset.go))); + all('[data-manage]').forEach(control => control.addEventListener('click', () => { + switchView('manage'); + switchManageTab(control.dataset.manage); + })); + const planBadge = byId('plan-badge'); + if (planBadge) { + planBadge.addEventListener('click', event => { + if (event.currentTarget.dataset.opensAccount === 'true') return; + event.preventDefault(); + switchView('manage'); + switchManageTab('plans'); + }); + } + all('[data-provenance]').forEach(control => control.addEventListener('click', () => { + switchView('provenance'); + switchProvenanceTab(control.dataset.provenance); + })); + all('[data-provenance-tab]').forEach(control => control.addEventListener('click', () => switchProvenanceTab(control.dataset.provenanceTab))); + all('[data-manage-tab]').forEach(control => control.addEventListener('click', () => switchManageTab(control.dataset.manageTab))); + function wireTabKeyboard(selector, dataKey, activate) { + const controls = all(selector); + controls.forEach((control, index) => { + control.tabIndex = control.getAttribute('aria-selected') === 'true' ? 0 : (index ? -1 : 0); + control.addEventListener('keydown', event => { + const direction = event.key === 'ArrowRight' || event.key === 'ArrowDown' ? 1 + : event.key === 'ArrowLeft' || event.key === 'ArrowUp' ? -1 : 0; + let nextIndex = index; + if (event.key === 'Home') nextIndex = 0; + else if (event.key === 'End') nextIndex = controls.length - 1; + else if (direction) nextIndex = (index + direction + controls.length) % controls.length; + else return; + event.preventDefault(); + const next = controls[nextIndex]; + next.focus(); + activate(next.dataset[dataKey]); + }); + }); + } + wireTabKeyboard('[data-graph-tab]', 'graphTab', setGraphTab); + wireTabKeyboard('[data-provenance-tab]', 'provenanceTab', switchProvenanceTab); + wireTabKeyboard('[data-manage-tab]', 'manageTab', switchManageTab); + window.addEventListener('popstate', event => { + const view = event.state && event.state.view + ? event.state.view + : new URL(location.href).searchParams.get('view') || 'today'; + switchView(view, { pushHistory: false }); + }); + + byId('workspace-select').addEventListener('change', event => selectWorkspace(event.target.value)); + byId('ask-form').addEventListener('submit', askMemory); + byId('library-filter').addEventListener('input', renderLibrary); + byId('library-type').addEventListener('change', renderLibrary); + byId('new-memory-button').addEventListener('click', () => openEditor()); + byId('editor-close').addEventListener('click', closeEditor); + byId('editor-cancel').addEventListener('click', closeEditor); + byId('memory-editor').addEventListener('submit', saveMemory); + byId('import-button').addEventListener('click', () => byId('import-files').click()); + byId('import-files').addEventListener('change', event => importFiles(event.target.files)); + byId('obsidian-import-button').addEventListener('click', openObsidianImport); + byId('obsidian-import-close').addEventListener('click', () => byId('obsidian-import-dialog').close()); + byId('obsidian-preview').addEventListener('click', previewObsidianImport); + byId('obsidian-cancel').addEventListener('click', cancelObsidianImport); + byId('obsidian-import-form').addEventListener('submit', runObsidianImport); + byId('obsidian-source-mode').addEventListener('change', updateDocumentImportMode); + byId('obsidian-vault-id').addEventListener('change', applySelectedDocumentSource); + byId('obsidian-import-files').addEventListener('change', () => invalidateDocumentImportPreview()); + byId('obsidian-import-folder').addEventListener('change', () => { + prefillNewSourceLabelFromFolder(); + invalidateDocumentImportPreview(); + }); + [ + ['obsidian-workspace', 'input'], + ['obsidian-repo', 'input'], + ['obsidian-session', 'input'], + ['obsidian-scope', 'change'], + ['obsidian-memory-type', 'change'], + ['obsidian-vault-label', 'input'], + ['obsidian-conflict', 'change'], + ].forEach(([id, eventName]) => { + byId(id).addEventListener(eventName, () => invalidateDocumentImportPreview()); + }); + byId('obsidian-report-filter').addEventListener('change', () => renderObsidianReport(obsidianImport.job || obsidianImport.preview)); + + all('[data-graph-tab]').forEach(control => control.addEventListener('click', () => setGraphTab(control.dataset.graphTab))); + byId('graph-fit').addEventListener('click', () => state.graphEngine && state.graphEngine.fit()); + byId('graph-reheat').addEventListener('click', () => state.graphEngine && state.graphEngine.reheat()); + byId('graph-clear-focus').addEventListener('click', () => { + if (state.graphEngine) state.graphEngine.clearFocus(); + }); + byId('graph-freeze').addEventListener('click', () => { + state.graphFrozen = !state.graphFrozen; + setGraphSwitch('graph-freeze', state.graphFrozen); + if (state.graphEngine) state.graphEngine.freeze(state.graphFrozen); + saveGraphPreferences(); + }); + byId('graph-flow').addEventListener('click', event => { + const on = event.currentTarget.getAttribute('aria-checked') !== 'true'; + setGraphSwitch('graph-flow', on); + if (state.graphEngine) state.graphEngine.setSettings({ flow: on }); + clearGraphSavedView(); + saveGraphPreferences(); + }); + byId('graph-labels').addEventListener('click', event => { + const on = event.currentTarget.getAttribute('aria-checked') !== 'true'; + setGraphSwitch('graph-labels', on); + if (state.graphEngine) state.graphEngine.setSettings({ labels: on }); + clearGraphSavedView(); + saveGraphPreferences(); + }); + byId('graph-flow-speed').addEventListener('input', event => { + const speed = graphValueInRange('graph-flow-speed', event.target.value, 45); + byId('graph-flow-speed').value = String(speed); + byId('graph-flow-speed-output').value = String(Math.round(speed)); + byId('graph-flow-speed-output').textContent = String(Math.round(speed)); + if (state.graphEngine) state.graphEngine.setSettings({ flowSpeed: speed }); + clearGraphSavedView(); + saveGraphPreferences(); + }); + byId('graph-search').addEventListener('input', event => searchGraph(event.target.value)); + byId('graph-repo-filter').addEventListener('input', event => { + if (state.graphEngine) state.graphEngine.setRepoFilter(event.target.value); + clearGraphSavedView(); + saveGraphPreferences(); + }); + all('[data-graph-preset-choice]').forEach(control => control.addEventListener('click', () => { + const preset = control.dataset.graphPresetChoice; + const resumeLayout = state.graphFrozen; + byId('graph-preset').value = preset; + if (state.graphEngine && resumeLayout) { + // Freeze is the safe default for arranging nodes by hand. Selecting a named layout is an + // explicit request to run physics, so make that transition visible and leave the switch + // truthful; the person can freeze the settled arrangement again when they are happy. + state.graphFrozen = false; + setGraphSwitch('graph-freeze', false); + state.graphEngine.freeze(false); + } + let settings = graphPresetTuning(preset); + if (state.graphEngine) settings = state.graphEngine.setPreset(preset); + syncGraphTuning(settings); + updateGraphModeControls(); + clearGraphSavedView(); + syncGraphChoices(); + saveGraphPreferences(); + if (resumeLayout) showNotice('Layout applied. Simulation resumed — freeze it to lock node positions.'); + })); + all('[data-graph-style-choice]').forEach(control => control.addEventListener('click', () => { + byId('graph-style').value = control.dataset.graphStyleChoice; + if (state.graphEngine) state.graphEngine.setStyle(control.dataset.graphStyleChoice); + clearGraphSavedView(); + syncGraphChoices(); + saveGraphPreferences(); + })); + all('[data-graph-color-choice]').forEach(control => control.addEventListener('click', () => { + byId('graph-color').value = control.dataset.graphColorChoice; + if (state.graphEngine) state.graphEngine.setColorBy(control.dataset.graphColorChoice); + clearGraphSavedView(); + syncGraphChoices(); + saveGraphPreferences(); + })); + all('[data-graph-palette-choice]').forEach(control => control.addEventListener('click', () => { + const palette = control.dataset.graphPaletteChoice; + byId('graph-palette').value = palette; + applyGraphPalette(palette); + clearGraphSavedView(); + syncGraphChoices(); + saveGraphPreferences(); + showNotice(`${control.textContent.trim()} palette applied to the graph.`); + })); + byId('graph-min-degree').addEventListener('input', event => { + setGraphMinDegree(event.target.value); + clearGraphSavedView(); + saveGraphPreferences(); + }); + byId('graph-show-unlinked').addEventListener('click', event => { + setGraphShowUnlinked(event.currentTarget.getAttribute('aria-pressed') !== 'true'); + clearGraphSavedView(); + saveGraphPreferences(); + loadGraph({ force: true }); + }); + byId('graph-tune-min-degree').addEventListener('input', event => { + setGraphMinDegree(event.target.value); + clearGraphSavedView(); + saveGraphPreferences(); + }); + byId('graph-depth').addEventListener('input', event => { + setGraphDepth(event.target.value); + clearGraphSavedView(); + saveGraphPreferences(); + }); + GRAPH_TUNING.forEach(item => byId(item.id).addEventListener('input', event => { + const value = setGraphTuningControl(item, event.target.value); + if (state.graphEngine) state.graphEngine.setSettings({ [item.key]: value }); + clearGraphSavedView(); + saveGraphPreferences(); + })); + all('[data-graph-layer]').forEach(control => control.addEventListener('click', () => { + const layers = graphLayerState(); + const layer = control.dataset.graphLayer; + layers[layer] = !layers[layer]; + const previousIncludeCode = state.graphIncludeCode; + state.graphIncludeCode = layers.code === true; + setGraphLayers(layers); + if (state.graphEngine) state.graphEngine.setLayers(layers); + clearGraphSavedView(); + saveGraphPreferences(); + if (previousIncludeCode !== state.graphIncludeCode) loadGraph({ force: true }); + })); + all('[data-graph-saved-view]').forEach(control => control.addEventListener('click', () => applyGraphView(control.dataset.graphSavedView))); + byId('graph-save-view').addEventListener('click', saveCurrentGraphView); + byId('graph-reset-tuning').addEventListener('click', resetGraphTuning); + byId('graph-retry').addEventListener('click', () => loadGraph({ force: true })); + byId('graph-bridges').addEventListener('change', event => { + if (state.graphEngine) state.graphEngine.setBridges(event.target.checked); + saveGraphPreferences(); + }); + byId('graph-collapse').addEventListener('change', event => { + if (state.graphEngine) state.graphEngine.setCollapse(event.target.checked ? 'auto' : false); + saveGraphPreferences(); + }); + byId('graph-as-of').addEventListener('change', event => { + if (state.graphEngine) state.graphEngine.setAsOf(graphAsOfTimestamp()); + saveGraphPreferences(); + loadGraph({ force: true }); + }); + byId('graph-ghosts').addEventListener('change', event => { + if (state.graphEngine) state.graphEngine.setGhosts(event.target.checked); + saveGraphPreferences(); + }); + byId('graph-size').addEventListener('change', event => { + if (state.graphEngine) state.graphEngine.setSizeBy(event.target.value); + saveGraphPreferences(); + }); + byId('graph-export').addEventListener('click', () => { + const menu = byId('graph-export-menu'); + const open = menu.hidden; + menu.hidden = !open; + byId('graph-export').setAttribute('aria-expanded', String(open)); + }); + byId('graph-export-png').addEventListener('click', () => { + byId('graph-export-menu').hidden = true; + byId('graph-export').setAttribute('aria-expanded', 'false'); + exportGraphPng(); + }); + byId('graph-export-json').addEventListener('click', () => { + byId('graph-export-menu').hidden = true; + byId('graph-export').setAttribute('aria-expanded', 'false'); + exportGraphJson(); + }); + byId('graph-connections-close').addEventListener('click', closeGraphConnections); + byId('graph-connections-dialog').addEventListener('click', event => { + if (event.target === event.currentTarget) closeGraphConnections(); + }); + restoreGraphPreferences(); + syncGraphChoices(); + + byId('why-form').addEventListener('submit', whySearch); + byId('timeline-form').addEventListener('submit', event => timelineSearch(event, false)); + byId('supersession-form').addEventListener('submit', event => timelineSearch(event, true)); + byId('verify-receipts').addEventListener('click', verifyReceipts); + byId('export-receipts').addEventListener('click', exportReceipts); + + byId('create-workspace-toggle').addEventListener('click', () => { + byId('create-workspace-form').hidden = !byId('create-workspace-form').hidden; + if (!byId('create-workspace-form').hidden) byId('new-workspace-name').focus(); + }); + byId('create-workspace-form').addEventListener('submit', createWorkspace); + byId('consolidate-form').addEventListener('submit', previewConsolidation); + byId('consolidate-commit').addEventListener('click', commitConsolidation); + ['consolidate-structured'].forEach(id => { + byId(id).addEventListener('change', invalidateConsolidationReview); + }); + byId('billing-select').addEventListener('change', renderPlans); + byId('dashboard-select').addEventListener('change', event => { + location.assign(event.target.value === 'classic' ? '/classic' : '/'); + }); + byId('theme-select').addEventListener('change', event => applyTheme(event.target.value)); + byId('sidebar-theme-select').addEventListener('change', event => applyTheme(event.target.value)); + boot(); +})(); diff --git a/engraphis/document_import.py b/engraphis/document_import.py new file mode 100644 index 00000000..a9f55d50 --- /dev/null +++ b/engraphis/document_import.py @@ -0,0 +1,278 @@ +"""Universal, offline-first document import orchestration for Engraphis v2. + +The format adapters and secure filesystem walk live in :mod:`engraphis.core.documents`. +This module binds their source-neutral records to the already-proven temporal import +planner, per-document transaction finalizer, manifest, graph, jobs, and receipts. +Obsidian remains a compatibility adapter, not the persistence model. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from pathlib import PurePosixPath +import re +from typing import Any, Iterable, Optional +import unicodedata + +from engraphis.core.documents import ( + IMPORTER_VERSION, + MAX_DOCUMENT_BYTES, + MAX_DOCUMENT_FILES, + MAX_DOCUMENT_TREE_BYTES, + SENSITIVE_FILENAMES, + DocumentFileIssue, + DocumentRecord, + DocumentScan, + document_format_for_path, + normalize_document_path, + parse_document, +) +from engraphis.obsidian_import import ObsidianImportCancelled, ObsidianImporter + + +DocumentImportCancelled = ObsidianImportCancelled + + +def local_document_adapter( + raw: bytes, relative_path: str, source_mtime_ns: Optional[int] = None, +) -> DocumentRecord: + """Use installed v2 local resource adapters without crossing into ``core``. + + PDF and OCR adapters are fully local. Audio/video transcription additionally + requires ``ENGRAPHIS_WHISPER_MODEL`` to name an existing local path, preventing + the backend library from resolving a model name over the network. + """ + spec = document_format_for_path(relative_path) + if spec is None or not spec.requires_adapter: + raise ValueError("document format does not require a local adapter") + if spec.name in {"audio", "video"}: + model_path = os.environ.get("ENGRAPHIS_WHISPER_MODEL", "").strip() + selected_model = Path(model_path).expanduser() if model_path else None + if not selected_model or not ( + selected_model.is_file() or selected_model.is_dir() + ): + raise ValueError( + "transcription requires an existing local model file or directory" + ) + # Concrete backend composition deliberately stays outside engraphis/core/. + from engraphis.backends.resources import ResourceExtractionError, get_resource_extractor + + try: + resource = get_resource_extractor().extract_bytes(relative_path, raw) + except ResourceExtractionError as exc: + text = str(exc).casefold() + if "needs" in text or "requires" in text: + raise ValueError("optional local document extractor is unavailable") from None + raise ValueError(_safe_reason(exc)) from None + readable = str(resource.text or "").strip() + if not readable: + raise ValueError("document produced no readable text") + return DocumentRecord( + relative_path=relative_path, format=spec.name, + media_type=str(resource.media_type or spec.media_type), + title=str(resource.title or Path(relative_path).stem)[:300], + content=readable, body=readable, + raw_sha256=hashlib.sha256(raw).hexdigest(), + canonical_sha256=hashlib.sha256(readable.encode("utf-8")).hexdigest(), + source_size=len(raw), source_mtime_ns=source_mtime_ns, + title_source="extracted", + metadata={"resource_kind": resource.kind, **dict(resource.metadata or {})}, + warnings=[str(value)[:500] for value in list(resource.warnings or [])[:100]], + ) + + +def _sensitive_filename(name: str) -> bool: + lowered = name.casefold() + return ( + lowered in SENSITIVE_FILENAMES + or lowered.startswith(".env.") + or lowered.endswith((".pem", ".key", ".p12", ".pfx")) + or bool(re.search(r"(?:credential|recovery[-_ ]?code|secret|token)", lowered)) + ) + + +def _safe_reason(exc: BaseException) -> str: + text = str(exc).casefold() + for label in ( + "secret", "safety limit", "unsupported", "invalid", "unsafe", "binary", + "too large", "no readable text", + ): + if label in text: + return f"source rejected: {label}" + return "source rejected" + + +def scan_document_upload( + files: Iterable[tuple[str, bytes]], *, source_label: str, +) -> DocumentScan: + """Parse browser-selected document bytes without creating an upload copy.""" + label = unicodedata.normalize("NFC", str(source_label or "").strip()[:200]) + if not label: + raise ValueError("source_label is required for a browser source") + source_id = hashlib.sha256( + ("documents-browser\0" + label.casefold()).encode("utf-8", "surrogatepass") + ).hexdigest() + scan = DocumentScan(root_path="", source_id=source_id) + total = 0 + seen: set[str] = set() + for index, (raw_path, raw) in enumerate(files): + if index >= MAX_DOCUMENT_FILES: + scan.rejected.append(DocumentFileIssue( + "(collection)", "source exceeds document file safety limit", + )) + scan.complete = False + break + try: + relative_path = normalize_document_path(raw_path) + except ValueError: + scan.rejected.append(DocumentFileIssue("(invalid path)", "invalid source path")) + continue + parts = PurePosixPath(relative_path).parts + if any(part.startswith(".") for part in parts): + scan.skipped.append(DocumentFileIssue( + relative_path, "hidden/configuration path skipped", + )) + continue + portable_path = relative_path.casefold() + if portable_path in seen: + scan.rejected.append(DocumentFileIssue(relative_path, "duplicate upload path")) + continue + seen.add(portable_path) + if document_format_for_path(relative_path) is None: + scan.skipped.append(DocumentFileIssue(relative_path, "unsupported document format")) + continue + if _sensitive_filename(parts[-1]): + scan.rejected.append(DocumentFileIssue(relative_path, "sensitive filename")) + continue + if not isinstance(raw, bytes): + scan.rejected.append(DocumentFileIssue(relative_path, "invalid upload")) + continue + if len(raw) > MAX_DOCUMENT_BYTES: + scan.rejected.append(DocumentFileIssue( + relative_path, "document exceeds byte safety limit", + )) + continue + total += len(raw) + if total > MAX_DOCUMENT_TREE_BYTES: + scan.rejected.append(DocumentFileIssue( + relative_path, "source exceeds total byte safety limit", + )) + scan.complete = False + break + try: + scan.documents.append(parse_document( + raw, relative_path, adapter=local_document_adapter, + )) + except ValueError as exc: + scan.rejected.append(DocumentFileIssue(relative_path, _safe_reason(exc))) + return scan + + +def _bounded_source_metadata(value: Any) -> tuple[dict[str, Any], int]: + """Return a small JSON-safe format metadata object with deterministic bounds.""" + if not isinstance(value, dict): + return {}, 0 + result: dict[str, Any] = {} + for raw_key, raw_value in list(value.items())[:64]: + key = str(raw_key)[:80] + if isinstance(raw_value, (str, int, float, bool)) or raw_value is None: + result[key] = raw_value if not isinstance(raw_value, str) else raw_value[:1000] + elif isinstance(raw_value, (list, tuple)): + result[key] = [str(item)[:300] for item in list(raw_value)[:100]] + elif isinstance(raw_value, dict): + result[key] = { + str(child_key)[:80]: str(child_value)[:300] + for child_key, child_value in list(raw_value.items())[:32] + } + while result and len( + json.dumps(result, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ) > 4_000: + result.pop(next(reversed(result))) + return result, max(0, len(value) - len(result)) + + +class DocumentImporter(ObsidianImporter): + """Source-neutral temporal importer for mixed local document collections.""" + + SOURCE_KIND = "documents" + JOB_KIND = "document_import" + RECEIPT_OPERATION = "document_import" + CLAIM_KIND = "source_document" + SUBJECT_PREFIX = "document" + METADATA_KEY = "document" + DEFAULT_LABEL = "Document collection" + IMPORTER_VERSION = IMPORTER_VERSION + COUNT_KEY = "documents" + LINK_REASON = "document_reference" + LINK_IMPORTED_ATTACHMENTS = True + + def preview( + self, scan: DocumentScan, *, source_id: Optional[str] = None, + source_label: str = "", **kwargs: Any, + ) -> dict: + vault_id = kwargs.pop("vault_id", None) + vault_label = kwargs.pop("vault_label", "") + return super().preview( + scan, vault_id=source_id or vault_id, + vault_label=source_label or vault_label, **kwargs, + ) + + def import_scan( + self, scan: DocumentScan, *, source_id: Optional[str] = None, + source_label: str = "", **kwargs: Any, + ) -> dict: + vault_id = kwargs.pop("vault_id", None) + vault_label = kwargs.pop("vault_label", "") + return super().import_scan( + scan, vault_id=source_id or vault_id, + vault_label=source_label or vault_label, **kwargs, + ) + + def prepare_import( + self, scan: DocumentScan, *, source_id: Optional[str] = None, + source_label: str = "", **kwargs: Any, + ) -> dict: + vault_id = kwargs.pop("vault_id", None) + vault_label = kwargs.pop("vault_label", "") + return super().prepare_import( + scan, vault_id=source_id or vault_id, + vault_label=source_label or vault_label, **kwargs, + ) + + @classmethod + def _metadata( + cls, note: DocumentRecord, *, vault_id: str, source_id: str, + imported_at: float, actor: str, branch: str, + ) -> dict: + envelope = super()._metadata( + note, vault_id=vault_id, source_id=source_id, + imported_at=imported_at, actor=actor, branch=branch, + ) + document = envelope[cls.METADATA_KEY] + document["format"] = str(note.format)[:64] + document["media_type"] = str(note.media_type)[:200] + format_metadata, pre_omitted = _bounded_source_metadata(note.metadata) + original_format_keys = len(format_metadata) + document["format_metadata"] = format_metadata + document["original_title"] = str(note.title)[:1000] + # The base envelope is already bounded near the Store ceiling. Format + # adapters can add another 4 KiB, so trim their least-significant tail + # against the complete envelope rather than allowing a valid document to + # fail only when the canonical memory is written. + while format_metadata and len( + json.dumps(envelope, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ) > 14_000: + format_metadata.pop(next(reversed(format_metadata))) + omitted = pre_omitted + original_format_keys - len(format_metadata) + if omitted: + counts = document.setdefault("omitted_counts", {}) + counts["format_metadata"] = int(counts.get("format_metadata", 0)) + omitted + return envelope + + +__all__ = [ + "DocumentImportCancelled", "DocumentImporter", "local_document_adapter", + "scan_document_upload", +] diff --git a/engraphis/llm/client.py b/engraphis/llm/client.py index 4ba8a282..738335ed 100644 --- a/engraphis/llm/client.py +++ b/engraphis/llm/client.py @@ -15,7 +15,10 @@ from typing import Any, Optional from urllib.parse import urlsplit, urlunsplit -import httpx +try: + import httpx +except ImportError: # pragma: no cover - core-floor (numpy-only) installs + httpx = None # type: ignore[assignment] from engraphis.config import settings diff --git a/engraphis/mcp_server.py b/engraphis/mcp_server.py index 69afde6e..cecee651 100644 --- a/engraphis/mcp_server.py +++ b/engraphis/mcp_server.py @@ -34,7 +34,12 @@ from dataclasses import dataclass from typing import Any, Annotated, Callable, List, Optional -from pydantic import Field, StrictBool, StrictInt +try: + from pydantic import Field, StrictBool, StrictInt +except ImportError: # pragma: no cover - core-floor (numpy-only) installs + Field = None # type: ignore[assignment,misc] + StrictBool = None # type: ignore[assignment,misc] + StrictInt = None # type: ignore[assignment,misc] try: from mcp.server.fastmcp import FastMCP @@ -2659,71 +2664,74 @@ def engraphis_get_memory( record = service().inspect(memory_id=memory_id, workspace=workspace, repo=repo) except Exception as exc: # noqa: BLE001 — Smart gateway classification return _classify_gateway_exception(exc) - mem = record.get("memory") or {} - if not mem.get("id"): - return _gateway_error("memory_not_found") - svc = service() - target = svc.store.get_memory(mem["id"]) - if target is None: - return _gateway_error("memory_not_found") - provenance = target.provenance - metadata = target.metadata - if not prompt_eligible(provenance, metadata): - return _gateway_error("memory_not_prompt_eligible") - # ``inspect`` serializes the governed record, but the store object is the - # authoritative source for fields that must not be lost in projection. - confidence = mem.get("confidence") - if confidence is None: - confidence = target.confidence - # ``inspect`` authorizes the target against the requested scope, while related - # records are intentionally returned as a bounded projection. Keep the same - # hierarchy for that projection: an explicit repo request includes that repo and - # workspace-level records, whereas omitting repo retains the workspace-wide behavior. - requested_repo_id = None - if repo: - try: - _, requested_repo_id = svc._require_scope(workspace, repo) - except Exception as exc: # noqa: BLE001 — inspect already validated the request - return _classify_gateway_exception(exc) - safe_links = [] - for link in svc.store.get_links(mem["id"]): - other_id = ( - link.get("b") if link.get("a") == mem["id"] else link.get("a") - ) - other = svc.store.get_memory(other_id) if other_id else None - if (other is None or other.workspace_id != target.workspace_id - or not prompt_eligible(other.provenance, other.metadata) - or not svc._memory_visible_to_caller(other)): - continue - if (requested_repo_id is not None - and other.repo_id not in (None, requested_repo_id)): - continue - safe_links.append({ - "id": other.id, - "relation": link.get("relation") or "related", - "layer": link.get("layer") or "semantic", - "reason": link.get("reason") or "", - "title": other.title or other.content[:80], - "live": bool(other.expired_at is None and other.valid_to is None), + try: + mem = record.get("memory") or {} + if not mem.get("id"): + return _gateway_error("memory_not_found") + svc = service() + target = svc.store.get_memory(mem["id"]) + if target is None: + return _gateway_error("memory_not_found") + provenance = target.provenance + metadata = target.metadata + if not prompt_eligible(provenance, metadata): + return _gateway_error("memory_not_prompt_eligible") + # ``inspect`` serializes the governed record, but the store object is the + # authoritative source for fields that must not be lost in projection. + confidence = mem.get("confidence") + if confidence is None: + confidence = target.confidence + # ``inspect`` authorizes the target against the requested scope, while related + # records are intentionally returned as a bounded projection. Keep the same + # hierarchy for that projection: an explicit repo request includes that repo and + # workspace-level records, whereas omitting repo retains the workspace-wide behavior. + requested_repo_id = None + if repo: + try: + _, requested_repo_id = svc._require_scope(workspace, repo) + except Exception as exc: # noqa: BLE001 — inspect already validated the request + return _classify_gateway_exception(exc) + safe_links = [] + for link in svc.store.get_links(mem["id"]): + other_id = ( + link.get("b") if link.get("a") == mem["id"] else link.get("a") + ) + other = svc.store.get_memory(other_id) if other_id else None + if (other is None or other.workspace_id != target.workspace_id + or not prompt_eligible(other.provenance, other.metadata) + or not svc._memory_visible_to_caller(other)): + continue + if (requested_repo_id is not None + and other.repo_id not in (None, requested_repo_id)): + continue + safe_links.append({ + "id": other.id, + "relation": link.get("relation") or "related", + "layer": link.get("layer") or "semantic", + "reason": link.get("reason") or "", + "title": other.title or other.content[:80], + "live": bool(other.expired_at is None and other.valid_to is None), + }) + safe_chain = [] + for entry in record.get("chain") or []: + other = svc.store.get_memory(entry.get("id")) if entry.get("id") else None + if (other is not None and other.workspace_id == target.workspace_id + and prompt_eligible(other.provenance, other.metadata) + and svc._memory_visible_to_caller(other) + and (requested_repo_id is None + or other.repo_id in (None, requested_repo_id))): + safe_chain.append(entry) + return _ok({ + "id": mem.get("id"), "content": mem.get("content"), "title": mem.get("title"), + "mtype": mem.get("mtype"), "scope": mem.get("scope"), + "importance": mem.get("importance"), "confidence": confidence, + "valid_from": mem.get("valid_from"), "valid_to": mem.get("valid_to"), + "ingested_at": mem.get("ingested_at"), + "provenance": {k: provenance.get(k) for k in ("source", "trusted", "review_state")}, + "links": safe_links, "chain": safe_chain, }) - safe_chain = [] - for entry in record.get("chain") or []: - other = svc.store.get_memory(entry.get("id")) if entry.get("id") else None - if (other is not None and other.workspace_id == target.workspace_id - and prompt_eligible(other.provenance, other.metadata) - and svc._memory_visible_to_caller(other) - and (requested_repo_id is None - or other.repo_id in (None, requested_repo_id))): - safe_chain.append(entry) - return _ok({ - "id": mem.get("id"), "content": mem.get("content"), "title": mem.get("title"), - "mtype": mem.get("mtype"), "scope": mem.get("scope"), - "importance": mem.get("importance"), "confidence": confidence, - "valid_from": mem.get("valid_from"), "valid_to": mem.get("valid_to"), - "ingested_at": mem.get("ingested_at"), - "provenance": {k: provenance.get(k) for k in ("source", "trusted", "review_state")}, - "links": safe_links, "chain": safe_chain, - }) + except Exception as exc: # noqa: BLE001 — Smart gateway classification + return _classify_gateway_exception(exc) @smart_mcp.tool( diff --git a/engraphis/obsidian_import.py b/engraphis/obsidian_import.py new file mode 100644 index 00000000..a006ece7 --- /dev/null +++ b/engraphis/obsidian_import.py @@ -0,0 +1,1465 @@ +"""Production Obsidian import orchestration for the v2 memory engine. + +The dependency-free parser lives in :mod:`engraphis.core.obsidian`. This outer +module owns persistence and deliberately receives an already-composed +``MemoryService`` so no concrete backend crosses into ``core``. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import PurePosixPath +import posixpath +import re +import time +from typing import Any, Callable, Iterable, Optional, Protocol, Sequence +import unicodedata +from urllib.parse import unquote, urlsplit + +from engraphis.core.ids import new_id +from engraphis.core.interfaces import GraphLayer, MemoryType, Scope +from engraphis.core.obsidian import ( + IMPORTER_VERSION, + MAX_NOTE_BYTES, + MAX_VAULT_BYTES, + MAX_VAULT_FILES, + ObsidianFileIssue, + ObsidianVaultScan, + normalize_obsidian_path, + parse_obsidian_note, +) + + +_SAFE_ERROR = "note import failed" +_CONFLICT_POLICIES = {"error", "replace", "new"} +_ACTIVE_ITEM_STATES = {"imported", "unchanged", "renamed", "skipped"} +_SENSITIVE_NAMES = { + ".env", "credentials", "credentials.json", "id_dsa", "id_rsa", + "id_ecdsa", "id_ed25519", "authorized_keys", "known_hosts", + "recovery-codes", "recovery_codes", "secret", "secret.json", + "secrets", "secrets.json", "token", "tokens", +} + + +class _ImportLink(Protocol): + """The source-neutral link shape consumed by the import planner.""" + + @property + def target(self) -> str: ... + + @property + def display_text(self) -> Optional[str]: ... + + @property + def heading(self) -> Optional[str]: ... + + @property + def block_id(self) -> Optional[str]: ... + + @property + def embedded(self) -> bool: ... + + +class _ImportAttachment(Protocol): + """The source-neutral attachment shape consumed by the import planner.""" + + @property + def path(self) -> str: ... + + +class _ImportNote(Protocol): + """Readable source record accepted by the temporal import planner. + + Both ``ObsidianNote`` and the universal ``DocumentRecord`` deliberately + implement this narrow, read-only shape. Keeping it structural prevents the + document adapter from inheriting an Obsidian-only type contract while keeping + the runtime planner entirely source-neutral. + """ + + @property + def relative_path(self) -> str: ... + + @property + def title(self) -> str: ... + + @property + def body(self) -> str: ... + + @property + def raw_sha256(self) -> str: ... + + @property + def canonical_sha256(self) -> str: ... + + @property + def source_size(self) -> int: ... + + @property + def source_mtime_ns(self) -> Optional[int]: ... + + @property + def title_source(self) -> str: ... + + @property + def aliases(self) -> Sequence[str]: ... + + @property + def tags(self) -> Sequence[str]: ... + + @property + def dates(self) -> dict[str, str]: ... + + @property + def headings(self) -> Sequence[str]: ... + + @property + def links(self) -> Sequence[_ImportLink]: ... + + @property + def attachments(self) -> Sequence[_ImportAttachment]: ... + + @property + def warnings(self) -> Sequence[str]: ... + + +class _ImportIssue(Protocol): + @property + def relative_path(self) -> str: ... + + @property + def reason(self) -> str: ... + + +class _ImportScan(Protocol): + """Minimal source collection shape needed for plan/report execution.""" + + @property + def vault_id(self) -> str: ... + + @property + def notes(self) -> Sequence[_ImportNote]: ... + + @property + def rejected(self) -> Sequence[_ImportIssue]: ... + + @property + def skipped(self) -> Sequence[_ImportIssue]: ... + + @property + def complete(self) -> bool: ... + + +class ObsidianImportCancelled(Exception): + """Raised at a note boundary after a caller requests cancellation.""" + + +@dataclass +class _Plan: + note: _ImportNote + action: str + item: Optional[dict] = None + reason: str = "" + + +def scan_obsidian_upload( + files: Iterable[tuple[str, bytes]], *, vault_label: str, +) -> ObsidianVaultScan: + """Parse browser-selected Markdown bytes without persisting an upload copy.""" + label = unicodedata.normalize("NFC", str(vault_label or "").strip()[:200]) + if not label: + raise ValueError("vault_label is required for a browser source") + root_digest = hashlib.sha256( + ("obsidian-browser\0" + label.casefold()).encode("utf-8", "surrogatepass") + ).hexdigest() + scan = ObsidianVaultScan(vault_path="", vault_id=root_digest) + total = 0 + seen: set[str] = set() + for index, (raw_path, raw) in enumerate(files): + if index >= MAX_VAULT_FILES: + scan.rejected.append(ObsidianFileIssue( + "(vault)", "vault exceeds Markdown file safety limit", + )) + scan.complete = False + break + try: + relative_path = normalize_obsidian_path(raw_path) + except ValueError: + scan.rejected.append(ObsidianFileIssue("(invalid path)", "invalid source path")) + continue + parts = PurePosixPath(relative_path).parts + if any(part.startswith(".") for part in parts): + scan.skipped.append(ObsidianFileIssue(relative_path, "hidden/configuration path skipped")) + continue + portable_path = relative_path.casefold() + if portable_path in seen: + scan.rejected.append(ObsidianFileIssue(relative_path, "duplicate upload path")) + continue + seen.add(portable_path) + if not relative_path.casefold().endswith(".md"): + scan.skipped.append(ObsidianFileIssue(relative_path, "non-Markdown file skipped")) + continue + name = parts[-1].casefold() + if _sensitive_name(name): + scan.rejected.append(ObsidianFileIssue(relative_path, "sensitive filename")) + continue + if not isinstance(raw, bytes): + scan.rejected.append(ObsidianFileIssue(relative_path, "invalid upload")) + continue + if len(raw) > MAX_NOTE_BYTES: + scan.rejected.append(ObsidianFileIssue(relative_path, "note exceeds byte safety limit")) + continue + total += len(raw) + if total > MAX_VAULT_BYTES: + scan.rejected.append(ObsidianFileIssue( + relative_path, "vault exceeds total byte safety limit", + )) + scan.complete = False + break + try: + scan.notes.append(parse_obsidian_note(raw, relative_path)) + except ValueError as exc: + scan.rejected.append(ObsidianFileIssue(relative_path, _safe_parse_reason(exc))) + return scan + + +def _sensitive_name(name: str) -> bool: + lowered = name.casefold() + return ( + lowered in _SENSITIVE_NAMES + or lowered.startswith(".env.") + or lowered.endswith((".key", ".p12", ".pem", ".pfx")) + or bool(re.search(r"(?:credential|recovery[-_ ]?code|secret|token)", lowered)) + ) + + +def _safe_parse_reason(exc: BaseException) -> str: + text = str(exc) + allowed = ( + "secret", "character safety limit", "byte safety limit", "invalid source path", + ) + return next((f"source rejected: {label}" for label in allowed if label in text), "source rejected") + + +def stable_source_key(vault_id: str, relative_path: str, *, branch: str = "") -> str: + material = f"{vault_id}\0{normalize_obsidian_path(relative_path)}\0{branch}" + return hashlib.sha256(material.encode("utf-8", "surrogatepass")).hexdigest() + + +class ObsidianImporter: + """Plan and execute repeatable imports through one injected v2 service.""" + + SOURCE_KIND = "obsidian" + JOB_KIND = "obsidian_import" + RECEIPT_OPERATION = "obsidian_import" + CLAIM_KIND = "obsidian_note" + SUBJECT_PREFIX = "obsidian" + METADATA_KEY = "obsidian" + DEFAULT_LABEL = "Obsidian vault" + IMPORTER_VERSION = IMPORTER_VERSION + COUNT_KEY = "markdown" + LINK_REASON = "obsidian_wikilink" + LINK_IMPORTED_ATTACHMENTS = False + + def __init__(self, service: Any = None) -> None: + self.service = service + # ``Any`` is deliberate: preview-only CLI construction has no Store at all, + # while live construction receives the service's concrete engine/Store pair. + self.engine: Any = service.engine if service is not None else None + self.store: Any = service.store if service is not None else None + + def preview( + self, scan: _ImportScan, *, workspace_id: Optional[str], + repo_id: Optional[str], session_id: Optional[str], scope: Scope, + memory_type: MemoryType, vault_id: Optional[str] = None, + vault_label: str = "", on_conflict: str = "error", + manifest: Optional[dict] = None, strict_root: bool = True, + attachment_manifest: Optional[list[dict]] = None, + ) -> dict: + policy = self._policy(on_conflict) + vault, items = self._preview_manifest( + scan, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, vault_id=vault_id, manifest=manifest, + scope=scope, memory_type=memory_type, strict_root=strict_root, + ) + identity = str((vault or {}).get("id") or f"preview:{scan.vault_id}") + plans, missing = self._plan( + scan, identity, items, inspect_memories=manifest is None, + ) + return self._report( + plans, missing, scan, state="preview", vault_id=(vault or {}).get("id"), + workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, + scope=scope, memory_type=memory_type, policy=policy, + vault_label=vault_label, attachment_manifest=attachment_manifest, + ) + + def import_scan( + self, scan: _ImportScan, *, workspace_id: str, + repo_id: Optional[str], session_id: Optional[str], scope: Scope, + memory_type: MemoryType, vault_id: Optional[str] = None, + vault_label: str = "", on_conflict: str = "error", + confirmed: bool = False, actor: str = "local_cli_operator", + strict_root: bool = True, attachment_manifest: Optional[list[dict]] = None, + cancel_check: Optional[Callable[[], bool]] = None, + progress: Optional[Callable[[dict], None]] = None, + prepared: Optional[dict] = None, + ) -> dict: + if confirmed is not True: + raise ValueError("trusted-local confirmation is required") + policy = self._policy(on_conflict) + prepared = prepared or self.prepare_import( + scan, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, scope=scope, memory_type=memory_type, + vault_id=vault_id, vault_label=vault_label, on_conflict=policy, + confirmed=True, strict_root=strict_root, + ) + vault_id = str(prepared["vault_id"]) + run_started = time.time() + job_id = str(prepared["job_id"]) + import_id = str(prepared["import_id"]) + items = self.store.list_source_import_items(vault_id=vault_id) + plans, missing = self._plan(scan, vault_id, items, inspect_memories=True) + for plan in plans: + self.store.record_source_import_job_item( + job_id=job_id, source_id=(plan.item or {}).get("id"), + relative_path=plan.note.relative_path, + planned_action=plan.action, result_state="pending", + warning_count=len(plan.note.warnings), + source_format=str(getattr(plan.note, "format", "markdown"))[:64], + ) + for issue in scan.rejected: + self.store.record_source_import_job_item( + job_id=job_id, relative_path=issue.relative_path, + planned_action="rejected", result_state="rejected", + error_code="source_rejected", + ) + for issue in scan.skipped: + self.store.record_source_import_job_item( + job_id=job_id, relative_path=issue.relative_path, + planned_action="skipped", result_state="skipped", + ) + for item in missing: + self.store.record_source_import_job_item( + job_id=job_id, source_id=item.get("id"), + relative_path=str(item.get("relative_path") or "(missing)"), + planned_action="missing", result_state="pending", + ) + outcomes: list[dict] = [] + finalized_missing: list[dict] = [] + pending_missing = list(missing) + unreadable_directories = self._unreadable_directories(scan) + can_finalize_missing = scan.complete and not unreadable_directories + terminal_state = "completed" + try: + for index, plan in enumerate(plans, 1): + self._check_cancel(job_id, cancel_check) + outcome = self._apply_plan( + plan, vault_id=vault_id, import_id=import_id, + workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, scope=scope, memory_type=memory_type, + policy=policy, actor=actor, + ) + outcomes.append(outcome) + self._update_job_progress(job_id, index, outcomes) + if progress is not None: + progress(dict(outcome)) + self._check_cancel(job_id, cancel_check) + if can_finalize_missing: + self.store.mark_source_import_items_missing( + vault_id=vault_id, seen_before=run_started, + preserve_paths=self._rejected_paths(scan), + ) + for item in missing: + self.store.record_source_import_job_item( + job_id=job_id, source_id=item.get("id"), + relative_path=str(item.get("relative_path") or "(missing)"), + planned_action="missing", result_state="missing", + ) + finalized_missing = missing + pending_missing = [] + # Link reconciliation is safe only for a complete view of the source. + # An incomplete scan must not retire a valid edge merely because its target + # was hidden by a transient filesystem or scan-budget failure. + link_warnings: list[dict] = [] + if can_finalize_missing: + link_warnings = self._reconcile_links( + scan, vault_id=vault_id, job_id=job_id, cancel_check=cancel_check, + ) + self._persist_link_warnings(job_id, link_warnings) + outcomes.extend(link_warnings) + if scan.rejected or not scan.complete or unreadable_directories or any( + row["status"] in {"error", "conflict", "rejected"} for row in outcomes + ): + terminal_state = "partial" + except (KeyboardInterrupt, ObsidianImportCancelled): + terminal_state = "cancelled" + except Exception: + terminal_state = "failed" + report = self._final_report( + plans, outcomes, finalized_missing, scan, state=terminal_state, + pending_missing=pending_missing, + vault_id=vault_id, job_id=job_id, import_id=import_id, + workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, + scope=scope, memory_type=memory_type, policy=policy, + vault_label=vault_label, attachment_manifest=attachment_manifest, + ) + if terminal_state == "completed" and report["counts"].get("conflict", 0): + terminal_state = "partial" + report["state"] = terminal_state + self._finish_job(job_id, terminal_state, report) + self._record_receipt( + report, workspace_id=workspace_id, repo_id=repo_id, actor=actor, + ) + return report + + def prepare_import( + self, scan: _ImportScan, *, workspace_id: str, + repo_id: Optional[str], session_id: Optional[str], scope: Scope, + memory_type: MemoryType, vault_id: Optional[str] = None, + vault_label: str = "", on_conflict: str = "error", + confirmed: bool = False, strict_root: bool = True, + ) -> dict: + """Persist only the run header so a dashboard worker can start asynchronously.""" + if confirmed is not True: + raise ValueError("trusted-local confirmation is required") + policy = self._policy(on_conflict) + vault = self._resolve_or_register_vault( + scan, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, scope=scope, memory_type=memory_type, + vault_id=vault_id, label=vault_label, strict_root=strict_root, + ) + selected_vault_id = str(vault["id"]) + job_id = self._create_job( + workspace_id, repo_id, session_id=session_id, + total=len(scan.notes) + len(scan.rejected) + len(scan.skipped), + policy=policy, scope=scope, memory_type=memory_type, + ) + return { + "vault_id": selected_vault_id, "job_id": job_id, + "import_id": job_id, + } + + @staticmethod + def _policy(value: str) -> str: + policy = str(value or "error").strip().casefold() + policy = {"report": "error", "supersede": "replace"}.get(policy, policy) + if policy not in _CONFLICT_POLICIES: + raise ValueError("on_conflict must be error, replace, or new") + return policy + + def _preview_manifest( + self, scan: _ImportScan, *, workspace_id: Optional[str], + repo_id: Optional[str], session_id: Optional[str], vault_id: Optional[str], + scope: Scope, memory_type: MemoryType, manifest: Optional[dict], + strict_root: bool, + ) -> tuple[Optional[dict], list[dict]]: + vaults = ( + list((manifest or {}).get("vaults") or []) + if manifest is not None else self.store.list_source_vaults(kind=self.SOURCE_KIND) + ) + items = ( + list((manifest or {}).get("items") or []) + if manifest is not None else [] + ) + vault: Optional[dict] = None + if vault_id: + vault = ( + self.store.get_source_vault(vault_id) + if manifest is None else + next((row for row in vaults if row.get("id") == vault_id), None) + ) + if vault is None: + raise ValueError("registered vault was not found") + else: + matches = [ + row for row in vaults + if row.get("kind") == self.SOURCE_KIND + and row.get("root_digest") == scan.vault_id + and (workspace_id is None or row.get("workspace_id") == workspace_id) + and row.get("repo_id") == repo_id + and row.get("session_id") == session_id + ] + vault = matches[0] if len(matches) == 1 else None + if vault is not None: + self._validate_vault_target( + vault, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, root_digest=scan.vault_id, + scope=scope, memory_type=memory_type, strict_root=strict_root, + ) + if manifest is None: + items = self.store.list_source_import_items(vault_id=str(vault["id"])) + else: + items = [row for row in items if row.get("vault_id") == vault.get("id")] + else: + items = [] + return vault, items + + def _resolve_or_register_vault( + self, scan: _ImportScan, *, workspace_id: str, + repo_id: Optional[str], session_id: Optional[str], scope: Scope, + memory_type: MemoryType, vault_id: Optional[str], label: str, + strict_root: bool, + ) -> dict: + if vault_id: + vault = self.store.get_source_vault(vault_id) + if vault is None: + raise ValueError("registered vault was not found") + self._validate_vault_target( + vault, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, root_digest=scan.vault_id, + scope=scope, memory_type=memory_type, strict_root=strict_root, + ) + return vault + vault_id = self.store.register_source_vault( + kind=self.SOURCE_KIND, root_digest=scan.vault_id, + workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, + display_name=str(label or self.DEFAULT_LABEL)[:200], scope=scope.value, + memory_type=memory_type.value, importer_version=self.IMPORTER_VERSION, + ) + vault = self.store.get_source_vault(vault_id) + if vault is None: + raise RuntimeError("registered vault identity was not persisted") + return vault + + @classmethod + def _validate_vault_target( + cls, vault: dict, *, workspace_id: Optional[str], repo_id: Optional[str], + session_id: Optional[str], root_digest: str, scope: Scope, + memory_type: MemoryType, strict_root: bool, + ) -> None: + if vault.get("kind") != cls.SOURCE_KIND: + raise ValueError("registered source uses a different import adapter") + if workspace_id is not None and vault.get("workspace_id") != workspace_id: + raise ValueError("registered vault belongs to another workspace") + if vault.get("repo_id") != repo_id or vault.get("session_id") != session_id: + raise ValueError("registered vault has a different target scope") + if vault.get("scope") != scope.value or vault.get("memory_type") != memory_type.value: + raise ValueError("registered vault has different import defaults") + if strict_root and vault.get("root_digest") != root_digest: + raise ValueError("selected path does not match the registered vault") + + def _plan( + self, scan: _ImportScan, vault_identity: str, items: list[dict], *, + inspect_memories: bool, + ) -> tuple[list[_Plan], list[dict]]: + scan_paths = { + note.relative_path for note in scan.notes + } | self._rejected_paths(scan) + unreadable_directories = self._unreadable_directories(scan) + by_path: dict[str, list[dict]] = {} + for item in items: + relative_path = str(item.get("relative_path") or "") + by_path.setdefault(relative_path, []).append(item) + if self._under_directory(relative_path, unreadable_directories): + scan_paths.add(relative_path) + # Exact-content rename detection stays conservative but must remain + # linear for a full 10k-file source. Index eligible historical paths once. + renames_by_hash: dict[str, list[dict]] = {} + for item in items: + relative_path = str(item.get("relative_path") or "") + content_hash = str(item.get("content_sha256") or "") + if ( + relative_path not in scan_paths + and content_hash + and item.get("state") != "conflict" + ): + renames_by_hash.setdefault(content_hash, []).append(item) + used: set[str] = set() + plans: list[_Plan] = [] + for note in sorted(scan.notes, key=lambda entry: entry.relative_path.casefold()): + candidates = list(by_path.get(note.relative_path, [])) + exact = [ + row for row in candidates + if row.get("content_sha256") == note.raw_sha256 + and row.get("importer_version") == self.IMPORTER_VERSION + and row.get("state") != "conflict" + ] + selected = self._newest(exact) if exact else self._newest([ + row for row in candidates if row.get("state") in _ACTIVE_ITEM_STATES + ]) + if selected is None and candidates: + selected = self._newest(candidates) + if selected is not None: + used.add(str(selected.get("source_key"))) + if exact: + action = "skipped" + reason = "unchanged" + else: + action = "updated" + reason = "source_changed" + if inspect_memories and not self._manifest_memory_is_current(selected): + action, reason = "conflict", "memory_lineage_diverged" + plans.append(_Plan(note, action, selected, reason)) + continue + rename_candidates = [ + row for row in renames_by_hash.get(note.raw_sha256, ()) + if row.get("source_key") not in used + ] + if len(rename_candidates) == 1: + selected = rename_candidates[0] + used.add(str(selected.get("source_key"))) + action, reason = "renamed", "unique_content_path_move" + if inspect_memories and not self._manifest_memory_is_current(selected): + action, reason = "conflict", "memory_lineage_diverged" + plans.append(_Plan(note, action, selected, reason)) + elif len(rename_candidates) > 1: + plans.append(_Plan(note, "conflict", None, "ambiguous_rename")) + else: + plans.append(_Plan(note, "imported", None, "new_source")) + missing = [ + row for row in items + if row.get("relative_path") not in scan_paths + and row.get("source_key") not in used + and row.get("state") not in {"missing", "conflict"} + ] + return plans, missing + + @staticmethod + def _rejected_paths(scan: _ImportScan) -> set[str]: + """Keep durable rows for files seen but rejected by the parser.""" + return {str(issue.relative_path) for issue in scan.rejected} + + @staticmethod + def _unreadable_directories(scan: _ImportScan) -> set[str]: + return { + str(issue.relative_path).rstrip("/") + for issue in scan.skipped + if str(issue.reason) == "unreadable directory" + } + + @staticmethod + def _under_directory(relative_path: str, directories: set[str]) -> bool: + return any( + relative_path == directory or relative_path.startswith(directory + "/") + for directory in directories + ) + + @staticmethod + def _newest(items: list[dict]) -> Optional[dict]: + return max( + items, + key=lambda row: (float(row.get("last_seen_at") or 0), str(row.get("id") or "")), + default=None, + ) + + def _manifest_memory_is_current(self, item: dict) -> bool: + memory_id = str(item.get("memory_id") or "") + subject_key = str(item.get("subject_key") or "") + if not memory_id: + return False + rec = self.store.get_memory(memory_id) + if rec is None or rec.valid_to is not None or rec.expired_at is not None: + return False + live_id = self._live_subject_memory(subject_key) + if live_id and live_id != memory_id: + return False + source_metadata = ( + rec.metadata.get(self.METADATA_KEY) if isinstance(rec.metadata, dict) else None + ) + return ( + isinstance(source_metadata, dict) + and source_metadata.get("raw_sha256") == item.get("content_sha256") + and source_metadata.get("source_id") == item.get("id") + ) + + def _live_subject_memory(self, subject_key: str) -> Optional[str]: + live = self._live_subject_memories(subject_key) + return live[0] if live else None + + def _live_subject_memories(self, subject_key: str) -> list[str]: + """Return every currently-live revision for a source subject.""" + if not subject_key: + return [] + rows = self.store.conn.execute( + "SELECT id FROM memories WHERE subject_key=? AND valid_to IS NULL " + "AND expired_at IS NULL ORDER BY valid_from DESC, ingested_at DESC, id DESC", + (subject_key,), + ).fetchall() + return [str(row["id"]) for row in rows] + + def _apply_plan( + self, plan: _Plan, *, vault_id: str, import_id: str, + workspace_id: str, repo_id: Optional[str], session_id: Optional[str], + scope: Scope, memory_type: MemoryType, policy: str, actor: str, + ) -> dict: + note = plan.note + if plan.action == "skipped" and plan.item is not None: + self.store.upsert_source_import_item( + vault_id=vault_id, source_key=str(plan.item["source_key"]), + source_id=str(plan.item["id"]), relative_path=note.relative_path, + memory_id=plan.item.get("memory_id"), + subject_key=str(plan.item.get("subject_key") or ""), + content_sha256=note.raw_sha256, canonical_sha256=note.canonical_sha256, + file_size=note.source_size, + file_mtime_ns=note.source_mtime_ns, + importer_version=self.IMPORTER_VERSION, + state="unchanged", import_id=import_id, + ) + self.store.record_source_import_job_item( + job_id=import_id, source_id=str(plan.item["id"]), + relative_path=note.relative_path, planned_action="skipped", + result_state="skipped", warning_count=len(note.warnings), + source_format=str(getattr(note, "format", "markdown"))[:64], + ) + return self._outcome(note, "skipped", "unchanged") + if plan.action == "conflict" and policy == "error": + if plan.item is not None: + self.store.upsert_source_import_item( + vault_id=vault_id, source_key=str(plan.item["source_key"]), + source_id=str(plan.item["id"]), relative_path=note.relative_path, + memory_id=plan.item.get("memory_id"), + subject_key=str(plan.item.get("subject_key") or ""), + content_sha256=str(plan.item.get("content_sha256") or ""), + canonical_sha256=str(plan.item.get("canonical_sha256") or ""), + file_size=int(plan.item.get("file_size") or 0), + file_mtime_ns=plan.item.get("file_mtime_ns"), + importer_version=str(plan.item.get("importer_version") or ""), + state="conflict", import_id=import_id, + ) + self.store.record_source_import_job_item( + job_id=import_id, source_id=(plan.item or {}).get("id"), + relative_path=note.relative_path, planned_action="conflict", + result_state="conflict", warning_count=len(note.warnings), + source_format=str(getattr(note, "format", "markdown"))[:64], + ) + return self._outcome(note, "conflict", plan.reason) + + old_item = plan.item + old_memory_id = None + if old_item is not None: + old_memory_id = self._live_subject_memory(str(old_item.get("subject_key") or "")) + old_memory_id = old_memory_id or str(old_item.get("memory_id") or "") or None + branch = "" + source_id = str(old_item.get("id")) if old_item is not None else new_id("source") + source_key = ( + str(old_item.get("source_key")) + if old_item is not None else stable_source_key(vault_id, note.relative_path) + ) + if plan.action == "conflict" and policy == "new": + branch = f"branch:{note.raw_sha256}" + source_id = new_id("source") + source_key = stable_source_key(vault_id, note.relative_path, branch=branch) + old_memory_id = None + subject_key = f"{self.SUBJECT_PREFIX}:{source_id}" + imported_at = self._revision_time(old_memory_id) + metadata = self._metadata( + note, vault_id=vault_id, source_id=source_id, imported_at=imported_at, + actor=actor, branch=branch, + ) + state = "renamed" if plan.action == "renamed" else "imported" + + def finalize(memory_id: str) -> None: + # The plan was prepared outside the engine's write transaction. Re-read + # the subject here so a concurrent importer cannot leave two live revisions. + predecessor_ids = { + candidate for candidate in ( + old_memory_id, *self._live_subject_memories(subject_key) + ) if candidate and candidate != memory_id + } + successor = self.store.get_memory(memory_id) + successor_at = ( + successor.valid_from + if successor is not None and successor.valid_from is not None + else imported_at + ) + predecessors = [] + for predecessor_id in sorted(predecessor_ids): + old = self.store.get_memory(predecessor_id) + if old is not None and old.valid_to is None: + if old.valid_from is not None: + successor_at = max(successor_at, old.valid_from + 0.000001) + predecessors.append(old) + # Two processes can compute valid_from before either acquires the write + # transaction. Move a skewed successor forward so its interval never + # overlaps the predecessor it is closing. + if successor is not None and successor.valid_from != successor_at: + self.store.conn.execute( + "UPDATE memories SET valid_from=? WHERE id=?", + (successor_at, memory_id), + ) + for old in predecessors: + close_at = successor_at + if old.valid_from is not None: + close_at = max(close_at, old.valid_from + 0.000001) + self.store.close_validity( + old.id, at=close_at, + actor=f"{self.SOURCE_KIND}_importer", + reason=f"{self.SOURCE_KIND}_source_revision", commit=False, + ) + self.store.retire_memory_graph_state( + old.id, at=close_at, commit=False, + ) + if plan.action == "conflict" and policy == "new" and old_item is not None: + self.store.upsert_source_import_item( + vault_id=vault_id, source_key=str(old_item["source_key"]), + source_id=str(old_item["id"]), relative_path=note.relative_path, + memory_id=old_item.get("memory_id"), + subject_key=str(old_item.get("subject_key") or ""), + content_sha256=str(old_item.get("content_sha256") or ""), + canonical_sha256=str(old_item.get("canonical_sha256") or ""), + file_size=int(old_item.get("file_size") or 0), + file_mtime_ns=old_item.get("file_mtime_ns"), + importer_version=str(old_item.get("importer_version") or ""), + state="conflict", import_id=import_id, commit=False, + ) + self.store.upsert_source_import_item( + vault_id=vault_id, source_key=source_key, source_id=source_id, + relative_path=note.relative_path, memory_id=memory_id, + subject_key=subject_key, content_sha256=note.raw_sha256, + canonical_sha256=note.canonical_sha256, + file_size=note.source_size, + file_mtime_ns=note.source_mtime_ns, + importer_version=self.IMPORTER_VERSION, + state=state, import_id=import_id, commit=False, + ) + result_state = plan.action + if result_state == "conflict": + result_state = "imported" if policy == "new" else "updated" + self.store.record_source_import_job_item( + job_id=import_id, source_id=source_id, + relative_path=note.relative_path, planned_action=plan.action, + result_state=result_state, warning_count=len(note.warnings), + source_format=str(getattr(note, "format", "markdown"))[:64], + commit=False, + ) + + if old_memory_id: + metadata["supersedes"] = [old_memory_id] + try: + result = self.engine.remember_with_resolution( + note.body, workspace_id=workspace_id, repo_id=repo_id, + session_id=session_id, mtype=memory_type, scope=scope, + title=note.title, keywords=self._keywords(note), metadata=metadata, + valid_from=imported_at, subject_key=subject_key, + claim_kind=self.CLAIM_KIND, resolve_conflicts=False, + _transactional_finalizer=finalize, + ) + except Exception: + if old_item is not None: + # The source was seen, but its successor could not commit. Preserve + # the last durable hashes/memory so the next run plans a retry instead + # of misclassifying the present file as deleted. + self.store.upsert_source_import_item( + vault_id=vault_id, source_key=str(old_item["source_key"]), + source_id=str(old_item["id"]), relative_path=note.relative_path, + memory_id=old_item.get("memory_id"), + subject_key=str(old_item.get("subject_key") or ""), + content_sha256=str(old_item.get("content_sha256") or ""), + canonical_sha256=str(old_item.get("canonical_sha256") or ""), + file_size=int(old_item.get("file_size") or 0), + file_mtime_ns=old_item.get("file_mtime_ns"), + importer_version=str(old_item.get("importer_version") or ""), + state="error", import_id=import_id, + last_error="note_import_failed", + ) + durable_source_id = ( + source_id if self.store.get_source_import(source_id) is not None else None + ) + self.store.record_source_import_job_item( + job_id=import_id, source_id=durable_source_id, + relative_path=note.relative_path, planned_action=plan.action, + result_state="error", warning_count=len(note.warnings), + error_code="note_import_failed", + source_format=str(getattr(note, "format", "markdown"))[:64], + ) + return self._outcome(note, "error", _SAFE_ERROR) + action = plan.action + if action == "conflict": + action = "imported" if policy == "new" else "updated" + return self._outcome(note, action, plan.reason, memory_id=str(result["id"])) + + def _revision_time(self, old_memory_id: Optional[str]) -> float: + stamp = time.time() + if old_memory_id: + old = self.store.get_memory(old_memory_id) + if old is not None and old.valid_from is not None and stamp <= old.valid_from: + stamp = old.valid_from + 0.000001 + return stamp + + @staticmethod + def _keywords(note: _ImportNote) -> list[str]: + values = [*note.tags, *note.aliases] + out: list[str] = [] + for value in values: + text = str(value).strip()[:128] + if text and text not in out: + out.append(text) + if len(out) >= 64: + break + return out + + @classmethod + def _metadata( + cls, note: _ImportNote, *, vault_id: str, source_id: str, + imported_at: float, actor: str, branch: str, + ) -> dict: + folder = str(PurePosixPath(note.relative_path).parent) + folder = "" if folder == "." else folder + links = [ + { + "target": link.target[:500], "display_text": (link.display_text or "")[:500], + "heading": (link.heading or "")[:500], "block_id": (link.block_id or "")[:200], + "embedded": bool(link.embedded), + } + for link in note.links[:256] + ] + obsidian: dict[str, Any] = { + "vault_id": vault_id, + "source_id": source_id, + "relative_path": note.relative_path, + "folder": folder, + "original_title": note.title[:1000], + "title_source": note.title_source, + "aliases": [str(value)[:200] for value in note.aliases[:64]], + "tags": [str(value)[:128] for value in note.tags[:64]], + "dates": {str(key)[:64]: str(value)[:200] for key, value in list(note.dates.items())[:16]}, + "headings": [str(value)[:240] for value in note.headings[:64]], + "links": links, + "attachments": [entry.path[:300] for entry in note.attachments[:64]], + "raw_sha256": note.raw_sha256, + "canonical_sha256": note.canonical_sha256, + "file": {"size": int(note.source_size), "mtime_ns": note.source_mtime_ns}, + "importer_version": cls.IMPORTER_VERSION, + "imported_at": imported_at, + } + if branch: + obsidian["branch"] = branch[:100] + # The Store enforces a 16 KiB metadata ceiling. Preserve the first parsed + # values deterministically and record how many were omitted rather than + # allowing a heavily linked note to fail after preview. + omitted: dict[str, int] = {} + for key in ("links", "headings", "attachments", "aliases", "tags"): + values = obsidian.get(key) + while isinstance(values, list) and len( + json.dumps(obsidian, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ) > 13_500 and values: + before = len(values) + del values[max(1, before // 2):] + omitted[key] = omitted.get(key, 0) + before - len(values) + if omitted: + obsidian["omitted_counts"] = omitted + return { + cls.METADATA_KEY: obsidian, + "provenance": { + "source": cls.SOURCE_KIND, "kind": "document_import", "trusted": True, + "review_state": "approved", "trust_origin": actor, + "ingress": cls.JOB_KIND, + }, + } + + def _reconcile_links( + self, scan: _ImportScan, *, vault_id: str, job_id: Optional[str] = None, + cancel_check: Optional[Callable[[], bool]] = None, + ) -> list[dict]: + """Resolve derived links in bounded, cancellable, replay-safe batches.""" + items = self.store.list_source_import_items( + vault_id=vault_id, + states=["imported", "unchanged", "renamed", "skipped", "missing"], + ) + memory_by_path = { + str(item["relative_path"]): str(item["memory_id"]) + for item in items if item.get("memory_id") + } + note_by_path = {note.relative_path: note for note in scan.notes} + exact: dict[str, list[str]] = {} + historical_exact: dict[str, list[str]] = {} + names: dict[str, list[str]] = {} + + def add_exact(key: str, path: str) -> None: + candidates = exact.setdefault(key, []) + if path not in candidates: + candidates.append(path) + + for path, note in note_by_path.items(): + add_exact(path.casefold(), path) + suffixless = str(PurePosixPath(path).with_suffix("")) + if suffixless != path: + add_exact(suffixless.casefold(), path) + keys = {PurePosixPath(path).stem.casefold(), note.title.casefold()} + keys.update(alias.casefold() for alias in note.aliases) + for key in keys: + names.setdefault(key, []).append(path) + for path in memory_by_path: + add_exact_path = historical_exact.setdefault(path.casefold(), []) + if path not in add_exact_path: + add_exact_path.append(path) + suffixless = str(PurePosixPath(path).with_suffix("")) + if suffixless != path: + add_exact_path = historical_exact.setdefault(suffixless.casefold(), []) + if path not in add_exact_path: + add_exact_path.append(path) + warnings: list[dict] = [] + batch_open = False + writes_in_batch = 0 + references_seen = 0 + retire_pairs: set[tuple[str, str]] = set() + desired_pairs: set[tuple[str, str]] = set() + + def check_cancel() -> None: + if job_id is not None: + self._check_cancel(job_id, cancel_check) + elif cancel_check is not None and cancel_check(): + raise ObsidianImportCancelled + + def flush() -> None: + nonlocal batch_open, writes_in_batch + if batch_open: + # This method OWNS the batch transaction it opened (``BEGIN IMMEDIATE`` + # in the add_link loop and in retire_unsupported_links). Commit it + # even when the connection reports no ownership, so the batch rows are + # never silently dropped while the run reports success. + self.store.conn.commit() + batch_open = False + writes_in_batch = 0 + + def pair_key(a: str, b: str) -> tuple[str, str]: + return (a, b) if a <= b else (b, a) + + def retire_ambiguous_links(source_id: str, target_ids: list[str]) -> None: + # Defer retirement until every note has been examined. mem_links is an + # aggregate undirected edge, so another note may still support this pair. + for target_id in target_ids: + if target_id != source_id: + retire_pairs.add(pair_key(source_id, target_id)) + + def retire_unsupported_links() -> None: + nonlocal batch_open, writes_in_batch + for a, b in sorted(retire_pairs - desired_pairs): + if not batch_open: + self.store.conn.execute("BEGIN IMMEDIATE") + batch_open = True + stamp = time.time() + cursor = self.store.conn.execute( + "UPDATE mem_links SET valid_to=?, valid_to_recorded_at=? " + "WHERE reason=? AND valid_to IS NULL AND expired_at IS NULL " + "AND ((a=? AND b=?) OR (a=? AND b=?))", + (stamp, stamp, self.LINK_REASON, a, b, b, a), + ) + writes_in_batch += max(int(cursor.rowcount), 0) + if writes_in_batch >= 128: + flush() + + try: + for source_path, note in note_by_path.items(): + check_cancel() + source_id = memory_by_path.get(source_path) + if not source_id: + continue + source_folder = PurePosixPath(source_path).parent + for raw_reference, embedded in self._note_references(note): + references_seen += 1 + if references_seen % 64 == 0: + check_cancel() + paths, ignored = self._reference_paths(source_folder, raw_reference) + if ignored: + continue + candidates: list[str] = [] + for candidate in paths: + key = candidate.casefold() + if key in exact: + candidates = sorted(set(exact[key])) + break + if not candidates and paths: + candidates = sorted(set( + names.get(PurePosixPath(paths[-1]).stem.casefold(), []) + )) + if len(candidates) != 1: + target_ids = [ + memory_by_path[path] + for path in candidates + if memory_by_path.get(path) + ] + if not target_ids and not candidates: + historical_paths = [ + historical_path + for path in paths + for historical_path in historical_exact.get( + path.casefold(), [] + ) + ] + target_ids = [ + memory_by_path[path] + for path in sorted(set(historical_paths)) + if memory_by_path.get(path) + ] + if target_ids: + retire_ambiguous_links(source_id, sorted(set(target_ids))) + warnings.append(self._outcome( + note, "warning", + "ambiguous_wikilink" if candidates else "unresolved_wikilink", + )) + continue + target_id = memory_by_path.get(candidates[0]) + if not target_id or target_id == source_id: + continue + desired_pairs.add(pair_key(source_id, target_id)) + if not batch_open: + self.store.conn.execute("BEGIN IMMEDIATE") + batch_open = True + self.store.add_link( + source_id, target_id, "embeds" if embedded else "references", + layer=GraphLayer.SEMANTIC, reason=self.LINK_REASON, commit=False, + ) + writes_in_batch += 1 + if writes_in_batch >= 128: + flush() + current_source_ids = { + memory_by_path[path] for path in note_by_path if memory_by_path.get(path) + } + for source_id in current_source_ids: + existing = self.store.conn.execute( + "SELECT a, b FROM mem_links " + "WHERE reason=? AND valid_to IS NULL AND expired_at IS NULL " + "AND (a=? OR b=?)", + (self.LINK_REASON, source_id, source_id), + ).fetchall() + for row in existing: + pair = pair_key(str(row["a"]), str(row["b"])) + if pair not in desired_pairs: + retire_pairs.add(pair) + retire_unsupported_links() + except BaseException: + if batch_open and self.store.conn.transaction_owned_by_current_thread(): + self.store.conn.rollback() + raise + flush() + return warnings + + def _persist_link_warnings(self, job_id: str, warnings: list[dict]) -> None: + """Attach reconciliation warnings to the durable polling rows.""" + if not warnings: + return + items = { + str(item.get("relative_path")): item + for item in self.store.list_source_import_job_items(job_id=job_id) + } + for warning in warnings: + relative_path = str(warning.get("relative_path") or "") + if not relative_path: + continue + item = items.get(relative_path) + reason = str(warning.get("reason") or "link_reconciliation_warning")[:100] + if item is None: + planned_action = "warning" + result_state = "warning" + source_id = None + source_format = str(warning.get("format") or "")[:64] + warning_count = 1 + else: + planned_action = str(item.get("planned_action") or "warning") + result_state = str(item.get("result_state") or "warning") + source_id = item.get("source_id") + source_format = str(item.get("source_format") or "")[:64] + warning_count = int(item.get("warning_count") or 0) + 1 + self.store.record_source_import_job_item( + job_id=job_id, source_id=source_id, relative_path=relative_path, + planned_action=planned_action, result_state=result_state, + warning_count=warning_count, error_code=reason, + source_format=source_format, + ) + items[relative_path] = { + "source_id": source_id, "planned_action": planned_action, + "result_state": result_state, "warning_count": warning_count, + "source_format": source_format, + } + + def _note_references(self, note: _ImportNote) -> list[tuple[str, bool]]: + """Return source references while preserving Obsidian attachment semantics.""" + attachments = {str(item.path) for item in note.attachments} + result: list[tuple[str, bool]] = [] + seen: set[tuple[str, bool]] = set() + for link in note.links: + value = (str(link.target), bool(link.embedded)) + if value[0] in attachments and not self.LINK_IMPORTED_ATTACHMENTS: + continue + if value not in seen: + seen.add(value) + result.append(value) + if self.LINK_IMPORTED_ATTACHMENTS: + for attachment in note.attachments: + value = (str(attachment.path), True) + if value not in seen: + seen.add(value) + result.append(value) + return result + + @staticmethod + def _reference_paths( + source_folder: PurePosixPath, target: str, + ) -> tuple[list[str], bool]: + """Resolve a source reference without filesystem access or root traversal.""" + raw = str(target or "").strip() + if not raw or raw.startswith("#"): + return [], True + parsed = urlsplit(raw) + if parsed.scheme or parsed.netloc: + return [], True + decoded = unquote(parsed.path).replace("\\", "/") + if not decoded: + return [], True + + values: list[str] = [] + raw_root = decoded.lstrip("/") + raw_candidates = ( + [raw_root] if decoded.startswith("/") + else [posixpath.join(str(source_folder), raw_root), raw_root] + ) + for candidate in raw_candidates: + normalized = posixpath.normpath(candidate) + if ( + normalized in {"", ".", ".."} + or normalized.startswith("../") + or normalized.startswith("/") + ): + continue + if normalized not in values: + values.append(normalized) + return values, False + + @staticmethod + def _outcome( + note: _ImportNote, status: str, reason: str, *, memory_id: str = "", + ) -> dict: + row = { + "relative_path": note.relative_path, "status": status, "action": status, + "reason": reason, "warnings": list(note.warnings[:20]), + "format": str(getattr(note, "format", "markdown")), + "media_type": str(getattr(note, "media_type", "text/markdown")), + } + if memory_id: + row["memory_id"] = memory_id + return row + + def _report( + self, plans: list[_Plan], missing: list[dict], scan: _ImportScan, *, + state: str, vault_id: Optional[str], workspace_id: Optional[str], + repo_id: Optional[str], session_id: Optional[str], scope: Scope, + memory_type: MemoryType, policy: str, vault_label: str, + attachment_manifest: Optional[list[dict]], + ) -> dict: + files = [self._preview_row(plan) for plan in plans] + files.extend({ + "relative_path": issue.relative_path, "status": "rejected", "action": "rejected", + "reason": issue.reason, "warnings": [], + } for issue in scan.rejected) + files.extend({ + "relative_path": issue.relative_path, "status": "skipped", "action": "skipped", + "reason": issue.reason, "warnings": [], + } for issue in scan.skipped) + files.extend({ + "relative_path": str(item.get("relative_path") or ""), "status": "missing", + "action": "missing", "reason": "source_not_seen", "warnings": [], + } for item in missing) + return self._report_payload( + files, scan, state=state, vault_id=vault_id, + workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, + scope=scope, memory_type=memory_type, policy=policy, + vault_label=vault_label, attachment_manifest=attachment_manifest, + ) + + def _final_report( + self, plans: list[_Plan], outcomes: list[dict], missing: list[dict], + scan: _ImportScan, *, pending_missing: list[dict], state: str, + vault_id: str, job_id: str, + import_id: str, workspace_id: str, repo_id: Optional[str], + session_id: Optional[str], scope: Scope, memory_type: MemoryType, + policy: str, vault_label: str, attachment_manifest: Optional[list[dict]], + ) -> dict: + files = list(outcomes) + processed_paths = { + str(row.get("relative_path") or "") for row in outcomes + } + files.extend( + { + **self._preview_row(plan), + "status": "pending", + "action": "pending", + "reason": "import_deferred", + } + for plan in plans + if plan.note.relative_path not in processed_paths + ) + files.extend({ + "relative_path": issue.relative_path, "status": "rejected", "action": "rejected", + "reason": issue.reason, "warnings": [], + } for issue in scan.rejected) + files.extend({ + "relative_path": issue.relative_path, "status": "skipped", "action": "skipped", + "reason": issue.reason, "warnings": [], + } for issue in scan.skipped) + files.extend({ + "relative_path": str(item.get("relative_path") or ""), "status": "missing", + "action": "missing", "reason": "source_not_seen", "warnings": [], + } for item in missing) + files.extend({ + "relative_path": str(item.get("relative_path") or ""), "status": "pending", + "action": "pending", "reason": "missing_check_deferred", "warnings": [], + } for item in pending_missing) + report = self._report_payload( + files, scan, state=state, vault_id=vault_id, + workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, + scope=scope, memory_type=memory_type, policy=policy, + vault_label=vault_label, attachment_manifest=attachment_manifest, + ) + report.update({"job_id": job_id, "import_id": import_id}) + return report + + @staticmethod + def _preview_row(plan: _Plan) -> dict: + note = plan.note + return { + "relative_path": note.relative_path, "status": plan.action, + "action": plan.action, "reason": plan.reason, "title": note.title, + "folder": ( + "" if str(PurePosixPath(note.relative_path).parent) == "." + else str(PurePosixPath(note.relative_path).parent) + ), + "aliases": note.aliases, "tags": note.tags, "headings": note.headings, + "links": [link.__dict__ for link in note.links], + "attachments": [entry.__dict__ for entry in note.attachments], + "warnings": note.warnings, + "format": str(getattr(note, "format", "markdown")), + "media_type": str(getattr(note, "media_type", "text/markdown")), + } + + def _report_payload( + self, files: list[dict], scan: _ImportScan, *, state: str, + vault_id: Optional[str], workspace_id: Optional[str], repo_id: Optional[str], + session_id: Optional[str], scope: Scope, memory_type: MemoryType, + policy: str, vault_label: str, attachment_manifest: Optional[list[dict]], + ) -> dict: + counts: dict[str, int] = {} + for row in files: + status = str(row.get("status") or "reported") + counts[status] = counts.get(status, 0) + 1 + counts[self.COUNT_KEY] = len(scan.notes) + len(scan.rejected) + folders = sorted({ + str(PurePosixPath(note.relative_path).parent) + for note in scan.notes if str(PurePosixPath(note.relative_path).parent) != "." + }) + tags = sorted({tag for note in scan.notes for tag in note.tags}) + formats: dict[str, int] = {} + for note in scan.notes: + name = str(getattr(note, "format", "markdown") or "unknown") + formats[name] = formats.get(name, 0) + 1 + return { + "state": state, "status": state, "vault_id": vault_id, + "vault_label": str(vault_label or "")[:200], + "source_id": vault_id, + "source_label": str(vault_label or "")[:200], + "source_kind": self.SOURCE_KIND, + "target": { + "workspace_id": workspace_id, "repo_id": repo_id, + "session_id": session_id, "scope": scope.value, + "memory_type": memory_type.value, + }, + "on_conflict": policy, "counts": counts, + "summary": { + "folders": folders[:500], "tags": tags[:500], + "aliases": sum(len(note.aliases) for note in scan.notes), + "wikilinks": sum(len(note.links) for note in scan.notes), + "attachments": sum(len(note.attachments) for note in scan.notes), + "attachment_manifest": len(attachment_manifest or []), + "warnings": sum(len(note.warnings) for note in scan.notes), + "skipped_paths": len(scan.skipped), + "formats": formats, + }, + "files": files, + } + + def _create_job( + self, workspace_id: str, repo_id: Optional[str], *, session_id: Optional[str], total: int, + policy: str, scope: Scope, memory_type: MemoryType, + ) -> str: + job_id = new_id("job") + stamp = time.time() + self.store.conn.execute( + "INSERT INTO jobs(id, workspace_id, repo_id, session_id, kind, state, dry_run, total_items, " + "processed_items, counts, errors, request, cancel_requested, created_at, started_at) " + "VALUES (?,?,?,?,?,'running',0,?,0,'{}','[]',?,0,?,?)", + ( + job_id, workspace_id, repo_id, session_id, self.JOB_KIND, int(total), + json.dumps({ + "scope": scope.value, "memory_type": memory_type.value, + "on_conflict": policy, + }, sort_keys=True, separators=(",", ":")), + stamp, stamp, + ), + ) + self.store.conn.commit() + return job_id + + def _check_cancel( + self, job_id: str, cancel_check: Optional[Callable[[], bool]], + ) -> None: + if cancel_check is not None and cancel_check(): + raise ObsidianImportCancelled + row = self.store.conn.execute( + "SELECT cancel_requested FROM jobs WHERE id=?", (job_id,), + ).fetchone() + if row is not None and bool(row["cancel_requested"]): + raise ObsidianImportCancelled + + def _update_job_progress(self, job_id: str, processed: int, outcomes: list[dict]) -> None: + counts: dict[str, int] = {} + for row in outcomes: + status = str(row.get("status") or "reported") + counts[status] = counts.get(status, 0) + 1 + self.store.conn.execute( + "UPDATE jobs SET processed_items=?, counts=?, heartbeat_at=? WHERE id=?", + (processed, json.dumps(counts, sort_keys=True, separators=(",", ":")), time.time(), job_id), + ) + self.store.conn.commit() + + def _finish_job(self, job_id: str, state: str, report: dict) -> None: + errors = [ + {"status": row.get("status"), "reason": row.get("reason")} + for row in report.get("files", []) + if row.get("status") in {"error", "conflict", "rejected"} + ][:1000] + self.store.conn.execute( + "UPDATE jobs SET state=?, processed_items=?, counts=?, errors=?, finished_at=?, " + "heartbeat_at=? WHERE id=?", + ( + state, sum(report.get("counts", {}).get(key, 0) for key in ( + "imported", "updated", "renamed", "skipped", "conflict", "error", + "rejected", + )), + json.dumps(report.get("counts", {}), sort_keys=True, separators=(",", ":")), + json.dumps(errors, sort_keys=True, separators=(",", ":")), + time.time(), time.time(), job_id, + ), + ) + self.store.conn.commit() + + def _record_receipt( + self, report: dict, *, workspace_id: str, repo_id: Optional[str], actor: str, + ) -> None: + counts = report.get("counts", {}) + summary = report.get("summary", {}) + self.store.record_receipt( + self.RECEIPT_OPERATION, workspace_id=workspace_id, repo_id=repo_id or "", + actor=actor, target_count=int(counts.get(self.COUNT_KEY, 0)), + status=str(report.get("state") or "partial"), + metadata={ + "files_imported": int(counts.get("imported", 0)), + "files_updated": int(counts.get("updated", 0)), + "files_renamed": int(counts.get("renamed", 0)), + "files_skipped": int(counts.get("skipped", 0)), + "files_rejected": int(counts.get("rejected", 0)), + "files_missing": int(counts.get("missing", 0)), + "files_errored": int(counts.get("error", 0)), + "conflicts": int(counts.get("conflict", 0)), + "warnings": int(summary.get("warnings", 0)), + "attachments": int(summary.get("attachments", 0)), + "wikilinks": int(summary.get("wikilinks", 0)), + "aliases": int(summary.get("aliases", 0)), + "tags": len(summary.get("tags", [])), + }, + ) + + +__all__ = [ + "ObsidianImportCancelled", "ObsidianImporter", "scan_obsidian_upload", + "stable_source_key", +] diff --git a/engraphis/read_only_api.py b/engraphis/read_only_api.py index 372d1c04..0fff71fb 100644 --- a/engraphis/read_only_api.py +++ b/engraphis/read_only_api.py @@ -24,33 +24,45 @@ logger = logging.getLogger("engraphis.read_only") +MAX_READ_ONLY_BODY_BYTES = 2_000_000 +MAX_READ_ONLY_TEXT_CHARS = 100_000 +MAX_READ_ONLY_LIST_ITEMS = 2_000 + + +class BodyTooLarge(Exception): + """Internal marker: a streamed request exceeded the body limit. + + Raised inside the receive hook where FastAPI's request-body parser would + otherwise swallow it as a generic parse error; the middleware translates + it to the same 413 the declared-length path returns. + """ class IntentRecallRequest(BaseModel): - query: str - intent: str = "recall" - workspace: Optional[str] = None - repo: Optional[str] = None - mtypes: Optional[list[str]] = None - k: int = 8 + query: str = Field(..., min_length=1, max_length=MAX_READ_ONLY_TEXT_CHARS) + intent: str = Field("recall", max_length=64) + workspace: Optional[str] = Field(None, max_length=256) + repo: Optional[str] = Field(None, max_length=256) + mtypes: Optional[list[str]] = Field(None, max_length=16) + k: int = Field(8, ge=1, le=500) as_of: Optional[float] = None valid_at: Optional[float] = None known_at: Optional[float] = None - token_budget: Optional[int] = None - retrieval_profile: str = "balanced" - candidate_depth: str = "fixed" - response_mode: str = "compact" + token_budget: Optional[int] = Field(None, ge=1, le=100_000) + retrieval_profile: str = Field("balanced", max_length=32) + candidate_depth: str = Field("fixed", max_length=32) + response_mode: str = Field("compact", max_length=32) diagnostics: bool = False - planning: str = "off" - mtype_limits: Optional[dict[str, StrictInt]] = None + planning: str = Field("off", max_length=32) + mtype_limits: Optional[dict[str, StrictInt]] = Field(None, max_length=16) class CodePathRequest(BaseModel): - workspace: str - repo: str - source: str - target: str - max_depth: int = 8 + workspace: str = Field(..., min_length=1, max_length=256) + repo: str = Field(..., min_length=1, max_length=256) + source: str = Field(..., min_length=1, max_length=MAX_READ_ONLY_TEXT_CHARS) + target: str = Field(..., min_length=1, max_length=MAX_READ_ONLY_TEXT_CHARS) + max_depth: int = Field(8, ge=1, le=128) capacity: int = Field( default=DEFAULT_CODE_QUERY_CAPACITY, ge=1, le=MAX_CODE_QUERY_CAPACITY ) @@ -60,9 +72,11 @@ class CodePathRequest(BaseModel): class CodeImpactRequest(BaseModel): - workspace: str - repo: str - changed_files: list[str] + workspace: str = Field(..., min_length=1, max_length=256) + repo: str = Field(..., min_length=1, max_length=256) + changed_files: list[str] = Field( + ..., min_length=1, max_length=MAX_READ_ONLY_LIST_ITEMS, + ) capacity: int = Field( default=DEFAULT_CODE_QUERY_CAPACITY, ge=1, le=MAX_CODE_QUERY_CAPACITY ) @@ -139,6 +153,42 @@ async def redact_unhandled_errors(request, call_next): {"error": "internal server error"}, status_code=500 ) + @app.middleware("http") + async def limit_request_body(request, call_next): + content_length = request.headers.get("content-length") + try: + declared_length = int(content_length) if content_length else 0 + except ValueError: + declared_length = 0 + if declared_length > MAX_READ_ONLY_BODY_BYTES: + return JSONResponse({"detail": "request body too large"}, status_code=413) + received = 0 + original_receive = request.receive + + async def limited_receive(): + nonlocal received + message = await original_receive() + if message.get("type") == "http.request": + received += len(message.get("body") or b"") + if received > MAX_READ_ONLY_BODY_BYTES: + # Raising here is consumed by FastAPI's request-body parser for + # chunked/streamed bodies, which reports a generic 400 before our + # middleware can translate it. Emit the 413 response directly. + raise BodyTooLarge + return message + + request._receive = limited_receive + try: + return await call_next(request) + except BodyTooLarge: + return JSONResponse({"detail": "request body too large"}, status_code=413) + except ValueError as exc: + if str(exc) == "request body too large": + return JSONResponse( + {"detail": "request body too large"}, status_code=413 + ) + raise + def run(fn, *args, **kwargs): try: return fn(*args, **kwargs) @@ -150,18 +200,20 @@ def health(): return {"ok": True, "mode": "read-only"} @app.get("/recall") - def recall(query: str, workspace: Optional[str] = None, - repo: Optional[str] = None, k: int = 8, + def recall(query: str = Query(..., min_length=1, max_length=MAX_READ_ONLY_TEXT_CHARS), + workspace: Optional[str] = Query(None, max_length=256), + repo: Optional[str] = Query(None, max_length=256), + k: int = Query(8, ge=1, le=500), as_of: Optional[float] = None, valid_at: Optional[float] = None, known_at: Optional[float] = None, - token_budget: Optional[int] = None, - retrieval_profile: str = "balanced", - candidate_depth: str = "fixed", - response_mode: str = "compact", + token_budget: Optional[int] = Query(None, ge=1, le=100_000), + retrieval_profile: str = Query("balanced", max_length=32), + candidate_depth: str = Query("fixed", max_length=32), + response_mode: str = Query("compact", max_length=32), diagnostics: bool = False, - planning: str = "off", - mtype_limits: Optional[str] = None): + planning: str = Query("off", max_length=32), + mtype_limits: Optional[str] = Query(None, max_length=4_000)): try: parsed_limits = json.loads(mtype_limits) if mtype_limits else None if parsed_limits is not None and not isinstance(parsed_limits, dict): diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index 06415f6d..6bd6fecf 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -127,6 +127,7 @@ def service() -> MemoryService: vector_backend=settings.vector_backend, rerank_model=getattr(settings, "rerank_model", "") or None, rerank_revision=getattr(settings, "rerank_revision", "") or None, + allowed_workspaces=settings.allowed_workspaces, ) return _service @@ -357,9 +358,12 @@ def _keyword_search(ws, q, limit=20, *, as_of: Optional[float] = None, Recall/Why/Timeline tabs still return results when the embedder is unavailable.""" import json as _json import sqlite3 as _sql - ws = service()._clean_ws(ws) - conn = _sql.connect("file:%s?mode=ro" % settings.db_path, uri=True) - conn.row_factory = _sql.Row + current_service = service() + ws = current_service._clean_ws(ws) + # Read through the active store rather than opening a second raw SQLite connection. + # This keeps dashboard reads on the same database/connection semantics as writes, + # including :memory: databases, SQLCipher, and custom store connectors. + conn = current_service.store.conn try: row = conn.execute("SELECT id FROM workspaces WHERE name=?", (ws,)).fetchone() if row is None: @@ -394,8 +398,9 @@ def _keyword_search(ws, q, limit=20, *, as_of: Optional[float] = None, args += ["%" + _escape_like(t) + "%", "%" + _escape_like(t) + "%"] sql += " ORDER BY COALESCE(last_access, valid_from) DESC" rows = conn.execute(sql, args).fetchall() - finally: - conn.close() + except _sql.Error as exc: + logger.error("dashboard keyword search failed (%s)", type(exc).__name__) + raise HTTPException(status_code=500, detail={"error": "internal server error"}) from None def _prov(pp): try: @@ -1341,18 +1346,22 @@ def memories(workspace: Optional[str] = None, q: Optional[str] = Query(default=N without sentence-transformers. Live memories only (not superseded/expired).""" import json as _json import sqlite3 as _sql + current_service = service() ws = workspace or _default_ws() if not ws: # No workspace exists yet (fresh install) — nothing to list. Return an empty # result instead of letting _clean_ws(None) raise a 500. return {"workspace": "", "count": 0, "memories": []} try: - ws = service()._clean_ws(ws) + ws = current_service._clean_ws(ws) except (ValidationError, ValueError): logger.info("dashboard memories request rejected") raise _invalid_request() from None - conn = _sql.connect("file:%s?mode=ro" % settings.db_path, uri=True) - conn.row_factory = _sql.Row + # Keep this read on the live service store. A second sqlite3 connection points at + # a different database for :memory: stores and bypasses SQLCipher/custom connector + # semantics, which made the dashboard report no memories even while stats and writes + # used the populated active store. + conn = current_service.store.conn try: row = conn.execute("SELECT id FROM workspaces WHERE name=?", (ws,)).fetchone() if row is None: @@ -1374,8 +1383,6 @@ def memories(workspace: Optional[str] = None, q: Optional[str] = Query(default=N except _sql.Error as exc: logger.error("dashboard memory listing failed (%s)", type(exc).__name__) raise HTTPException(status_code=500, detail={"error": "internal server error"}) from None - finally: - conn.close() def _prov(p): try: @@ -2127,9 +2134,7 @@ def graph(workspace: Optional[str] = None, service closes that gap. """ ws = workspace or _default_ws() - selected = None if layers is None else [ - x.strip() for x in layers.split(",") if x.strip() - ] + selected = _graph_csv(layers) return _run( service().graph, workspace=ws, limit=limit, layers=selected, include_code=include_code, repo=repo, backfill=False, full=full, diff --git a/engraphis/routes/vault.py b/engraphis/routes/vault.py index 6b561ccc..b4680312 100644 --- a/engraphis/routes/vault.py +++ b/engraphis/routes/vault.py @@ -50,65 +50,74 @@ _DUPLICATE_BLOCK_SIZE = 256 -def _read_import_bytes(path: Path, max_bytes: int) -> bytes: - """Read one imported file without following a swapped leaf or reparse point.""" +def _is_within(root: Path, candidate: Path) -> bool: + """Return whether *candidate* is a descendant of (or equal to) *root*.""" + try: + candidate.relative_to(root) + return True + except ValueError: + return False + + +def _same_identity(left: os.stat_result, right: os.stat_result) -> bool: + """Return whether two stat results reference the same file on disk.""" + if left.st_dev or left.st_ino or right.st_dev or right.st_ino: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + return True + + +def _is_reparse_point(info: os.stat_result) -> bool: + """Return whether a stat result carries the Windows reparse-point attribute.""" + marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(getattr(info, "st_file_attributes", 0) & marker) + + +def _read_import_file(folder: Path, path: Path, limit: int) -> bytes: + """Read *path* descriptor-safely, bounding it to *limit* bytes. + + Mirrors ``engraphis.core.documents._read_tree_file``: the path is re-validated + against *folder* at open time (lstat -> type/symlink/reparse rejection -> + containment -> ``O_NOFOLLOW`` open -> fstat identity -> size bound -> read -> + post-read identity/containment recheck) so a symlink swapped in between the + enumeration phase and this read cannot escape the import root. + """ + before = os.lstat(path) + if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_reparse_point(before): + raise OSError("unsafe file type") + if not _is_within(folder, path.resolve(strict=True)): + raise OSError("path escapes import root") flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(str(path), flags) + fd = os.open(path, flags) try: - opened = os.fstat(descriptor) - if not stat.S_ISREG(opened.st_mode) or getattr(opened, "st_nlink", 1) != 1: - raise ValueError("import path is not a single-link regular file") - current = os.lstat(str(path)) - reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) - if stat.S_ISLNK(current.st_mode) or ( - reparse_flag and getattr(current, "st_file_attributes", 0) & reparse_flag - ) or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino): - raise ValueError("import path changed while it was opened") - data = bytearray() - while len(data) <= max_bytes: - chunk = os.read(descriptor, min(65_536, max_bytes + 1 - len(data))) + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_identity(before, opened): + raise OSError("file changed during import") + if opened.st_size > limit: + raise OSError("import resource exceeds its byte limit") + chunks = [] + total = 0 + while True: + chunk = os.read(fd, min(64 * 1024, limit + 1 - total)) if not chunk: break - data.extend(chunk) - if len(data) > max_bytes: - raise ValueError("file grew beyond the import resource limit") - after = os.fstat(descriptor) + chunks.append(chunk) + total += len(chunk) + if total > limit: + raise OSError("import resource exceeds its byte limit") + finished, after = os.fstat(fd), os.lstat(path) if ( - after.st_size != opened.st_size - or after.st_mtime_ns != opened.st_mtime_ns + not _same_identity(opened, finished) + or opened.st_size != finished.st_size + or opened.st_mtime_ns != finished.st_mtime_ns + or stat.S_ISLNK(after.st_mode) + or _is_reparse_point(after) + or not _same_identity(finished, after) + or not _is_within(folder, path.resolve(strict=True)) ): - raise ValueError("import file changed while it was read") - return bytes(data) + raise OSError("file changed during import") + return b"".join(chunks) finally: - os.close(descriptor) - - -def _read_import_size(path: Path) -> int: - """Return the size of a previously validated imported file path.""" - flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(str(path), flags) - try: - opened = os.fstat(descriptor) - if not stat.S_ISREG(opened.st_mode) or getattr(opened, "st_nlink", 1) != 1: - raise ValueError("import path is not a single-link regular file") - current = os.lstat(str(path)) - reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) - if stat.S_ISLNK(current.st_mode) or ( - reparse_flag and getattr(current, "st_file_attributes", 0) & reparse_flag - ) or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino): - raise ValueError("import path changed while it was opened") - return opened.st_size - finally: - os.close(descriptor) - - -def _path_within_root(path: Path, root: Path) -> bool: - """Return True only when ``path`` is the root itself or a descendant of it.""" - try: - path.relative_to(root) - return True - except ValueError: - return False + os.close(fd) class _BoundedUploadRoute(APIRoute): @@ -349,66 +358,49 @@ def import_folder(req: FolderImportReq): import os import re - home = os.path.normcase(os.path.realpath(str(Path.home().expanduser()))) + home = os.path.realpath(str(Path.home().expanduser())) allowed_roots = [home] env_roots = os.environ.get("ENGRAPHIS_IMPORT_ROOTS", "") if env_roots: allowed_roots.extend( - os.path.normcase(os.path.realpath(os.path.expanduser(root))) + os.path.realpath(os.path.expanduser(root)) for root in env_roots.split(os.pathsep) if root ) - requested_path = os.path.normcase(os.path.realpath(os.path.expanduser(req.path))) - if not any( - requested_path == root - or requested_path.startswith(root.rstrip(os.sep) + os.sep) - for root in allowed_roots - ): + real_path = os.path.realpath(os.path.expanduser(req.path)) + comparable_path = os.path.normcase(real_path) + safe_path = None + for root in allowed_roots: + comparable_root = os.path.normcase(root) + if comparable_path == comparable_root: + safe_path = comparable_root + break + root_prefix = comparable_root.rstrip(os.sep) + os.sep + if comparable_path.startswith(root_prefix): + safe_path = comparable_path + break + if safe_path is None: raise HTTPException( 403, "Import path must be under an allowed root " "(home directory or ENGRAPHIS_IMPORT_ROOTS)", ) - # Resolve the user-selected folder by walking from an allowlisted root. The - # filesystem APIs below receive only paths returned by that trusted walk; - # request text is used only for component comparisons, never as a path root. - folder: Optional[Path] = None - for root in allowed_roots: - relative = os.path.relpath(requested_path, root) - components = tuple(part for part in Path(relative).parts if part not in ("", ".")) - if any(part == ".." for part in components): - continue - try: - current = Path(root).resolve(strict=True) - for component in components: - with os.scandir(current) as entries: - match = next( - ( - entry for entry in entries - if os.path.normcase(entry.name) == component - ), - None, - ) - if match is None or match.is_symlink() or not match.is_dir(follow_symlinks=False): - current = None - break - current = Path(match.path) - except (OSError, RuntimeError, ValueError): - continue - if current is not None: - folder = current - break - if folder is None: - raise HTTPException(404, f"Path not found: {req.path}") from None - canonical_path = os.path.normcase(os.path.realpath(str(folder))) + # Carry only the path value produced by the successful containment branch into + # filesystem operations. Keeping the validated value distinct from ``req.path`` + # makes the trust boundary explicit to readers and static taint analysis alike. + folder = Path(safe_path) + if folder.is_symlink(): + raise HTTPException(403, "Import path must not be a symbolic link") + resolved_folder = folder.resolve() + resolved_comparable = os.path.normcase(str(resolved_folder)) if not any( - canonical_path == root - or canonical_path.startswith(root.rstrip(os.sep) + os.sep) + resolved_comparable == os.path.normcase(root) + or resolved_comparable.startswith(os.path.normcase(root).rstrip(os.sep) + os.sep) for root in allowed_roots ): raise HTTPException( 403, - "Import path must be under an allowed root " + "Import path must resolve under an allowed root " "(home directory or ENGRAPHIS_IMPORT_ROOTS)", ) if not folder.exists(): @@ -432,36 +424,19 @@ def import_folder(req: FolderImportReq): files: list[tuple[Path, Path]] = [] total_bytes = 0 - # Security: traversal begins only from the canonical trusted folder and each - # yielded path is re-resolved and re-contained before use. - # codeql[py/path-injection] for candidate in folder.rglob("*"): - # Resolve each candidate before trusting its location. This rejects - # symlink/reparse escapes and ensures the path used for stat/read is the - # same resolved path that was validated against the trusted folder. - if candidate.is_symlink() or not fnmatch.fnmatch(candidate.name, req.file_pattern): - continue - try: - resolved_candidate = candidate.resolve(strict=True) - except (OSError, ValueError): - continue - if not _path_within_root(resolved_candidate, folder): + if candidate.is_symlink(): continue - if not resolved_candidate.is_file(): + if not candidate.is_file() or not fnmatch.fnmatch( + candidate.name, req.file_pattern + ): continue try: - relative = resolved_candidate.relative_to(folder) - # Security: the size comes from a descriptor opened only after the - # candidate has been proven to stay under the trusted traversal root. - # codeql[py/path-injection] - size = _read_import_size(resolved_candidate) + resolved = candidate.resolve(strict=True) + relative = resolved.relative_to(resolved_folder) + size = resolved.stat().st_size except (OSError, ValueError): continue - if not any( - _path_within_root(resolved_candidate, Path(root)) - for root in allowed_roots - ): - continue if any(part in {"node_modules", ".git"} for part in relative.parts[:-1]): continue if size > MAX_IMPORT_RESOURCE_BYTES: @@ -469,7 +444,7 @@ def import_folder(req: FolderImportReq): 413, f"Import resource exceeds {MAX_IMPORT_RESOURCE_BYTES} bytes", ) - files.append((resolved_candidate, relative)) + files.append((resolved, relative)) if len(files) > MAX_IMPORT_FILES: raise HTTPException( 413, @@ -486,10 +461,12 @@ def import_folder(req: FolderImportReq): for file_path, relative_path in files: relative = relative_path.as_posix() try: - # Security: file_path already passed the trusted-root containment - # checks, and the reader re-checks the opened inode. - # codeql[py/path-injection] - raw = _read_import_bytes(file_path, MAX_IMPORT_RESOURCE_BYTES) + # The enumerated path may have been swapped for a symlink since the + # enumeration pass; _read_import_file re-validates type, containment, + # and identity at open/read time, so the import root cannot be escaped. + raw = _read_import_file(resolved_folder, file_path, MAX_IMPORT_RESOURCE_BYTES) + if len(raw) > MAX_IMPORT_RESOURCE_BYTES: + raise ValueError("file grew beyond the import resource limit") content = raw.decode("utf-8", errors="replace") if not content.strip(): results["skipped"] += 1 @@ -499,9 +476,7 @@ def import_folder(req: FolderImportReq): ) title_match = re.search(r"^#\s+(.+)$", content, re.MULTILINE) title = ( - title_match.group(1).strip() - if title_match - else _filename_stem(relative) + title_match.group(1).strip() if title_match else file_path.stem ) ingest_engine.ingest_document( namespace=namespace, diff --git a/engraphis/service.py b/engraphis/service.py index 31a69159..b7818491 100644 --- a/engraphis/service.py +++ b/engraphis/service.py @@ -17,15 +17,16 @@ import os import re +import sys import json import hashlib import contextvars import logging import math import copy -import sys import time import threading +import unicodedata import numpy as np from collections import Counter, OrderedDict from dataclasses import asdict @@ -50,6 +51,7 @@ Edge, GraphLayer, MemoryType, Node, Scope, SearchFilter, embedder_capabilities, embedding_space_fingerprint, vector_index_requires_sync, + vector_index_shares_store_transaction, ) from engraphis.core.poisoning import ( REVIEW_APPROVED, @@ -885,15 +887,9 @@ def _title_from_content(content: str, fallback: str) -> str: def _warn_if_db_empty_with_populated_sibling(db_path: str) -> None: - """Best-effort diagnostic when the configured database holds zero memories. - - Probes well-known sibling locations (the installed-default paths and the - owner-private ``~/.engraphis/engraphis.db``). When a sibling holds memories - and the configured path does not, emit a one-time stderr notice naming both - paths so the operator can reconcile. This is a diagnostic only — it never - raises, never blocks startup, and never reads memory content. - """ + """Warn, without blocking startup, when a sibling database holds the data.""" import sqlite3 + configured = Path(db_path) if not configured.is_file(): return @@ -903,31 +899,26 @@ def _warn_if_db_empty_with_populated_sibling(db_path: str) -> None: row = probe.execute("SELECT COUNT(*) FROM memories").fetchone() configured_count = int(row[0]) if row else 0 except sqlite3.Error: - return # no memories table yet, or unreadable — nothing to warn about + return finally: probe.close() except sqlite3.Error: return if configured_count > 0: - return # configured DB has data — no diagnostic needed - # Build the set of well-known sibling paths (excluding the configured one). + return + home = Path.home() - candidates: list[Path] = [] - # Owner-private home location (most common for real data). - candidates.append(home / ".engraphis" / "engraphis.db") - # Platform-specific installed defaults. + candidates: list[Path] = [home / ".engraphis" / "engraphis.db"] if os.name == "nt": local_appdata = os.environ.get("LOCALAPPDATA") if local_appdata: candidates.append(Path(local_appdata) / "engraphis" / "engraphis.db") elif sys.platform == "darwin": - candidates.append( - home / "Library" / "Application Support" / "engraphis" / "engraphis.db" - ) + candidates.append(home / "Library" / "Application Support" / "engraphis" / "engraphis.db") else: xdg = os.environ.get("XDG_DATA_HOME", str(home / ".local" / "share")) candidates.append(Path(xdg) / "engraphis" / "engraphis.db") - # Deduplicate and exclude the configured path. + configured_resolved = configured.resolve() seen: set[str] = set() for candidate in candidates: @@ -957,7 +948,8 @@ def _warn_if_db_empty_with_populated_sibling(db_path: str) -> None: "populated database." % (configured, candidate, count), file=sys.stderr, ) - return # one notice is enough + return + def _auto_migrate_v1_if_needed(db_path: str) -> None: """If *db_path* is an existing v1-shaped SQLite file, migrate it to the v2 schema @@ -990,13 +982,10 @@ def _auto_migrate_v1_if_needed(db_path: str) -> None: original file exactly as it was; ``Store`` then raises its normal (now unmasked) error instead of silently losing data.""" p = Path(db_path) - # Recover from a crash during a previous two-step swap (legacy code). + # Recover a crash from the legacy two-step Windows swap before inspecting the + # database. A completed migration output is preferred to the stale v1 backup. staging = p.with_suffix(".v2_swap") if not p.exists() and staging.exists(): - # If a completed-but-not-swapped migration output also exists - # (.v2-migrating-*), prefer it over the stale v1 original in - # .v2_swap — the migration had finished writing the new file but - # crashed before the final os.replace into db_path. migrating = sorted( p.parent.glob(p.stem + ".v2-migrating-*" + p.suffix), key=lambda f: f.stat().st_mtime, @@ -1016,8 +1005,6 @@ def _auto_migrate_v1_if_needed(db_path: str) -> None: else: os.replace(str(staging), str(p)) elif not p.exists(): - # No .v2_swap either, but a stray .v2-migrating-* might remain - # from a crash where the original was already removed. migrating = sorted( p.parent.glob(p.stem + ".v2-migrating-*" + p.suffix), key=lambda f: f.stat().st_mtime, @@ -1056,11 +1043,23 @@ def _auto_migrate_v1_if_needed(db_path: str) -> None: shutil.copy2(str(p), str(backup)) # preserve the untouched original first from scripts.migrate_to_v2 import migrate counts = migrate(str(p), str(tmp_new)) # reads p (untouched), writes tmp_new - # Both paths are in the same directory, so os.replace gives the only - # single-name swap available on Windows: a failed replacement leaves the - # original at p, while a successful replacement makes the complete migrated - # file visible without an intermediate missing-path window. - os.replace(str(tmp_new), str(p)) + # On Windows os.replace is not atomic; use a two-step rename with a staging + # file so a crash mid-swap leaves either the original or the migrated DB intact. + staging = p.with_suffix(".v2_swap") + try: + if staging.exists(): + staging.unlink() + os.rename(str(p), str(staging)) + os.rename(str(tmp_new), str(p)) + try: + staging.unlink() + except OSError: + pass # best-effort cleanup; backup still exists + except Exception: + # Rollback: restore the original if the swap failed partway through. + if staging.exists() and not p.exists(): + os.rename(str(staging), str(p)) + raise print("[engraphis] v1->v2 auto-migration complete: %s" % counts, file=sys.stderr) except Exception as exc: # noqa: BLE001 — must never brick startup worse than before print("[engraphis] v1->v2 auto-migration failed (%s) — leaving %s untouched; " @@ -1103,6 +1102,7 @@ def __init__(self, engine: MemoryEngine, *, self._graph_scene_cache: "OrderedDict[tuple, tuple[float, dict]]" = OrderedDict() self._graph_job_lock = threading.RLock() self._graph_job_threads: dict[str, threading.Thread] = {} + self._obsidian_job_threads: dict[str, threading.Thread] = {} self._graph_runner_id = make_id("device") self._service_close_lock = threading.Lock() self._closing = False @@ -1131,11 +1131,24 @@ def close(self, *, timeout: float = GRAPH_INDEX_SHUTDOWN_SECONDS) -> None: with self._graph_job_lock: workers = list(self._graph_job_threads.items()) + with self._graph_job_lock: + import_workers = list(self._obsidian_job_threads.items()) + for job_id, _thread in import_workers: + self.store.conn.execute( + "UPDATE jobs SET cancel_requested=1 WHERE id=? AND state='running'", + (job_id,), + ) + if import_workers: + self.store.conn.commit() + deadline = time.monotonic() + timeout_value - for _job_id, thread in workers: + for _job_id, thread in [*workers, *import_workers]: remaining = max(0.0, deadline - time.monotonic()) thread.join(remaining) - alive = [job_id for job_id, thread in workers if thread.is_alive()] + alive = [ + job_id for job_id, thread in [*workers, *import_workers] + if thread.is_alive() + ] if alive: raise RuntimeError( f"{len(alive)} graph index worker(s) did not stop before shutdown" @@ -1221,15 +1234,10 @@ def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, allow_automatic_critical_retention=bool(allow_automatic_critical_retention), query_planner=query_planner, read_only=read_only, ) - # Startup diagnostic: warn when the configured database exists but is empty, - # while a sibling database at a well-known location has memories. Catches the - # exact misconfiguration that caused silent memory loss (config.env pointing to - # an empty DB while the real data lived elsewhere). Non-fatal — the service - # still starts against the configured path so fresh installs are unaffected. if db_path != ":memory:" and not read_only: try: _warn_if_db_empty_with_populated_sibling(db_path) - except Exception: # noqa: BLE001 — diagnostic must never block startup + except Exception: # noqa: BLE001 — diagnostics never block startup pass return cls(engine, allowed_workspaces=allowed_workspaces) @@ -1661,9 +1669,7 @@ def remember_batch(self, memories: list[dict], *, workspace: str) -> dict: source=mem.get("source", "agent"), trusted=mem.get("trusted", False), kind=mem.get("kind"), - resolve_conflicts=mem.get( - "resolve_conflicts", mem.get("dedupe", True) - ), + resolve_conflicts=mem.get("resolve_conflicts", mem.get("dedupe", True)), retention_class=mem.get("retention_class"), retention_reason=mem.get("retention_reason", ""), valid_from=mem.get("valid_from"), @@ -2152,6 +2158,928 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman "skipped": skipped, "errors": errors, "derived_facts": derived_facts, "details": details[:50], "warnings": warnings[:50]} + # ── Universal local document import (v2 source manifest) ──────────────── + def _document_registered_target( + self, source_id: Optional[str], *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], memory_type: str, + ) -> Optional[str]: + """Validate a selected source collection before creating scope rows.""" + if source_id is None: + return None + clean_id = _clean_text( + source_id, field="source_id", max_chars=MAX_NAME_CHARS, + ) + if not clean_id.startswith("vlt_"): + raise ValidationError("registered document source was not found") + _ws, wid, rid, sid, selected_scope, selected_type = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + if wid is None or (repo is not None and rid is None): + raise ValidationError("registered document source was not found") + source = self.store.get_source_vault(clean_id) + if ( + source is None + or source.get("kind") != "documents" + or source.get("workspace_id") != wid + or source.get("repo_id") != rid + or source.get("session_id") != sid + ): + raise ValidationError("registered document source does not belong to that target") + if ( + source.get("scope") != selected_scope.value + or source.get("memory_type") != selected_type.value + ): + raise ValidationError("registered document source has different import defaults") + return clean_id + + def _document_label(self, value: str, *, source_id: Optional[str] = None) -> str: + label = unicodedata.normalize("NFC", _clean_text( + value, field="source_label", max_chars=MAX_NAME_CHARS, required=False, + )) + if not label and source_id: + source = self.store.get_source_vault(source_id) + label = str(source.get("display_name") or "") if source else "" + if not label: + raise ValidationError("source_label is required for a new browser source") + _reject_secret_capture((("source_label", label),)) + return label + + def _require_new_browser_source_label( + self, label: str, *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], memory_type: str, + source_kind: str, source_noun: str, + ) -> None: + """Keep independently selected browser uploads from sharing a label lineage. + + Browser uploads intentionally have no stable filesystem root. A new upload + therefore cannot safely infer that an existing same-label collection is the + same source. Require the owner to select that registered ``vlt_`` identity + explicitly; disk imports retain their root-digest auto-selection path. + """ + _ws, wid, rid, sid, _selected_scope, _selected_type = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + if wid is None: + return + # Browser source roots are deliberately derived only from the normalized + # owner label: source bytes change on every edit. Query that exact, + # indexed identity rather than a capped presentation list; otherwise a + # 101st registered source could silently reuse a live lineage. + root_digest = hashlib.sha256( + f"{source_kind}-browser\0{label.casefold()}".encode("utf-8", "surrogatepass"), + ).hexdigest() + source = self.store.get_source_vault_by_root_digest( + kind=source_kind, root_digest=root_digest, workspace_id=wid, + repo_id=rid, session_id=sid, + ) + if source is not None: + raise ValidationError( + f"a {source_noun} with this label already exists; select its source_id to resume" + ) + + def _require_new_document_label( + self, label: str, *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], memory_type: str, + ) -> None: + self._require_new_browser_source_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, source_kind="documents", + source_noun="source", + ) + + def _require_new_obsidian_label( + self, label: str, *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], memory_type: str, + ) -> None: + self._require_new_browser_source_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, source_kind="obsidian", + source_noun="vault", + ) + + @staticmethod + def _document_report(report: dict) -> dict: + """Expose the selected adapter explicitly on every generic response.""" + report.setdefault("adapter", "documents") + report.setdefault("source_adapter", "documents") + return report + + @staticmethod + def _document_upload_inputs( + files: list[tuple[str, bytes]], attachment_manifest: Optional[list[dict]], + ) -> tuple[list[tuple[str, bytes]], list[dict]]: + """Validate mixed document bytes and content-free attachment metadata.""" + if not isinstance(files, list) or not files or len(files) > MAX_IMPORT_FILES: + raise ValidationError(f"files must contain 1 to {MAX_IMPORT_FILES} uploads") + from engraphis.core.documents import normalize_document_path + + uploads: list[tuple[str, bytes]] = [] + upload_paths: set[str] = set() + total_bytes = 0 + for entry in files: + if not isinstance(entry, tuple) or len(entry) != 2: + raise ValidationError("each upload must contain a path and bytes") + relative_path, raw = entry + if not isinstance(relative_path, str) or not isinstance(raw, bytes): + raise ValidationError("each upload must contain a path and bytes") + try: + path = normalize_document_path(relative_path) + except ValueError: + raise ValidationError("upload contains an invalid path") from None + path_key = path.casefold() + if path_key in upload_paths: + raise ValidationError("upload contains a duplicate path") + upload_paths.add(path_key) + if len(raw) > MAX_IMPORT_RESOURCE_BYTES: + raise ValidationError("upload contains a file that is too large") + total_bytes += len(raw) + if total_bytes > MAX_IMPORT_TOTAL_BYTES: + raise ValidationError("uploads exceed the total size limit") + uploads.append((path, raw)) + + manifest = [] if attachment_manifest is None else attachment_manifest + if not isinstance(manifest, list) or len(manifest) > MAX_IMPORT_FILES * 20: + raise ValidationError("attachment_manifest is invalid") + attachments: list[dict] = [] + attachment_paths: set[str] = set() + for entry in manifest: + if not isinstance(entry, dict): + raise ValidationError("attachment_manifest is invalid") + raw_path = entry.get("path") + if not isinstance(raw_path, str): + raise ValidationError("attachment_manifest contains an invalid path") + try: + path = normalize_document_path(raw_path) + except ValueError: + raise ValidationError("attachment_manifest contains an invalid path") from None + path_key = path.casefold() + if path_key in attachment_paths: + raise ValidationError("attachment_manifest contains a duplicate path") + attachment_paths.add(path_key) + size = entry.get("size") + if ( + isinstance(size, bool) or not isinstance(size, int) + or not 0 <= size <= MAX_IMPORT_RESOURCE_BYTES + ): + raise ValidationError("attachment_manifest contains an invalid size") + attachments.append({"path": path, "size": size}) + if upload_paths.intersection(attachment_paths): + raise ValidationError("upload and attachment paths overlap") + return uploads, attachments + + def preview_document_tree( + self, path: str, *, workspace: str, repo: Optional[str] = None, + session_id: Optional[str] = None, scope: Optional[str] = None, + memory_type: str = "semantic", source_id: Optional[str] = None, + source_label: str = "", on_conflict: str = "error", + ) -> dict: + """Scan and plan a mixed local document collection without Store writes.""" + from engraphis.core.documents import scan_document_tree + from engraphis.document_import import DocumentImporter, local_document_adapter + + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + policy = self._obsidian_conflict_policy(on_conflict) + source_id = self._document_registered_target( + source_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + scan = scan_document_tree(path, adapter=local_document_adapter) + label = self._document_label(source_label or Path(path).name) + report = DocumentImporter(self).preview( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, source_id=source_id, + source_label=label, on_conflict=policy, + manifest={"vaults": [], "items": []} if wid is None else None, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return self._document_report(report) + + def import_document_tree( + self, path: str, *, workspace: str, repo: Optional[str] = None, + session_id: Optional[str] = None, scope: Optional[str] = None, + memory_type: str = "semantic", source_id: Optional[str] = None, + source_label: str = "", on_conflict: str = "error", + confirmed: bool = False, actor: str = "local_cli_operator", + cancel_check=None, progress=None, _scan=None, + ) -> dict: + """Import mixed local documents synchronously and atomically per source file.""" + from engraphis.core.documents import scan_document_tree + from engraphis.document_import import DocumentImporter, local_document_adapter + + if confirmed is not True: + raise ValidationError("trusted-local confirmation is required") + policy = self._obsidian_conflict_policy(on_conflict) + source_id = self._document_registered_target( + source_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + # The local CLI passes its already-previewed immutable scan so confirmation + # applies to exactly those bytes. Other callers receive the same secure scan + # here, immediately before target creation. + scan = _scan or scan_document_tree(path, adapter=local_document_adapter) + label = self._document_label(source_label or Path(path).name) + clean_actor = _clean_text(actor, field="actor", max_chars=MAX_NAME_CHARS) + _reject_secret_capture((("actor", clean_actor),)) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=True, + ) + if wid is None: + raise RuntimeError("workspace creation failed") + report = DocumentImporter(self).import_scan( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, source_id=source_id, + source_label=label, on_conflict=policy, confirmed=True, + actor=clean_actor, strict_root=True, + cancel_check=cancel_check, progress=progress, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return self._document_report(report) + + def preview_document_upload( + self, *, files: list[tuple[str, bytes]], + attachment_manifest: Optional[list[dict]], workspace: str, + repo: Optional[str] = None, session_id: Optional[str] = None, + scope: Optional[str] = None, memory_type: str = "semantic", + source_id: Optional[str] = None, source_label: str = "", + on_conflict: str = "error", confirmed: bool = False, + ) -> dict: + """Preview browser-selected mixed document bytes without persisting a copy.""" + del confirmed + from engraphis.document_import import DocumentImporter, scan_document_upload + + policy = self._obsidian_conflict_policy(on_conflict) + source_id = self._document_registered_target( + source_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + uploads, attachments = self._document_upload_inputs(files, attachment_manifest) + label = self._document_label(source_label, source_id=source_id) + if source_id is None: + self._require_new_document_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + scan = scan_document_upload(uploads, source_label=label) + report = DocumentImporter(self).preview( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, source_id=source_id, + source_label=label, on_conflict=policy, strict_root=False, + attachment_manifest=attachments, + manifest={"vaults": [], "items": []} if wid is None else None, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return self._document_report(report) + + def import_document_upload( + self, *, files: list[tuple[str, bytes]], + attachment_manifest: Optional[list[dict]], workspace: str, + repo: Optional[str] = None, session_id: Optional[str] = None, + scope: Optional[str] = None, memory_type: str = "semantic", + source_id: Optional[str] = None, source_label: str = "", + on_conflict: str = "error", confirmed: bool = False, + ) -> dict: + """Start an owner-confirmed mixed-document import without upload persistence.""" + from engraphis.document_import import DocumentImporter, scan_document_upload + + if confirmed is not True: + raise ValidationError("trusted-local confirmation is required") + policy = self._obsidian_conflict_policy(on_conflict) + source_id = self._document_registered_target( + source_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + uploads, attachments = self._document_upload_inputs(files, attachment_manifest) + label = self._document_label(source_label, source_id=source_id) + if source_id is None: + self._require_new_document_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + scan = scan_document_upload(uploads, source_label=label) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=True, + ) + if wid is None: + raise RuntimeError("workspace creation failed") + importer = DocumentImporter(self) + prepared = importer.prepare_import( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, source_id=source_id, + source_label=label, on_conflict=policy, confirmed=True, + strict_root=False, + ) + job_id = str(prepared["job_id"]) + + def run() -> None: + try: + importer.import_scan( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, source_id=source_id, + source_label=label, on_conflict=policy, confirmed=True, + actor="dashboard_browser_session", strict_root=False, + attachment_manifest=attachments, prepared=prepared, + ) + except BaseException: + logger.exception("Document import worker failed before final reporting") + try: + self.store.conn.execute( + "UPDATE jobs SET state='failed', finished_at=?, heartbeat_at=? WHERE id=?", + (time.time(), time.time(), job_id), + ) + self.store.conn.commit() + except Exception: + logger.exception("Document import worker finalization failed") + finally: + with self._graph_job_lock: + self._obsidian_job_threads.pop(job_id, None) + + worker = threading.Thread( + target=run, name=f"engraphis-document-import-{job_id[-8:]}", daemon=True, + ) + with self._graph_job_lock: + self._obsidian_job_threads[job_id] = worker + try: + worker.start() + except BaseException: + with self._graph_job_lock: + self._obsidian_job_threads.pop(job_id, None) + failed_at = time.time() + self.store.conn.execute( + "UPDATE jobs SET state='failed', finished_at=?, heartbeat_at=? " + "WHERE id=?", + (failed_at, failed_at, job_id), + ) + self.store.conn.commit() + raise + return { + "job_id": job_id, "id": job_id, "state": "running", "status": "running", + "source_id": prepared.get("source_id", prepared["vault_id"]), + "adapter": "documents", "source_adapter": "documents", + "workspace": ws, "repo": repo, + "total_items": len(scan.documents) + len(scan.rejected) + len(scan.skipped), + } + + def list_document_sources(self, workspace: str) -> list[dict]: + """List universal and legacy Markdown source identities for one workspace.""" + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + if wid is None: + return [] + result = [] + for row in self.store.list_source_vaults(workspace_id=wid): + if row.get("kind") not in {"documents", "obsidian"}: + continue + repo_name = None + if row.get("repo_id"): + repo_row = self.store.conn.execute( + "SELECT name FROM repos WHERE id=?", (row["repo_id"],), + ).fetchone() + repo_name = str(repo_row["name"]) if repo_row else None + result.append({ + "id": row["id"], + "label": row.get("display_name") or "Local documents", + "kind": row.get("kind"), + "adapter": "obsidian" if row.get("kind") == "obsidian" else "documents", + "formats": row.get("formats") or {}, + "workspace": ws, "repo": repo_name, + "session_id": row.get("session_id"), "scope": row.get("scope"), + "memory_type": row.get("memory_type"), + "importer_version": row.get("importer_version"), + }) + return result + + def get_document_import_job(self, job_id: str, *, workspace: str) -> dict: + """Return a content-free universal or compatibility import report.""" + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + if wid is None: + raise KeyError(clean_id) + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=? AND workspace_id=? " + "AND kind IN ('document_import','obsidian_import')", + (clean_id, wid), + ).fetchone() + if row is None: + raise KeyError(clean_id) + items = self.store.list_source_import_job_items(job_id=clean_id) + files = [{ + "relative_path": item["relative_path"], + "status": item["result_state"], "action": item["planned_action"], + "reason": item.get("error_code") or "", + "warning_count": int(item.get("warning_count") or 0), + "format": item.get("source_format") or "", + } for item in items] + counts = _loads(row["counts"], {}) + return { + "id": clean_id, "job_id": clean_id, "workspace": ws, + "kind": row["kind"], "state": row["state"], "status": row["state"], + "total_items": int(row["total_items"]), + "processed_items": int(row["processed_items"]), + "counts": counts, "files": files, + "report": {"state": row["state"], "counts": counts, "files": files}, + } + + def cancel_document_import_job(self, job_id: str, *, workspace: str) -> dict: + """Request cancellation at the next per-document atomic boundary.""" + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + if wid is None: + raise KeyError(clean_id) + changed = self.store.conn.execute( + "UPDATE jobs SET cancel_requested=1 WHERE id=? AND workspace_id=? " + "AND kind IN ('document_import','obsidian_import') " + "AND state IN ('queued','running')", + (clean_id, wid), + ).rowcount + self.store.conn.commit() + if not changed: + row = self.store.conn.execute( + "SELECT state FROM jobs WHERE id=? AND workspace_id=? " + "AND kind IN ('document_import','obsidian_import')", (clean_id, wid), + ).fetchone() + if row is None: + raise KeyError(clean_id) + return {"id": clean_id, "state": row["state"], "cancel_requested": False} + return {"id": clean_id, "state": "running", "cancel_requested": True} + + # ── Obsidian compatibility import ──────────────────────────────────────── + def _obsidian_target(self, *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], + memory_type: str, create: bool) -> tuple[ + str, Optional[str], Optional[str], Optional[str], Scope, MemoryType + ]: + """Resolve an import target through the ordinary v2 hierarchy rules.""" + ws = self._clean_ws(workspace) + rp = _clean_name(repo, field="repo") if repo else None + mt = _enum(memory_type, MemoryType, "memory_type") + sc = _write_scope(scope, repo=rp, session_id=session_id) + if sc == Scope.USER: + raise ValidationError("user scope is read-only") + if sc == Scope.WORKSPACE and (rp or session_id): + raise ValidationError( + "workspace scope requires repo and session_id to be omitted" + ) + wid = self._get_or_create_workspace(ws) if create else self._lookup_workspace(ws) + if wid is None: + if session_id: + raise ValidationError("session scope requires an existing workspace") + return ws, None, None, None, sc, mt + rid = ( + self.store.get_or_create_repo(wid, rp) if create and rp + else self._lookup_repo(wid, rp) if rp else None + ) + if rp and rid is None: + return ws, wid, None, None, sc, mt + if session_id: + session = self._session_for_write(session_id, wid, rid) + if session is None: + raise ValidationError("session_id is required") + session_id = str(session["id"]) + if rid is None: + rid = session.get("repo_id") + if sc == Scope.REPO and rid is None: + raise ValidationError("repo scope requires a repo-backed session_id") + if sc == Scope.REPO: + # The session is used only to infer and validate its parent repo; + # repo-scoped source manifests must not retain a session target. + session_id = None + if sc == Scope.REPO and rid is None: + raise ValidationError("repo scope requires repo") + return ws, wid, rid, session_id, sc, mt + + def _obsidian_registered_target( + self, vault_id: Optional[str], *, workspace: str, repo: Optional[str], + session_id: Optional[str], scope: Optional[str], memory_type: str, + ) -> Optional[str]: + """Validate a selected vault before an import may create hierarchy rows. + + A new import is allowed to create its workspace/repository. Re-importing a + registered vault is different: its target already exists, and a misspelled or + cross-workspace request must fail without first creating attacker-controlled + hierarchy state. The importer performs the authoritative vault-row comparison; + this preflight makes that comparison mutation-free. + """ + if vault_id is None: + return None + clean_id = _clean_text( + vault_id, field="vault_id", max_chars=MAX_NAME_CHARS, + ) + if not clean_id.startswith("vlt_"): + raise ValidationError("registered vault was not found") + _ws, wid, rid, sid, selected_scope, selected_type = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + if wid is None or (repo is not None and rid is None): + raise ValidationError("registered vault was not found") + vault = self.store.get_source_vault(clean_id) + if ( + vault is None + or vault.get("kind") != "obsidian" + or vault.get("workspace_id") != wid + or vault.get("repo_id") != rid + or vault.get("session_id") != sid + ): + raise ValidationError("registered vault does not belong to that target") + if ( + vault.get("scope") != selected_scope.value + or vault.get("memory_type") != selected_type.value + ): + raise ValidationError("registered vault has different import defaults") + return clean_id + + def _obsidian_label(self, value: str, *, vault_id: Optional[str] = None) -> str: + label = unicodedata.normalize("NFC", _clean_text( + value, field="vault_label", max_chars=MAX_NAME_CHARS, required=False, + )) + if not label and vault_id: + vault = self.store.get_source_vault(vault_id) + label = str(vault.get("display_name") or "") if vault else "" + if not label: + raise ValidationError("vault_label is required for a new browser source") + _reject_secret_capture((("vault_label", label),)) + return label + + @staticmethod + def _obsidian_conflict_policy(value: str) -> str: + policy = _clean_text( + value, field="on_conflict", max_chars=MAX_NAME_CHARS, + ).casefold() + policy = {"report": "error", "supersede": "replace"}.get(policy, policy) + if policy not in {"error", "replace", "new"}: + raise ValidationError("on_conflict must be error, replace, or new") + return policy + + @staticmethod + def _obsidian_upload_inputs( + files: list[tuple[str, bytes]], attachment_manifest: Optional[list[dict]], + ) -> tuple[list[tuple[str, bytes]], list[dict]]: + """Apply transport-independent upload bounds and manifest validation.""" + if not isinstance(files, list) or not files or len(files) > MAX_IMPORT_FILES: + raise ValidationError(f"files must contain 1 to {MAX_IMPORT_FILES} uploads") + from engraphis.core.obsidian import ( + MAX_NOTE_BYTES, + MAX_VAULT_BYTES, + normalize_obsidian_path, + ) + + uploads: list[tuple[str, bytes]] = [] + upload_paths: set[str] = set() + total_bytes = 0 + for entry in files: + if not isinstance(entry, tuple) or len(entry) != 2: + raise ValidationError("each upload must contain a path and bytes") + relative_path, raw = entry + if not isinstance(relative_path, str) or not isinstance(raw, bytes): + raise ValidationError("each upload must contain a path and bytes") + if len(raw) > MAX_NOTE_BYTES: + raise ValidationError("upload contains a file that is too large") + total_bytes += len(raw) + if total_bytes > MAX_VAULT_BYTES: + raise ValidationError("uploads exceed the total size limit") + # Retain an unsafe path for the scanner to put in the content-free + # per-file preview report; only valid paths participate in the + # transport-level duplicate/overlap check. + try: + path = normalize_obsidian_path(relative_path) + except ValueError: + uploads.append((relative_path, raw)) + continue + path_key = path.casefold() + if path_key in upload_paths: + raise ValidationError("upload contains a duplicate path") + upload_paths.add(path_key) + uploads.append((path, raw)) + + manifest = [] if attachment_manifest is None else attachment_manifest + if not isinstance(manifest, list) or len(manifest) > MAX_IMPORT_FILES * 20: + raise ValidationError("attachment_manifest is invalid") + + attachments: list[dict] = [] + attachment_paths: set[str] = set() + for entry in manifest: + if not isinstance(entry, dict): + raise ValidationError("attachment_manifest is invalid") + raw_path = entry.get("path") + if not isinstance(raw_path, str): + raise ValidationError("attachment_manifest contains an invalid path") + try: + path = normalize_obsidian_path(raw_path) + except ValueError: + raise ValidationError("attachment_manifest contains an invalid path") from None + path_key = path.casefold() + if path_key in attachment_paths: + raise ValidationError("attachment_manifest contains a duplicate path") + attachment_paths.add(path_key) + size = entry.get("size") + if ( + isinstance(size, bool) or not isinstance(size, int) + or not 0 <= size <= MAX_IMPORT_RESOURCE_BYTES + ): + raise ValidationError("attachment_manifest contains an invalid size") + attachments.append({"path": path, "size": size}) + if upload_paths.intersection(attachment_paths): + raise ValidationError("upload and attachment paths overlap") + return uploads, attachments + + def preview_obsidian_vault(self, path: str, *, workspace: str, + repo: Optional[str] = None, + session_id: Optional[str] = None, + scope: Optional[str] = None, + memory_type: str = "semantic", + vault_id: Optional[str] = None, + vault_label: str = "", + on_conflict: str = "error") -> dict: + """Read and plan one local vault without mutating the Store.""" + from engraphis.core.obsidian import scan_obsidian_vault + from engraphis.obsidian_import import ObsidianImporter + + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + policy = self._obsidian_conflict_policy(on_conflict) + scan = scan_obsidian_vault(path) + label = self._obsidian_label(vault_label or Path(path).name) + vault_id = self._obsidian_registered_target( + vault_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + report = ObsidianImporter(self).preview( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, vault_id=vault_id, + vault_label=label, on_conflict=policy, + manifest={"vaults": [], "items": []} if wid is None else None, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return report + + def import_obsidian_vault(self, path: str, *, workspace: str, + repo: Optional[str] = None, + session_id: Optional[str] = None, + scope: Optional[str] = None, + memory_type: str = "semantic", + vault_id: Optional[str] = None, + vault_label: str = "", + on_conflict: str = "error", + confirmed: bool = False, + actor: str = "local_cli_operator", + cancel_check=None, progress=None, + _scan=None) -> dict: + """Import one filesystem vault synchronously and atomically per note.""" + from engraphis.core.obsidian import scan_obsidian_vault + from engraphis.obsidian_import import ObsidianImporter + + if confirmed is not True: + raise ValidationError("trusted-local confirmation is required") + policy = self._obsidian_conflict_policy(on_conflict) + vault_id = self._obsidian_registered_target( + vault_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + # The local CLI passes its already-previewed immutable scan so operator + # confirmation applies to exactly those bytes. Other callers are scanned + # here immediately before target creation. + scan = _scan or scan_obsidian_vault(path) + label = self._obsidian_label(vault_label or Path(path).name) + clean_actor = _clean_text(actor, field="actor", max_chars=MAX_NAME_CHARS) + _reject_secret_capture((("actor", clean_actor),)) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=True, + ) + if wid is None: + raise RuntimeError("workspace creation failed") + report = ObsidianImporter(self).import_scan( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, vault_id=vault_id, + vault_label=label, on_conflict=policy, + confirmed=True, actor=clean_actor, strict_root=True, + cancel_check=cancel_check, progress=progress, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return report + + def preview_obsidian_upload(self, *, files: list[tuple[str, bytes]], + attachment_manifest: Optional[list[dict]], + workspace: str, repo: Optional[str] = None, + session_id: Optional[str] = None, + scope: Optional[str] = None, + memory_type: str = "semantic", + vault_id: Optional[str] = None, + vault_label: str = "", + on_conflict: str = "error", + confirmed: bool = False) -> dict: + """Preview browser-selected bytes; ``confirmed`` is intentionally ignored.""" + del confirmed + from engraphis.obsidian_import import ObsidianImporter, scan_obsidian_upload + + policy = self._obsidian_conflict_policy(on_conflict) + vault_id = self._obsidian_registered_target( + vault_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=False, + ) + uploads, attachments = self._obsidian_upload_inputs(files, attachment_manifest) + label = self._obsidian_label(vault_label, vault_id=vault_id) + if vault_id is None: + self._require_new_obsidian_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + scan = scan_obsidian_upload(uploads, vault_label=label) + report = ObsidianImporter(self).preview( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, vault_id=vault_id, + vault_label=label, on_conflict=policy, + strict_root=False, attachment_manifest=attachments, + manifest={"vaults": [], "items": []} if wid is None else None, + ) + report["target"].update({"workspace": ws, "repo": repo}) + return report + + def import_obsidian_upload(self, *, files: list[tuple[str, bytes]], + attachment_manifest: Optional[list[dict]], + workspace: str, repo: Optional[str] = None, + session_id: Optional[str] = None, + scope: Optional[str] = None, + memory_type: str = "semantic", + vault_id: Optional[str] = None, + vault_label: str = "", + on_conflict: str = "error", + confirmed: bool = False) -> dict: + """Import owner-confirmed browser bytes without creating an upload copy.""" + from engraphis.obsidian_import import ObsidianImporter, scan_obsidian_upload + + if confirmed is not True: + raise ValidationError("trusted-local confirmation is required") + policy = self._obsidian_conflict_policy(on_conflict) + vault_id = self._obsidian_registered_target( + vault_id, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + uploads, attachments = self._obsidian_upload_inputs(files, attachment_manifest) + label = self._obsidian_label(vault_label, vault_id=vault_id) + if vault_id is None: + self._require_new_obsidian_label( + label, workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, + ) + scan = scan_obsidian_upload(uploads, vault_label=label) + ws, wid, rid, sid, sc, mt = self._obsidian_target( + workspace=workspace, repo=repo, session_id=session_id, + scope=scope, memory_type=memory_type, create=True, + ) + if wid is None: + raise RuntimeError("workspace creation failed") + importer = ObsidianImporter(self) + prepared = importer.prepare_import( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, vault_id=vault_id, + vault_label=label, on_conflict=policy, + confirmed=True, strict_root=False, + ) + job_id = str(prepared["job_id"]) + + def run() -> None: + try: + importer.import_scan( + scan, workspace_id=wid, repo_id=rid, session_id=sid, + scope=sc, memory_type=mt, vault_id=vault_id, + vault_label=label, on_conflict=policy, + confirmed=True, actor="dashboard_browser_session", + strict_root=False, attachment_manifest=attachments, + prepared=prepared, + ) + except BaseException: + logger.exception("Obsidian import worker failed before final reporting") + try: + self.store.conn.execute( + "UPDATE jobs SET state='failed', finished_at=?, heartbeat_at=? WHERE id=?", + (time.time(), time.time(), job_id), + ) + self.store.conn.commit() + except Exception: + logger.exception("Obsidian import worker finalization failed") + finally: + with self._graph_job_lock: + self._obsidian_job_threads.pop(job_id, None) + + worker = threading.Thread( + target=run, name=f"engraphis-obsidian-import-{job_id[-8:]}", daemon=True, + ) + with self._graph_job_lock: + self._obsidian_job_threads[job_id] = worker + try: + worker.start() + except BaseException: + with self._graph_job_lock: + self._obsidian_job_threads.pop(job_id, None) + failed_at = time.time() + self.store.conn.execute( + "UPDATE jobs SET state='failed', finished_at=?, heartbeat_at=? " + "WHERE id=?", + (failed_at, failed_at, job_id), + ) + self.store.conn.commit() + raise + return { + "job_id": job_id, "id": job_id, "state": "running", "status": "running", + "vault_id": prepared["vault_id"], "workspace": ws, "repo": repo, + "total_items": len(scan.notes) + len(scan.rejected) + len(scan.skipped), + } + + def list_obsidian_vaults(self, workspace: str) -> list[dict]: + """Return registered identities/defaults without exposing local root digests.""" + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + if wid is None: + return [] + result = [] + for row in self.store.list_source_vaults(workspace_id=wid, kind="obsidian"): + repo_name = None + if row.get("repo_id"): + repo_row = self.store.conn.execute( + "SELECT name FROM repos WHERE id=?", (row["repo_id"],), + ).fetchone() + repo_name = str(repo_row["name"]) if repo_row else None + result.append({ + "id": row["id"], "label": row.get("display_name") or "Obsidian vault", + "workspace": ws, "repo": repo_name, "session_id": row.get("session_id"), + "scope": row.get("scope"), "memory_type": row.get("memory_type"), + "importer_version": row.get("importer_version"), + }) + return result + + def get_obsidian_import_job(self, job_id: str, *, workspace: str) -> dict: + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + if wid is None: + raise KeyError(clean_id) + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=? AND workspace_id=? AND kind='obsidian_import'", + (clean_id, wid), + ).fetchone() + if row is None: + raise KeyError(clean_id) + items = self.store.list_source_import_job_items(job_id=clean_id) + files = [{ + "relative_path": item["relative_path"], + "status": item["result_state"], + "action": item["planned_action"], + "reason": item.get("error_code") or "", + "warning_count": int(item.get("warning_count") or 0), + "format": item.get("source_format") or "", + } for item in items] + return { + "id": clean_id, "job_id": clean_id, "workspace": ws, + "state": row["state"], "status": row["state"], + "total_items": int(row["total_items"]), + "processed_items": int(row["processed_items"]), + "counts": _loads(row["counts"], {}), "files": files, + "report": {"state": row["state"], "counts": _loads(row["counts"], {}), + "files": files}, + } + + def cancel_obsidian_import_job(self, job_id: str, *, workspace: str) -> dict: + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + if wid is None: + raise KeyError(clean_id) + changed = self.store.conn.execute( + "UPDATE jobs SET cancel_requested=1 WHERE id=? AND workspace_id=? " + "AND kind='obsidian_import' AND state IN ('queued','running')", + (clean_id, wid), + ).rowcount + self.store.conn.commit() + if not changed: + row = self.store.conn.execute( + "SELECT state FROM jobs WHERE id=? AND workspace_id=? " + "AND kind='obsidian_import'", (clean_id, wid), + ).fetchone() + if row is None: + raise KeyError(clean_id) + return {"id": clean_id, "state": row["state"], "cancel_requested": False} + return {"id": clean_id, "state": "running", "cancel_requested": True} + def import_postgres_schema(self, dsn: str, *, workspace: str, repo: Optional[str] = None, schemas: Optional[list] = None, @@ -3627,9 +4555,7 @@ def link_symbol(self, symbol_id: str, memory_id: str, *, workspace: str, repo: s symbol_id = _clean_text(symbol_id, field="symbol_id", max_chars=500) memory_id = _clean_text(memory_id, field="memory_id", max_chars=500) relation = _clean_name(relation, field="relation") or "mentions" - reason = _clean_text( - reason, field="reason", max_chars=MAX_TITLE_CHARS, required=False - ) + reason = _clean_text(reason, field="reason", max_chars=MAX_TITLE_CHARS, required=False) _reject_secret_capture((("link_symbol reason", reason),)) try: confidence = max(0.0, min(1.0, float(confidence))) @@ -3656,9 +4582,11 @@ def link_symbol(self, symbol_id: str, memory_id: str, *, workspace: str, repo: s receipt = self.store.record_receipt( "link", workspace_id=wid, repo_id=rid, actor=actor, target_count=1, status="ok", - metadata={"relation": relation, "result_count": 1, - "reason": reason[:200] if reason else "", - "symbol_id": symbol["id"], "memory_id": memory_id}, + metadata={ + "relation": relation, "result_count": 1, + "reason": reason[:200] if reason else "", + "symbol_id": symbol["id"], "memory_id": memory_id, + }, ) self.store.audit( actor, "link_symbol", link_id, @@ -3667,8 +4595,8 @@ def link_symbol(self, symbol_id: str, memory_id: str, *, workspace: str, repo: s ) return {"link_id": link_id, "symbol_id": symbol["id"], "memory_id": memory_id, "relation": relation, - "reason": reason, - "workspace": workspace, "repo": repo, "receipt": receipt} + "reason": reason, "workspace": workspace, "repo": repo, + "receipt": receipt} # ── inspection (powers the Memory Inspector UI) ───────────────────────────── def list_workspaces(self) -> dict: @@ -3930,6 +4858,12 @@ def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: pass # sqlite-vec vector table only present when that backend is active c.execute(f"DELETE FROM mem_links WHERE a IN {msub} OR b IN {msub}", (wid, wid)) c.execute("DELETE FROM memories WHERE workspace_id=?", (wid,)) + # These content-free sync/governance rows are not foreign-key cascades. A + # hard workspace deletion must remove them too, otherwise stale markers can + # later authorize a tombstone or block a newly created workspace's sync. + c.execute("DELETE FROM memory_sync_exports WHERE workspace_id=?", (wid,)) + c.execute("DELETE FROM memory_tombstones WHERE workspace_id=?", (wid,)) + c.execute("DELETE FROM maintenance_cursors WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM entities WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM edges WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM sessions WHERE workspace_id=?", (wid,)) @@ -4290,6 +5224,27 @@ def _new_repo(old_repo_id): f"WHERE workspace_id=? AND repo_id IS ?", (wid_dst, _new_repo(b["repo_id"]), wid_src, b["repo_id"])) + # Export proofs and remote-erasure markers survive memory re-homing so the + # next sync can still converge. Their repository owner follows the same + # collision map as the memories themselves. + for row in [dict(x) for x in c.execute( + "SELECT memory_id, repo_id FROM memory_sync_exports " + "WHERE workspace_id=?", (wid_src,))]: + c.execute( + "UPDATE memory_sync_exports SET workspace_id=?, repo_id=? " + "WHERE memory_id=?", + (wid_dst, _new_repo(row["repo_id"]), row["memory_id"]), + ) + for row in [dict(x) for x in c.execute( + "SELECT memory_id, repo_id FROM memory_tombstones " + "WHERE workspace_id=?", (wid_src,))]: + c.execute( + "UPDATE memory_tombstones SET workspace_id=?, repo_id=? " + "WHERE memory_id=?", + (wid_dst, _new_repo(row["repo_id"]), row["memory_id"]), + ) + c.execute("DELETE FROM maintenance_cursors WHERE workspace_id=?", (wid_src,)) + # 5) Receipt payload hashes bind their original workspace scope digest and chain # predecessor. Re-homing them would either forge that evidence or fork the target # chain, so remove the source-only ledger with the source workspace. The merge's @@ -4745,7 +5700,7 @@ def update_memory( and (importance is None or importance == existing.importance) ): return {"id": mid, "updated": []} - return self._update_memory_transactional( + result, external_index_action = self._update_memory_transactional( mid, workspace=workspace, repo=repo, @@ -4754,14 +5709,55 @@ def update_memory( importance=importance, actor=actor, ) + if external_index_action is not None: + self._publish_memory_index_action(external_index_action) + return result + def _publish_memory_index_action( + self, action: tuple[str, str, Optional[np.ndarray], str], + ) -> None: + """Publish one committed title edit to a separately-backed vector index. + + The Store row/vector/FTS state is canonical and has already committed when this + runs. A provider failure therefore becomes explicit repair debt; it must never + be raised as though the canonical edit had rolled back. + """ + operation, memory_id, vector, model = action + try: + if operation == "delete": + self.engine.index.delete([memory_id]) + elif operation == "upsert" and vector is not None: + self.engine.index.upsert( + [memory_id], vector.reshape(1, -1), [{"model": model}], + ) + else: # pragma: no cover - action is constructed locally + raise RuntimeError("invalid deferred vector-index action") + except Exception as exc: # noqa: BLE001 - canonical Store state is committed + failure_type = type(exc).__name__ + logger.warning( + "vector-index %s failed for title update %s (%s)", + operation, memory_id, failure_type, + ) + try: + self.store.audit( + "engine", f"index_{operation}_failed", memory_id, + f"failure_type={failure_type}", + commit=not self.store.conn.transaction_owned_by_current_thread(), + ) + except Exception as audit_exc: # noqa: BLE001 - retain original repair debt + logger.warning( + "could not audit title-update vector-index failure (%s)", + type(audit_exc).__name__, + ) @_rollback_service_transaction def _update_memory_transactional( self, memory_id: str, *, workspace: str, repo: Optional[str] = None, title: Optional[str] = None, mtype: Optional[str] = None, importance: Optional[float] = None, - actor: str = "user") -> dict: + actor: str = "user") -> tuple[ + dict, Optional[tuple[str, str, Optional[np.ndarray], str]] + ]: """In-place edit of a memory's metadata fields. Content edits go through ``correct`` so bi-temporal history is preserved.""" mid = _clean_text(memory_id, field="memory_id", max_chars=MAX_NAME_CHARS) @@ -4772,6 +5768,7 @@ def _update_memory_transactional( old_title = existing.title if existing is not None else "" sets, params, changes = [], [], [] title_changed = False + external_index_action = None if title is not None: title = _clean_text(title, field="title", max_chars=MAX_TITLE_CHARS, required=False) _reject_secret_capture((("title", title),)) @@ -4799,11 +5796,8 @@ def _update_memory_transactional( params.append(importance) changes.append("importance") if not sets and title is None: - return {"id": mid, "updated": []} + return {"id": mid, "updated": []}, None if sets: - # Descriptive fields participate in sync conflict ordering. Advance the - # hybrid logical clock in the same transaction as the direct SQL update so - # a peer cannot observe a changed title/type/importance at the old clock. self.store.advance_memory_modified_hlc(mid, commit=False) params.append(mid) self.store.conn.execute(f"UPDATE memories SET {', '.join(sets)} WHERE id=?", params) @@ -4826,7 +5820,12 @@ def _update_memory_transactional( ): self.store.conn.execute("DELETE FROM mem_vectors WHERE id=?", (mid,)) if vector_index_requires_sync(self.engine.index, self.store): - self.engine.index.delete([mid], commit=False) + if vector_index_shares_store_transaction( + self.engine.index, self.store, + ): + self.engine.index.delete([mid], commit=False) + else: + external_index_action = ("delete", mid, None, "") else: # Existing rows may predate the write-path secret guard. Do not send # such content to a remote embedder while changing unrelated metadata. @@ -4860,23 +5859,19 @@ def _update_memory_transactional( ): raise ValidationError("embedder returned an invalid vector") if vector_index_requires_sync(self.engine.index, self.store): - try: + if vector_index_shares_store_transaction( + self.engine.index, self.store, + ): self.engine.index.upsert( [mid], vectors, [{"model": model}], commit=False ) - except Exception as exc: # noqa: BLE001 — preserve mirror atomicity - logger.warning("vector-index upsert failed for title update %s (%s)", - mid, type(exc).__name__) - try: - self.store.audit( - "engine", "index_upsert_failed", mid, - "failure_type=%s" % type(exc).__name__, commit=False, - ) - except Exception: - pass - raise + else: + external_index_action = ( + "upsert", mid, vectors[0].copy(), model, + ) # Store owns the portable mirror for every backend. A separate - # index was synchronized above; NumPy searches this row directly. + # index is published only after this transaction commits; NumPy + # searches this canonical row directly. self.store.put_vector(mid, vectors[0], model=model) self.store._fts_upsert( mid, row["title"] or "", row["content"] or "", kw, @@ -4884,7 +5879,7 @@ def _update_memory_transactional( self.store.audit(actor, "memory_update", mid, "; ".join(changes)) self.store.conn.commit() - return {"id": mid, "updated": changes} + return {"id": mid, "updated": changes}, external_index_action @_rollback_service_transaction def reorder_memories(self, ids: list, *, workspace: str, repo: Optional[str] = None, actor: str = "user") -> dict: @@ -5210,20 +6205,29 @@ def context_savings( ) base["group_by"] = gb base["by_group"] = rows - if fmt == "csv": - import csv as _csv - import io as _io - buf = _io.StringIO() + if fmt == "csv": + import csv as _csv + import io as _io + buf = _io.StringIO() + if gb: fields = [ "group_key", "token_counter", "receipt_count", "source_tokens", "context_tokens", "saved_tokens", "budget_tokens", "packed_count", "omitted_count", "savings_ratio", ] - writer = _csv.DictWriter(buf, fieldnames=fields) - writer.writeheader() - for row in rows: - writer.writerow({k: row.get(k, "") for k in fields}) - base["csv"] = buf.getvalue() + rows = base.get("by_group", []) + else: + fields = [ + "token_counter", "receipt_count", "source_tokens", "context_tokens", + "saved_tokens", "budget_tokens", "packed_count", "omitted_count", + "savings_ratio", + ] + rows = base.get("by_token_counter", []) + writer = _csv.DictWriter(buf, fieldnames=fields) + writer.writeheader() + for row in rows: + writer.writerow({k: row.get(k, "") for k in fields}) + base["csv"] = buf.getvalue() return base def verify_receipts(self, *, workspace: str, expected_head: str = "", @@ -6058,6 +7062,7 @@ def start_graph_index_job(self, *, workspace: str, repo: Optional[str] = None, with self._graph_job_lock: if self._closing or self._closed: raise ValidationError("memory service is shutting down") + self._recover_stale_graph_jobs() self._graph_job_threads = { key: value for key, value in self._graph_job_threads.items() if value.is_alive() @@ -6066,7 +7071,6 @@ def start_graph_index_job(self, *, workspace: str, repo: Optional[str] = None, if owns_graph_txn: self.store.conn.execute("BEGIN IMMEDIATE") try: - self._recover_stale_graph_jobs() current_scope = self.store.conn.execute( "SELECT 1 FROM workspaces WHERE id=?", (wid,) ).fetchone() @@ -8717,36 +9721,66 @@ def memory_health(self, *, workspace: str) -> dict: # ── Decay distribution (retention buckets) ────────────────────────────── # R(t) = exp(-Δt_days / S). Bucket into 5 bands: critical (<0.2), low # (0.2–0.4), medium (0.4–0.6), high (0.6–0.8), strong (>0.8). - # SQLite's optional math extension is not available in every supported - # build. Fetch the indexed candidate columns and keep the calculation - # deterministic in Python instead of requiring SQLite's EXP(). - decay_rows = conn.execute( - f"SELECT last_access, ingested_at, stability FROM memories{live_where}", - live_params, - ).fetchall() - decay_counts = [0, 0, 0, 0, 0] - for row in decay_rows: - anchor = row["last_access"] or row["ingested_at"] or now - stability = max(float(row["stability"] or 0.0), 0.01) - elapsed_days = max(0.0, (now - float(anchor)) / 86400.0) - retention = math.exp(-elapsed_days / stability) - if retention < 0.2: - decay_counts[0] += 1 - elif retention < 0.4: - decay_counts[1] += 1 - elif retention < 0.6: - decay_counts[2] += 1 - elif retention < 0.8: - decay_counts[3] += 1 - else: - decay_counts[4] += 1 - decay_distribution = [ - {"bucket": "critical", "label": "< 20%", "count": decay_counts[0]}, - {"bucket": "low", "label": "20–40%", "count": decay_counts[1]}, - {"bucket": "medium", "label": "40–60%", "count": decay_counts[2]}, - {"bucket": "high", "label": "60–80%", "count": decay_counts[3]}, - {"bucket": "strong", "label": "> 80%", "count": decay_counts[4]}, - ] + # Computed in SQL via CASE on the retention formula so this is one indexed + # scan, not a Python loop over every memory. + decay_sql = f""" + SELECT + SUM(CASE WHEN ret < 0.2 THEN 1 ELSE 0 END) AS critical, + SUM(CASE WHEN ret >= 0.2 AND ret < 0.4 THEN 1 ELSE 0 END) AS low, + SUM(CASE WHEN ret >= 0.4 AND ret < 0.6 THEN 1 ELSE 0 END) AS medium, + SUM(CASE WHEN ret >= 0.6 AND ret < 0.8 THEN 1 ELSE 0 END) AS high, + SUM(CASE WHEN ret >= 0.8 THEN 1 ELSE 0 END) AS strong + FROM ( + SELECT EXP( + -MAX(0, (? - COALESCE(last_access, ingested_at, ?)) / 86400.0) + / MAX(stability, 0.01) + ) AS ret + FROM memories{live_where} + ) + """ + try: + decay_row = conn.execute(decay_sql, [now, now, *live_params]).fetchone() + except Exception: # noqa: BLE001 — SQLite may lack SQLITE_ENABLE_MATH_FUNCTIONS + # EXP() is an optional SQLite math function. On builds compiled without + # it (or on SQLCipher), fall back to a portable Python computation so + # memory_health() keeps working everywhere. + decay_ret_sql = f""" + SELECT + MAX(0, (? - COALESCE(last_access, ingested_at, ?)) / 86400.0) + / MAX(stability, 0.01) AS days_ratio + FROM memories{live_where} + """ + ratios = [float(r["days_ratio"]) for r in conn.execute( + decay_ret_sql, [now, now, *live_params] + ).fetchall()] + buckets = {"critical": 0, "low": 0, "medium": 0, "high": 0, "strong": 0} + for ratio in ratios: + retention = math.exp(-ratio) + if retention < 0.2: + buckets["critical"] += 1 + elif retention < 0.4: + buckets["low"] += 1 + elif retention < 0.6: + buckets["medium"] += 1 + elif retention < 0.8: + buckets["high"] += 1 + else: + buckets["strong"] += 1 + decay_distribution = [ + {"bucket": "critical", "label": "< 20%", "count": buckets["critical"]}, + {"bucket": "low", "label": "20–40%", "count": buckets["low"]}, + {"bucket": "medium", "label": "40–60%", "count": buckets["medium"]}, + {"bucket": "high", "label": "60–80%", "count": buckets["high"]}, + {"bucket": "strong", "label": "> 80%", "count": buckets["strong"]}, + ] + else: + decay_distribution = [ + {"bucket": "critical", "label": "< 20%", "count": int(decay_row["critical"] or 0)}, + {"bucket": "low", "label": "20–40%", "count": int(decay_row["low"] or 0)}, + {"bucket": "medium", "label": "40–60%", "count": int(decay_row["medium"] or 0)}, + {"bucket": "high", "label": "60–80%", "count": int(decay_row["high"] or 0)}, + {"bucket": "strong", "label": "> 80%", "count": int(decay_row["strong"] or 0)}, + ] # ── Orphan count (memories with no entity links) ──────────────────────── # A memory is an orphan when it has zero live rows in memory_entities. # The NOT EXISTS subquery uses the existing idx_memory_entity_memory diff --git a/engraphis/static/dashboard.js b/engraphis/static/dashboard.js index d1333fff..76dd4ad0 100644 --- a/engraphis/static/dashboard.js +++ b/engraphis/static/dashboard.js @@ -427,7 +427,7 @@ async function loadReceipts(){const el=document.getElementById('audit-body');el. async function downloadReceipts(){try{const d=await api('/receipts/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-receipts-'+(WS||'workspace')+'.json';a.click();URL.revokeObjectURL(a.href);toast('Privacy-safe receipts exported','ok')}catch(e){toast(e.message,'err')}} let SAVINGS_PRESET='all'; -function savingsPresetQuery(){const p=new URLSearchParams({workspace:WS||''});if(SAVINGS_PRESET==='current')p.set('release_version','1.5.0');if(SAVINGS_PRESET==='7d')p.set('from_ts',String(Date.now()/1000-604800));return p.toString()} +function savingsPresetQuery(){const p=new URLSearchParams({workspace:WS||''});if(SAVINGS_PRESET==='current')p.set('release_version','1.6');if(SAVINGS_PRESET==='7d')p.set('from_ts',String(Date.now()/1000-604800));return p.toString()} function renderSavingsDetail(s){const e=(s&&s.estimated)||{},eligible=Number(e.eligible_receipt_count)||0,excluded=(Number(e.excluded_receipt_count)||0)+(Number(e.unclassified_receipt_count)||0)+(Number(e.invalid_estimate_count)||0),basisRows=(e.by_basis||[]).map(x=>'
'+esc((x.basis||'unclassified').replaceAll('_',' '))+' · '+esc(x.confidence||'unknown')+''+formatTokenCount(x.baseline_tokens)+' → '+formatTokenCount(x.emitted_tokens)+' · '+formatTokenCount(x.saved_tokens)+' saved ('+(x.receipt_count||0)+' delivery)
').join(''),counterRows=(e.by_token_counter||[]).map(x=>'
'+esc(x.token_counter||'unknown')+''+formatTokenCount(x.saved_tokens)+' saved · '+(x.receipt_count||0)+' eligible delivery
').join(''),preset=SAVINGS_PRESET==='current'?'Current release':SAVINGS_PRESET==='7d'?'Last 7 days':SAVINGS_PRESET==='since'?'Since tracking started':'All time';const buttons=['since','current','7d','all'].map(x=>'').join('');return '
Estimated context saved
View'+buttons+'
'+(eligible?'
'+formatTokenCount(e.saved_tokens)+' tokens
Baseline '+formatTokenCount(e.baseline_tokens)+' → emitted '+formatTokenCount(e.emitted_tokens)+' · '+(Number(e.savings_ratio||0)*100).toFixed(1)+'% estimated reduction
'+eligible+' eligible deliveries · confidence: '+esc(e.confidence||'unknown')+' · range: '+preset+'
'+(basisRows||'
No basis breakdown available.
')+(counterRows?'
Token counters
'+counterRows:''):'
No eligible estimates in this range.
')+'
'+excluded+' excluded or unclassified delivery(s). Measures estimated prompt-context reduction; it does not measure provider billing.
'} async function loadReceipts(){const el=document.getElementById('audit-body');el.innerHTML='';try{if(!window.__savingsPresetBound){window.__savingsPresetBound=true;document.addEventListener('click',function(ev){const button=ev.target.closest('[data-savings-preset]');if(!button)return;SAVINGS_PRESET=button.getAttribute('data-savings-preset')||'all';loadReceipts()})}const q='workspace='+encodeURIComponent(WS||''),sq=savingsPresetQuery();const [d,v,s]=await Promise.all([api('/receipts?'+q+'&limit=500'),api('/receipts/verify?'+q),api('/context-savings?'+sq)]);const rows=d.entries||[],packed=(s.by_token_counter||[]).map(x=>'
'+esc(x.token_counter||'unknown')+''+formatTokenCount(x.context_tokens)+' packed / '+formatTokenCount(x.source_tokens)+' source · '+formatTokenCount(x.saved_tokens)+' legacy saved
').join('');const packedCard='
Packed context accounting
Packing savings compare retrieved source tokens with emitted context. They are not added again to adaptive history savings.
'+(packed||'
No complete context-usage receipts yet.
')+'
';el.innerHTML=renderSavingsDetail(s)+packedCard+'
Receipt chain '+(v.valid?'verified':'invalid')+'
'+(v.count||0)+' receipts · head '+esc((v.head||'').slice(0,24))+'
'+(rows.length?'
'+rows.map(r=>'
'+esc(r.operation||'operation')+''+esc((r.hash||'').slice(0,20))+' · '+esc(r.status||'ok')+' · '+(r.target_count||0)+' target(s)'+(r.ts_ms?fmtRel(r.ts_ms/1000):'')+'
').join('')+'
':'
No receipts yet.
')}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} diff --git a/integrations/hermes/engraphis/plugin.yaml b/integrations/hermes/engraphis/plugin.yaml index 530bda19..d48b0c39 100644 --- a/integrations/hermes/engraphis/plugin.yaml +++ b/integrations/hermes/engraphis/plugin.yaml @@ -1,5 +1,5 @@ name: engraphis -version: 1.5.0 +version: 1.6.0 description: "Engraphis local memory provider with scoped recall and bounded turn history." pip_dependencies: [] requires_env: [] diff --git a/pyproject.toml b/pyproject.toml index a774fb15..053924ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "engraphis" -version = "1.5" +version = "1.6" description = "Local-first AI memory engine for agents — Ebbinghaus decay, interaction-aware recall, bi-temporal facts, hybrid retrieval, and an MCP server. You bring the LLM." readme = "README.md" license = "Apache-2.0" @@ -218,6 +218,7 @@ engraphis-dashboard = "scripts.start_dashboard:main" engraphis-consolidate = "scripts.consolidate:main" engraphis-graph = "scripts.graph_cli:main" engraphis-graph-server = "scripts.graph_server:main" +engraphis-import = "scripts.importer:main" engraphis-init = "scripts.init:main" engraphis-update = "scripts.update:main" @@ -261,6 +262,8 @@ select = ["E4", "E7", "E9", "F"] include = [ "engraphis/core", "engraphis/backends", + "engraphis/factory.py", + "engraphis/__init__.py", "eval/harness.py", "eval/external.py", ] diff --git a/scripts/entry.py b/scripts/entry.py index cc3492d4..ddf8281b 100644 --- a/scripts/entry.py +++ b/scripts/entry.py @@ -32,6 +32,7 @@ "consolidate": "scripts.consolidate:main", "graph": "scripts.graph_cli:main", "graph-server": "scripts.graph_server:main", + "import": "scripts.importer:main", "update": "scripts.update:main", } @@ -50,6 +51,7 @@ consolidate run consolidation over stored memories graph query the knowledge graph graph-server run the graph server + import import local Markdown, text, and document collections update check for and install a newer Engraphis release Run `engraphis --help` for a command's options. diff --git a/scripts/importer.py b/scripts/importer.py new file mode 100644 index 00000000..48af4d7e --- /dev/null +++ b/scripts/importer.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""Offline-first v2 source importer command.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import sys +from typing import Optional + +from engraphis.config import settings +from engraphis.core.documents import scan_document_tree +from engraphis.core.interfaces import MemoryType, Scope +from engraphis.core.obsidian import scan_obsidian_vault +from engraphis.core.store import Store +from engraphis.document_import import DocumentImporter, local_document_adapter +from engraphis.obsidian_import import ObsidianImporter +from engraphis.service import MemoryService + + +_CONSOLE_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") + + +def _console(value: object, *, file=None) -> None: + """Print untrusted source labels without control injection or codec failures.""" + stream = file or sys.stdout + rendered = _CONSOLE_CONTROL_RE.sub( + lambda match: "\\x%02x" % ord(match.group(0)), str(value), + ) + encoding = getattr(stream, "encoding", None) or "utf-8" + rendered = rendered.encode(encoding, errors="backslashreplace").decode(encoding) + print(rendered, file=stream) + + +def _json(value: object) -> None: + # JSON escapes stay lossless on Windows consoles using a legacy charmap. + print(json.dumps( + value, ensure_ascii=True, sort_keys=True, default=str, allow_nan=False, + )) + + +def _nonnegative_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("must be an integer") from exc + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="engraphis import", description="Import a local source into Engraphis v2.", + ) + sub = parser.add_subparsers(dest="source", required=True) + documents = sub.add_parser( + "documents", + help="import a mixed local folder of Markdown, text, and documents", + ) + _source_arguments(documents, path_help="path to the local document collection") + documents.add_argument( + "--source-id", help="reuse a registered vlt_ source collection identity", + ) + documents.add_argument( + "--source-label", default="", help="display label for a new source collection", + ) + + obsidian = sub.add_parser( + "obsidian", help="import an Obsidian Markdown vault (compatibility command)", + ) + _source_arguments(obsidian, path_help="path to the Obsidian vault") + obsidian.add_argument("--vault-id", help="reuse a registered vlt_ identity") + obsidian.add_argument("--vault-label", default="", help="display label for a new vault") + return parser + + +def _source_arguments(parser: argparse.ArgumentParser, *, path_help: str) -> None: + """Add the transport-neutral v2 import options shared by all adapters.""" + parser.add_argument("path", help=path_help) + parser.add_argument("--db", default=settings.db_path, help="v2 database path") + parser.add_argument("--workspace", help="target workspace name") + parser.add_argument( + "--repo", + help="target repository name (must match --session when both are supplied)", + ) + parser.add_argument( + "--session", dest="session_id", + help="active target session ID (defaults the import scope to session)", + ) + parser.add_argument("--scope", choices=("workspace", "repo", "session")) + parser.add_argument( + "--memory-type", default="semantic", + choices=("working", "episodic", "semantic", "procedural"), + ) + parser.add_argument("--dry-run", action="store_true", help="preview with zero database writes") + parser.add_argument( + "--on-conflict", default="error", choices=("error", "replace", "new"), + help="divergent lineage policy (default: error)", + ) + parser.add_argument("--yes", action="store_true", help="confirm this trusted-local import") + parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + parser.add_argument( + "--limit", type=_nonnegative_int, default=0, metavar="N", + help=( + "process at most N documents in this run, then " + "leave it resumable (0 = all)" + ), + ) + + +def _scope(args: argparse.Namespace) -> Scope: + scope = Scope(args.scope) if args.scope else ( + Scope.SESSION if args.session_id + else Scope.REPO if args.repo + else Scope.WORKSPACE + ) + if scope == Scope.SESSION and not args.session_id: + raise ValueError("session scope requires --session") + if scope == Scope.REPO and not (args.repo or args.session_id): + raise ValueError("repo scope requires --repo (or a repo-backed --session)") + if scope == Scope.WORKSPACE and (args.repo or args.session_id): + raise ValueError("workspace scope requires --repo and --session to be omitted") + return scope + + +def _workspace(args: argparse.Namespace, source: Path) -> str: + if args.workspace: + workspace = str(args.workspace).strip() + if not workspace: + raise ValueError("--workspace must not be blank") + return workspace + if sys.stdin.isatty(): + default = source.name or "documents" + entered = input(f"Target workspace [{default}]: ").strip() + return entered or default + raise ValueError("--workspace is required in non-interactive mode") + + +def _snapshot_target( + snapshot: dict, *, root_digest: str, workspace: str, + repo: Optional[str], session_id: Optional[str], vault_id: Optional[str], + source_kind: str, +) -> tuple[Optional[dict], Optional[str], Optional[str]]: + vaults = list(snapshot.get("vaults") or []) + if vault_id: + selected = next((row for row in vaults if row.get("id") == vault_id), None) + if selected is None: + raise ValueError("registered source was not found in the read-only manifest") + if selected.get("kind") != source_kind: + raise ValueError("registered source uses a different import adapter") + if selected.get("workspace_name") != workspace: + raise ValueError("registered vault belongs to another workspace") + if (selected.get("repo_name") or None) != repo or selected.get("session_id") != session_id: + raise ValueError("registered vault has a different target scope") + if selected.get("root_digest") != root_digest: + raise ValueError("selected path does not match the registered vault") + return selected, selected.get("workspace_id"), selected.get("repo_id") + matches = [ + row for row in vaults + if row.get("kind") == source_kind + and row.get("root_digest") == root_digest + and row.get("workspace_name") == workspace + and (row.get("repo_name") or None) == repo + and row.get("session_id") == session_id + ] + selected = matches[0] if len(matches) == 1 else None + return ( + selected, + selected.get("workspace_id") if selected else None, + selected.get("repo_id") if selected else None, + ) + + +def _effective_manifest_repo( + snapshot: dict, *, repo: Optional[str], session_id: Optional[str], +) -> Optional[str]: + """Use a session-backed manifest repository when the CLI omitted ``--repo``.""" + if repo is not None or session_id is None: + return repo + session_repos = { + str(row["repo_name"]) + for row in snapshot.get("vaults") or [] + if row.get("session_id") == session_id and row.get("repo_name") + } + session_repos.update( + str(row["repo_name"]) + for row in snapshot.get("sessions") or [] + if row.get("id") == session_id and row.get("repo_name") + ) + if len(session_repos) > 1: + raise ValueError("session maps to multiple repositories in the import manifest") + return next(iter(session_repos), None) + + +def _effective_manifest_target( + snapshot: dict, *, repo: Optional[str], session_id: Optional[str], scope: Scope, +) -> tuple[Optional[str], Optional[str]]: + """Normalize the manifest target the same way the service normalizes imports.""" + effective_repo = _effective_manifest_repo( + snapshot, repo=repo, session_id=session_id, + ) + effective_session = None if scope == Scope.REPO else session_id + return effective_repo, effective_session + + +def _preview(args: argparse.Namespace, scan, workspace: str) -> dict: + snapshot = _manifest_snapshot(args.db) + scope = _scope(args) + effective_repo, effective_session = _effective_manifest_target( + snapshot, repo=args.repo, session_id=args.session_id, scope=scope, + ) + selected, workspace_id, repo_id = _snapshot_target( + snapshot, root_digest=scan.vault_id, workspace=workspace, + repo=effective_repo, session_id=effective_session, vault_id=args.vault_id, + source_kind="obsidian", + ) + report = ObsidianImporter().preview( + scan, workspace_id=workspace_id, repo_id=repo_id, + session_id=effective_session, scope=scope, + memory_type=MemoryType(args.memory_type), + vault_id=selected.get("id") if selected else None, + vault_label=args.vault_label or Path(args.path).name, + on_conflict=args.on_conflict, + manifest=( + snapshot if selected is not None + else {"vaults": [], "items": []} + ), + ) + report["target"].update({"workspace": workspace, "repo": effective_repo}) + return report + + +def _preview_documents(args: argparse.Namespace, scan, workspace: str) -> dict: + snapshot = _manifest_snapshot(args.db) + scope = _scope(args) + effective_repo, effective_session = _effective_manifest_target( + snapshot, repo=args.repo, session_id=args.session_id, scope=scope, + ) + selected, workspace_id, repo_id = _snapshot_target( + snapshot, root_digest=scan.source_id, workspace=workspace, + repo=effective_repo, session_id=effective_session, vault_id=args.source_id, + source_kind="documents", + ) + report = DocumentImporter().preview( + scan, workspace_id=workspace_id, repo_id=repo_id, + session_id=effective_session, scope=scope, + memory_type=MemoryType(args.memory_type), + source_id=selected.get("id") if selected else None, + source_label=args.source_label or Path(args.path).name, + on_conflict=args.on_conflict, + manifest=( + snapshot if selected is not None + else {"vaults": [], "items": []} + ), + ) + report.setdefault("adapter", "documents") + report.setdefault("source_adapter", "documents") + report["target"].update({"workspace": workspace, "repo": effective_repo}) + return report + + +def _manifest_snapshot(db_path: str) -> dict: + """Read plaintext or SQLCipher manifests without migrations or sidecar writes.""" + from engraphis.backends.encrypted_db import connector_from_env + + return Store.snapshot_source_import_manifest( + db_path, connect=connector_from_env(), + ) + + +def _print_human(report: dict, *, heading: str) -> None: + counts = report.get("counts", {}) + summary = report.get("summary", {}) + _console("") + _console(heading) + _console( + "Documents: {documents} | import: {imported} | update: {updated} | " + "rename: {renamed} | skip: {skipped} | reject: {rejected} | " + "conflict: {conflict} | missing: {missing}".format( + **{ + "documents": int(counts.get("documents", counts.get("markdown", 0))), + **{key: int(counts.get(key, 0)) for key in ( + "imported", "updated", "renamed", "skipped", + "rejected", "conflict", "missing", + )}} + ) + ) + formats = summary.get("formats") or {} + format_text = ", ".join( + f"{name}: {count}" for name, count in sorted(formats.items()) + ) if isinstance(formats, dict) else "" + _console( + f"Detected: {len(summary.get('folders', []))} folders" + + (f", formats [{format_text}]" if format_text else "") + ", " + f"{len(summary.get('tags', []))} tags, {summary.get('aliases', 0)} aliases, " + f"{summary.get('wikilinks', 0)} wikilinks, " + f"{summary.get('attachments', 0)} attachment references, " + f"{summary.get('warnings', 0)} warnings" + ) + for row in report.get("files", []): + status = str(row.get("status") or "reported").upper() + path = row.get("relative_path") or "(vault)" + reason = row.get("reason") or "" + detected_format = row.get("format") or row.get("source_format") or "" + format_suffix = f" [{detected_format}]" if detected_format else "" + _console( + f" {status:9} {path}{format_suffix}" + + (f" — {reason}" if reason else "") + ) + for warning in row.get("warnings", [])[:5]: + _console(f" warning: {warning}") + + +def _local_service(db_path: str) -> MemoryService: + embed_model = str(settings.embed_model or "").strip() + if embed_model and not embed_model.startswith("local:"): + embed_model = "local:" + embed_model + service = MemoryService.create( + db_path, embed_model=embed_model or None, + embed_revision=getattr(settings, "embed_revision", "") or None, + require_immutable_models=bool(getattr(settings, "require_immutable_models", False)), + embed_dim=settings.embed_dim or 384, vector_backend=settings.vector_backend, + rerank_model=None, allowed_workspaces=settings.allowed_workspaces, + extractor="none", graph_extractor="none", retention_supervisor="none", + ) + if embed_model: + from engraphis.backends.embedder_st import LAST_EMBEDDER_ERROR + + if LAST_EMBEDDER_ERROR: + service.close() + raise RuntimeError( + "the configured embedding model is not available in the local cache; " + "the importer will not download it" + ) + return service + + +def _confirm(args: argparse.Namespace) -> bool: + if args.yes: + return True + if args.json: + raise ValueError("JSON-mode imports require --yes") + if not sys.stdin.isatty(): + raise ValueError("non-interactive imports require --yes") + answer = input( + "Import these local documents as trusted canonical memories? [y/N]: " + ).strip().casefold() + return answer in {"y", "yes"} + + +def _run_obsidian(args: argparse.Namespace) -> int: + vault = Path(args.path).expanduser() + workspace = _workspace(args, vault) + scan = scan_obsidian_vault(vault) + preview = _preview(args, scan, workspace) + if args.dry_run: + if args.json: + _json(preview) + else: + _print_human(preview, heading="Obsidian import preview (no writes)") + counts = preview.get("counts", {}) + return 3 if counts.get("conflict") or counts.get("rejected") else 0 + if not args.json: + _print_human(preview, heading="Obsidian import preview") + effective_limit = args.limit if 0 < args.limit < len(scan.notes) else 0 + if effective_limit and not args.json: + _console( + f"Legacy limit: this run will pause after {effective_limit} notes. " + "Rerun without --limit to finish reconciliation." + ) + if not _confirm(args): + if not args.json: + _console("Import cancelled; no memories were written.") + return 130 + + service = _local_service(args.db) + progress_count = 0 + + def progress(row: dict) -> None: + nonlocal progress_count + progress_count += 1 + if not args.json: + _console( + f"[{progress_count}/{len(scan.notes)}] " + f"{str(row.get('status', 'reported')).upper():9} " + f"{row.get('relative_path', '')}" + ) + + try: + report = service.import_obsidian_vault( + str(vault), workspace=workspace, repo=args.repo, + session_id=args.session_id, scope=_scope(args).value, + memory_type=args.memory_type, vault_id=args.vault_id, + vault_label=args.vault_label or vault.name, + on_conflict=args.on_conflict, confirmed=True, + actor="local_cli_operator", progress=progress, + _scan=scan, + cancel_check=( + (lambda: progress_count >= effective_limit) + if effective_limit else None + ), + ) + finally: + service.close() + if args.json: + payload = {"preview": preview, "report": report} + if args.limit: + payload["limit"] = { + "requested": args.limit, + "processed": progress_count, + "reached": bool(effective_limit and progress_count >= effective_limit), + } + _json(payload) + else: + _print_human(report, heading=f"Obsidian import {report.get('state', 'complete')}") + if effective_limit and progress_count >= effective_limit: + _console("Paused at --limit; the import remains resumable.") + if ( + effective_limit + and progress_count >= effective_limit + and str(report.get("state")) == "cancelled" + ): + return 3 + return { + "completed": 0, "partial": 3, "failed": 3, "cancelled": 130, + }.get(str(report.get("state")), 3) + + +def _run_documents(args: argparse.Namespace) -> int: + root = Path(args.path).expanduser() + workspace = _workspace(args, root) + scan = scan_document_tree(root, adapter=local_document_adapter) + preview = _preview_documents(args, scan, workspace) + if args.dry_run: + if args.json: + _json(preview) + else: + _print_human(preview, heading="Document import preview (no writes)") + counts = preview.get("counts", {}) + return 3 if counts.get("conflict") or counts.get("rejected") else 0 + if not args.json: + _print_human(preview, heading="Document import preview") + documents = list(getattr(scan, "documents", ())) + effective_limit = args.limit if 0 < args.limit < len(documents) else 0 + if effective_limit and not args.json: + _console( + f"This run will pause after {effective_limit} documents. " + "Rerun without --limit to finish reconciliation." + ) + if not _confirm(args): + if not args.json: + _console("Import cancelled; no memories were written.") + return 130 + + service = _local_service(args.db) + progress_count = 0 + + def progress(row: dict) -> None: + nonlocal progress_count + progress_count += 1 + if not args.json: + _console( + f"[{progress_count}/{len(documents)}] " + f"{str(row.get('status', 'reported')).upper():9} " + f"{row.get('relative_path', '')}" + ) + + try: + report = service.import_document_tree( + str(root), workspace=workspace, repo=args.repo, + session_id=args.session_id, scope=_scope(args).value, + memory_type=args.memory_type, source_id=args.source_id, + source_label=args.source_label or root.name, + on_conflict=args.on_conflict, confirmed=True, + actor="local_cli_operator", progress=progress, + _scan=scan, + cancel_check=( + (lambda: progress_count >= effective_limit) + if effective_limit else None + ), + ) + finally: + service.close() + if args.json: + payload = {"preview": preview, "report": report} + if args.limit: + payload["limit"] = { + "requested": args.limit, + "processed": progress_count, + "reached": bool(effective_limit and progress_count >= effective_limit), + } + _json(payload) + else: + _print_human(report, heading=f"Document import {report.get('state', 'complete')}") + if effective_limit and progress_count >= effective_limit: + _console("Paused at --limit; the import remains resumable.") + if ( + effective_limit + and progress_count >= effective_limit + and str(report.get("state")) == "cancelled" + ): + return 3 + return { + "completed": 0, "partial": 3, "failed": 3, "cancelled": 130, + }.get(str(report.get("state")), 3) + + +def main(argv=None) -> int: + parser = _parser() + try: + args = parser.parse_args(argv) + if args.source == "documents": + return _run_documents(args) + if args.source == "obsidian": + return _run_obsidian(args) + parser.error("unsupported import source") + except KeyboardInterrupt: + _console("Import cancelled.", file=sys.stderr) + return 130 + except (OSError, RuntimeError, ValueError) as exc: + _console(f"engraphis import: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/migrate_to_v2.py b/scripts/migrate_to_v2.py index fa63ed64..e4c04949 100644 --- a/scripts/migrate_to_v2.py +++ b/scripts/migrate_to_v2.py @@ -552,6 +552,26 @@ def edge_entity_for(namespace: object, name: object) -> str: source_id = _source_id(row, vcols) if source_id is not None: refs.append({"kind": "v1_event_id", "id": source_id}) + entity_name = str( + (row["entity_name"] if "entity_name" in vcols else "") or "" + ).strip() + if entity_name: + refs.append({"kind": "v1_entity", "name": entity_name}) + if "payload" in vcols: + raw_payload = row["payload"] + try: + payload = json.loads(raw_payload or "{}") + except (TypeError, ValueError, RecursionError): + payload = str(raw_payload or "") + refs.append({"kind": "v1_payload", "value": payload}) + event_repairs = [] + event_ts = _legacy_float( + row["timestamp"] if "timestamp" in vcols else migration_time, + default=migration_time, + field="timestamp", + repairs=event_repairs, + ) + counts["repaired_fields"] += len(event_repairs) reject_secrets((("event content", content), ("event refs", refs))) if store is not None: store.append_event( @@ -560,6 +580,7 @@ def edge_entity_for(namespace: object, name: object) -> str: workspace_id=wid, repo_id=rid, refs=refs, + ts=event_ts, ) if _has_table(src, "thoughts"): diff --git a/scripts/seed_from_obsidian.py b/scripts/seed_from_obsidian.py index 2367ec85..53485254 100644 --- a/scripts/seed_from_obsidian.py +++ b/scripts/seed_from_obsidian.py @@ -1,98 +1,80 @@ -"""Seed the memory system from an Obsidian vault (or any folder of markdown files). +"""Deprecated compatibility wrapper for the v2 Obsidian importer. -Usage: - python -m scripts.seed_from_obsidian [--namespace vault] - python -m scripts.seed_from_obsidian "/path/to/obsidian-vault" - -Each .md file becomes a memory document with: - document_id = relative path (sanitized) - title = first H1 or filename - content = full file text - metadata = {file, tags, links, word_count} +Use ``engraphis import obsidian PATH --workspace NAME``. This module remains so +older local automation cannot accidentally fall back to the legacy v1 namespace +ingester. """ from __future__ import annotations import argparse -import re import sys -import time -from pathlib import Path - -from engraphis.engines import ingest as ingest_engine - - -def extract_title(content: str, fallback: str) -> str: - m = re.search(r"^#\s+(.+)$", content, re.MULTILINE) - return m.group(1).strip() if m else fallback - - -def extract_tags(content: str) -> list[str]: - return re.findall(r"#([a-zA-Z][a-zA-Z0-9_-]+)", content) - - -def extract_links(content: str) -> list[str]: - return re.findall(r"\[\[([^\]]+)\]\]", content) - - -def seed_vault(vault_path: str, namespace: str = "vault", limit: int = 0) -> dict: - vault = Path(vault_path) - if not vault.exists(): - print(f"ERROR: path does not exist: {vault}") - sys.exit(1) - - md_files = sorted(vault.rglob("*.md")) - if limit: - md_files = md_files[:limit] - - print(f"Seeding {len(md_files)} markdown files from {vault} → namespace='{namespace}'") - successful = 0 - errors = 0 - t0 = time.time() - - for i, fpath in enumerate(md_files): - try: - rel = fpath.relative_to(vault).as_posix() - doc_id = rel.replace("/", "__").replace(".md", "") - content = fpath.read_text(encoding="utf-8", errors="replace") - if not content.strip(): - continue - title = extract_title(content, fpath.stem) - tags = extract_tags(content) - links = extract_links(content) - - ingest_engine.ingest_document( - namespace=namespace, - document_id=doc_id, - title=title, - content=content, - source_type="obsidian", - metadata={ - "file": rel, - "tags": tags, - "links": links, - "word_count": len(content.split()), - }, - ) - successful += 1 - if (i + 1) % 50 == 0: - print(f" ... {i + 1}/{len(md_files)} ingested") - except Exception as e: - errors += 1 - print(f" ERROR on {fpath}: {e}") - - elapsed = time.time() - t0 - print(f"\nDone: {successful} ingested, {errors} errors, {elapsed:.1f}s") - return {"ingested": successful, "errors": errors, "elapsed_s": elapsed} -def main() -> None: - parser = argparse.ArgumentParser(description="Seed Engraphis from an Obsidian vault") - parser.add_argument("vault_path", help="Path to the vault folder (containing .md files)") - parser.add_argument("--namespace", default="vault", help="Namespace to store under (default: vault)") - parser.add_argument("--limit", type=int, default=0, help="Max files to ingest (0 = all)") - args = parser.parse_args() - seed_vault(args.vault_path, namespace=args.namespace, limit=args.limit) +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Deprecated: import an Obsidian vault into Engraphis v2.", + ) + parser.add_argument("vault_path", help="path to the Obsidian vault") + target = parser.add_mutually_exclusive_group() + target.add_argument( + "--namespace", default=None, + help="legacy namespace; maps to a v2 workspace", + ) + target.add_argument("--workspace", help="v2 target workspace") + parser.add_argument("--repo", help="v2 target repository") + parser.add_argument("--session", help="active v2 target session ID") + parser.add_argument("--scope", choices=("workspace", "repo", "session")) + parser.add_argument( + "--memory-type", default="semantic", + choices=("working", "episodic", "semantic", "procedural"), + ) + parser.add_argument("--db", help="v2 database path") + parser.add_argument("--limit", type=int, default=0, help="process at most N notes this run") + parser.add_argument( + "--on-conflict", default="error", choices=("error", "replace", "new"), + ) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + if args.limit < 0: + parser.error("--limit must be zero or greater") + print( + "warning: scripts.seed_from_obsidian is deprecated; " + "use `engraphis import obsidian`", + file=sys.stderr, + ) + from scripts.importer import main as importer_main + + forwarded = [ + "obsidian", + args.vault_path, + "--workspace", + args.workspace or args.namespace or "vault", + "--memory-type", + args.memory_type, + "--on-conflict", + args.on_conflict, + ] + for option, value in ( + ("--repo", args.repo), + ("--session", args.session), + ("--scope", args.scope), + ("--db", args.db), + ): + if value: + forwarded.extend((option, value)) + if args.limit: + forwarded.extend(("--limit", str(args.limit))) + if args.dry_run: + forwarded.append("--dry-run") + else: + # Invoking the historical write command is the owner's explicit local + # confirmation; forwarding --yes preserves its non-interactive contract. + forwarded.append("--yes") + if args.json: + forwarded.append("--json") + return int(importer_main(forwarded)) if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/scripts/smoke_entry_points.py b/scripts/smoke_entry_points.py index d20f1944..f0544d76 100644 --- a/scripts/smoke_entry_points.py +++ b/scripts/smoke_entry_points.py @@ -45,6 +45,7 @@ "engraphis-consolidate": "scripts.consolidate:main", "engraphis-graph": "scripts.graph_cli:main", "engraphis-graph-server": "scripts.graph_server:main", + "engraphis-import": "scripts.importer:main", "engraphis-init": "scripts.init:main", "engraphis-update": "scripts.update:main", } diff --git a/scripts/verify_distribution_contents.py b/scripts/verify_distribution_contents.py index 72a36aa0..ab95fa06 100644 --- a/scripts/verify_distribution_contents.py +++ b/scripts/verify_distribution_contents.py @@ -35,6 +35,8 @@ }) REQUIRED_SDIST = REQUIRED_COMMON | frozenset({ "BENCHMARKS.md", + "docs/DOCUMENT_IMPORT.md", + "docs/OBSIDIAN_IMPORT.md", "docker-compose.lan.yml", "deploy/force-graph-1.51.4.licenses.json", "deploy/force-graph-1.51.4.yarn.lock", diff --git a/scripts/watch_repo.py b/scripts/watch_repo.py index 9739ab37..45dbd3d0 100644 --- a/scripts/watch_repo.py +++ b/scripts/watch_repo.py @@ -35,6 +35,38 @@ _WATCHED_EXTENSIONS = frozenset(LANG_BY_EXT) +def _watched_files(root: Path): + """Yield watched files using the code indexer's directory policy.""" + from engraphis.backends.codegraph import ( + _DEFAULT_EXCLUDE_DIRS, + _ignored_by_rules, + _rel_posix, + load_ignore_patterns, + ) + + names, globs, unignore = load_ignore_patterns(str(root)) + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + rel_dir = os.path.relpath(dirpath, str(root)) + dirnames[:] = [ + name + for name in dirnames + if name not in _DEFAULT_EXCLUDE_DIRS + and not _ignored_by_rules( + _rel_posix(rel_dir, name), name, names, globs, unignore + ) + ] + for fname in filenames: + if os.path.splitext(fname)[1].lower() not in _WATCHED_EXTENSIONS: + continue + if _ignored_by_rules( + _rel_posix(rel_dir, fname), fname, names, globs, unignore + ): + continue + full = os.path.join(dirpath, fname) + if not os.path.islink(full): + yield full + + class _PollingWatcher: """Poll-based detector using content-backed file signatures. @@ -48,6 +80,7 @@ def __init__(self, root: Path, interval: float = 5.0) -> None: self.root = root self.interval = max(1.0, interval) self._signatures: dict[str, tuple[int, int, bytes]] = {} + self._pending_signatures: dict[str, tuple[int, int, bytes]] | None = None self._initial_scan_done = False # Paths whose last reindex failed and are absent from the current scan. # Without this set, a failed deletion reindex would never retry because @@ -68,36 +101,41 @@ def __init__(self, root: Path, interval: float = 5.0) -> None: self._exclude_dirs |= ignore_names self._exclude_dirs -= unignore + def _is_excluded(self, path: str) -> bool: + """Return whether *path* (or any parent) is pruned by the exclude policy. + + Matches the polling walk's pruning: any path component equal to a + ``_DEFAULT_EXCLUDE_DIRS`` entry or a name ruled out by + ``.engraphisignore`` is excluded, exactly like ``_watched_files``. + """ + try: + relative = os.path.relpath(path, str(self.root)) + except ValueError: + return True + if relative in ("", "."): + return False + return any(part in self._exclude_dirs for part in Path(relative).parts) + def _scan(self) -> dict[str, tuple[int, int, bytes]]: """Walk the tree and collect content-backed signatures.""" signatures: dict[str, tuple[int, int, bytes]] = {} - for dirpath, dirnames, filenames in os.walk(self.root): - # Prune excluded directories in-place so os.walk does not descend. - dirnames[:] = [ - d for d in dirnames if d not in self._exclude_dirs - and not d.startswith(".") - ] - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in _WATCHED_EXTENSIONS: - continue - full = os.path.join(dirpath, fname) - try: - digest = hashlib.blake2b(digest_size=16) - with open(full, "rb") as handle: - info = os.fstat(handle.fileno()) - while True: - chunk = handle.read(64 * 1024) - if not chunk: - break - digest.update(chunk) - signatures[full] = ( - int(getattr(info, "st_mtime_ns", info.st_mtime * 1_000_000_000)), - int(info.st_size), - digest.digest(), - ) - except OSError: - pass + for full in _watched_files(self.root): + try: + digest = hashlib.blake2b(digest_size=16) + with open(full, "rb") as handle: + info = os.fstat(handle.fileno()) + while True: + chunk = handle.read(64 * 1024) + if not chunk: + break + digest.update(chunk) + signatures[full] = ( + int(getattr(info, "st_mtime_ns", info.st_mtime * 1_000_000_000)), + int(info.st_size), + digest.digest(), + ) + except OSError: + pass return signatures def poll(self) -> list[str]: @@ -135,7 +173,9 @@ def poll(self) -> list[str]: # Clear entries that have reappeared (file recreated between polls). self._pending_deletions -= set(current.keys()) - self._signatures = current + # Keep the candidate separate until the caller confirms that incremental + # indexing succeeded. A transient read/parse failure must be retried. + self._pending_signatures = current return changed def untrack(self, paths: list[str], *, deletions: bool = False) -> None: @@ -152,23 +192,81 @@ def untrack(self, paths: list[str], *, deletions: bool = False) -> None: absent paths are silently ignored. """ for path in paths: - if self._signatures.pop(path, None) is not None: - continue + was_tracked = self._signatures.pop(path, None) is not None if deletions and path in self._last_deletions: self._pending_deletions.add(path) + continue + if was_tracked: + continue + + def acknowledge(self) -> None: + """Accept the most recent poll after its changes were indexed successfully.""" + if self._pending_signatures is not None: + self._signatures = self._pending_signatures + self._pending_signatures = None -def _try_watchdog_watcher(root: Path, callback, stop_event): - """Attempt watchdog-based watching. Returns True if started, False if unavailable.""" +def _try_watchdog_watcher(root: Path, callback, stop_event, startup_reconcile): + """Watch while queuing events that arrive during startup reconciliation.""" try: from watchdog.observers import Observer # type: ignore[import-not-found] from watchdog.events import FileSystemEventHandler # type: ignore[import-not-found] except ImportError: - return False + return None + + import threading + + startup_done = threading.Event() + pending_paths: list[str] = [] + retry_paths: list[str] = [] + pending_lock = threading.Lock() + retry_lock = threading.Lock() + callback_lock = threading.Lock() + + def queue_retry(paths): + with retry_lock: + for path in paths: + if path not in retry_paths: + retry_paths.append(path) + + def dispatch(paths): + unique = list(dict.fromkeys(paths)) + if unique: + with callback_lock: + if not callback(unique): + queue_retry(unique) + + def enqueue(paths): + with pending_lock: + if not startup_done.is_set(): + pending_paths.extend(paths) + return + dispatch(paths) _MAX_RETRIES = 3 + def _build_handler() -> "_Handler": + watcher = _PollingWatcher(root) + handler = _Handler() + handler._exclude_dirs = watcher._exclude_dirs + return handler + class _Handler(FileSystemEventHandler): + #: Directory/name pruning shared with the polling backend (defaults + + #: ``.engraphisignore``). Assigned by :func:`_build_handler`; left as an + #: empty set so a hand-constructed handler never blocks on it. + _exclude_dirs: set[str] = set() + + def _excluded(self, path: str) -> bool: + """Return whether *path* (or any parent) is pruned by the exclude policy.""" + try: + relative = os.path.relpath(path, str(root)) + except ValueError: + return True + if relative in ("", "."): + return False + return any(part in self._exclude_dirs for part in Path(relative).parts) + def _dispatch(self, paths: list[str]) -> None: for attempt in range(1, _MAX_RETRIES + 1): if callback(paths): @@ -183,11 +281,25 @@ def _dispatch(self, paths: list[str]) -> None: _MAX_RETRIES, paths, ) + def on_any_event(self, event): + # Belt-and-suspenders gate at the framework entry point: reject events + # whose src or dest path is under an excluded directory (defaults + + # .engraphisignore) before they reach the specific handlers, mirroring + # the polling backend's pruning. + src = getattr(event, "src_path", "") + if src and self._excluded(src): + return + dest = getattr(event, "dest_path", "") + if dest and self._excluded(dest): + return + super().on_any_event(event) + def on_modified(self, event): - if not event.is_directory: - ext = os.path.splitext(event.src_path)[1].lower() - if ext in _WATCHED_EXTENSIONS: - self._dispatch([event.src_path]) + if event.is_directory or self._excluded(event.src_path): + return + ext = os.path.splitext(event.src_path)[1].lower() + if ext in _WATCHED_EXTENSIONS: + enqueue([event.src_path]) def on_created(self, event): self.on_modified(event) @@ -201,22 +313,35 @@ def on_moved(self, event): paths = [ path for path in (event.src_path, event.dest_path) - if os.path.splitext(path)[1].lower() in _WATCHED_EXTENSIONS + if not self._excluded(path) + and os.path.splitext(path)[1].lower() in _WATCHED_EXTENSIONS ] if paths: - self._dispatch(paths) + enqueue(paths) observer = Observer() - observer.schedule(_Handler(), str(root), recursive=True) + observer.schedule(_build_handler(), str(root), recursive=True) observer.start() logger.info("watchdog observer started on %s", root) try: + if not startup_reconcile(): + return 1 + with pending_lock: + startup_done.set() + startup_paths = list(pending_paths) + pending_paths.clear() + dispatch(startup_paths) while not stop_event.is_set(): + with retry_lock: + retry = list(retry_paths) + retry_paths.clear() + if retry: + dispatch(retry) stop_event.wait(timeout=1.0) finally: observer.stop() observer.join() - return True + return 0 def _run(args, engine) -> int: @@ -272,12 +397,9 @@ def reindex(paths: list[str], *, fail_full: bool = False) -> bool: ) return True - # Reconcile persisted code state before establishing any in-process watcher - # baseline. This catches edits, renames, and deletions made while the watcher - # was stopped and works for both one-shot and continuous modes. - if not reindex([], fail_full=True): - return 1 if args.no_watch: + if not reindex([], fail_full=True): + return 1 print("Reindex complete.") return 0 @@ -291,12 +413,26 @@ def _shutdown(signum, frame): signal.signal(signal.SIGINT, _shutdown) signal.signal(signal.SIGTERM, _shutdown) - if _try_watchdog_watcher(root, reindex, stop_event): - return 0 + watchdog_status = _try_watchdog_watcher( + root, reindex, stop_event, + lambda: reindex([], fail_full=True), + ) + if watchdog_status is not None: + return watchdog_status logger.info("watchdog not available; using polling (interval=%.1fs)", args.interval) watcher = _PollingWatcher(root, interval=args.interval) + # Establish the polling baseline before the full startup reconciliation so + # edits made during the scan are replayed after it completes. watcher.poll() + if not reindex([], fail_full=True): + return 1 + changed_during_startup = watcher.poll() + if changed_during_startup: + if reindex(changed_during_startup): + watcher.acknowledge() + else: + watcher.acknowledge() print(f"Watching {root} (poll every {args.interval}s, Ctrl+C to stop)...") while not stop_event.is_set(): @@ -305,13 +441,17 @@ def _shutdown(signum, frame): break changed = watcher.poll() if changed: - if not reindex(changed): + if reindex(changed): + watcher.acknowledge() + else: logger.warning( "incremental reindex failed for %d file(s); " "will retry on next poll cycle", len(changed), ) watcher.untrack(changed, deletions=True) + else: + watcher.acknowledge() print("Stopped.") return 0 diff --git a/tests/e2e/ledger.spec.js b/tests/e2e/ledger.spec.js index a0fdf1bb..dde34dde 100644 --- a/tests/e2e/ledger.spec.js +++ b/tests/e2e/ledger.spec.js @@ -46,11 +46,13 @@ async function mockApi(page, options = {}) { requests.automationBootstraps = []; requests.syncRuns = []; requests.details = []; + requests.documentImports = []; const audit = options.audit || []; const receipts = options.receipts || []; const workspaceList = options.workspaces || [{ name: workspace, memories: memories.length }]; const licenseState = options.license || license(); let automationPolicy = options.automationPolicy || null; + let documentPolls = 0; const memoriesFor = requestUrl => { const selected = requestUrl.searchParams.get('workspace') || workspace; return (options.memoriesByWorkspace && options.memoriesByWorkspace[selected]) || memories; @@ -302,6 +304,8 @@ test('Ledger is live, safe, lazy, accessible, and responsive', async ({ page }) expect(response.headers()['content-security-policy']).not.toContain("'unsafe-inline'"); await expect(page.getByRole('heading', { name: `What changed in ${workspace}` })).toBeVisible(); + await expect(page.locator('#context-savings-summary')).toHaveClass(/savings-overview-section/); + expect(await page.locator('#context-savings-summary').evaluate(element => Boolean(element.closest('.view-column')))).toBe(true); await expect(page.locator('#context-savings-summary-body .savings-number')).toHaveText('2,048'); await expect(page.locator('#context-savings-summary-body .savings-unit')).toHaveText('tokens avoided'); await expect(page.locator('#context-savings-summary-body .savings-rate-value')).toHaveText('50.0%'); @@ -336,6 +340,7 @@ test('Ledger is live, safe, lazy, accessible, and responsive', async ({ page }) await page.setViewportSize({ width: 375, height: 812 }); await expect(page.getByRole('button', { name: 'Manage' })).toBeVisible(); + await expect(page.locator('#workspace-select')).toBeVisible(); await expect(page.locator('#sidebar-pro-cta')).toBeVisible(); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); @@ -526,6 +531,14 @@ test('memory listings open the editable Library detail from every dashboard view await mockApi(page); await page.goto('/'); + await page.getByRole('button', { name: 'Library memories and imports' }).click(); + const libraryOptions = page.locator('#library-list [role="option"]'); + await expect(libraryOptions.first()).toHaveAttribute('tabindex', '0'); + await expect(libraryOptions.nth(1)).toHaveAttribute('tabindex', '-1'); + await libraryOptions.first().press('ArrowDown'); + await expect(libraryOptions.nth(1)).toHaveAttribute('tabindex', '0'); + await page.getByRole('button', { name: 'Today changes and decisions' }).click(); + await page.locator('#proactive-list [data-memory-id="mem_database"]').click(); await expect(page.locator('#memory-detail h2')).toHaveText('Database choice'); await expect(page.locator('#memory-detail').getByRole('button', { name: 'Edit' })).toBeVisible(); @@ -1295,6 +1308,7 @@ test('Ledger applies the configured LLM extraction toggle', async ({ page }) => }); test('Ledger gives active Pro members direct Cloud access and saves hosted policy changes', async ({ page }) => { + const errors = browserErrors(page); const activePro = { ...license(), plan: 'pro', @@ -1324,6 +1338,7 @@ test('Ledger gives active Pro members direct Cloud access and saves hosted polic 'href', 'https://cloud.engraphis.test/account?utm_source=engraphis&utm_medium=product&utm_campaign=pro_conversion&utm_content=sidebar', ); + await expect(page.locator('#plan-badge')).toHaveCount(0); await page.getByRole('button', { name: 'Manage' }).click(); await page.getByRole('tab', { name: 'Settings' }).click(); @@ -1362,6 +1377,7 @@ test('Ledger gives active Pro members direct Cloud access and saves hosted polic infer: false, }); await expect(page.getByRole('spinbutton', { name: 'Run every (hours)' })).toHaveValue('12'); + expect(errors).toEqual([]); }); test('billing cadence selects the exact Pro and Team checkout target', async ({ page }) => { diff --git a/tests/test_chunking_extractor.py b/tests/test_chunking_extractor.py index d02ddbbb..4624c778 100644 --- a/tests/test_chunking_extractor.py +++ b/tests/test_chunking_extractor.py @@ -6,6 +6,8 @@ control-character defanging. """ import pytest +import sys +import types import engraphis.backends.extractor as extractor_module from engraphis.backends.extractor import ( @@ -58,6 +60,32 @@ def test_chunk_tokenizer_strict_mode_rejects_mutable_remote_revision_before_load ) +def test_chunk_tokenizer_local_selector_forces_local_files_only(monkeypatch): + calls = [] + + class FakeTokenizer: + def encode(self, text, add_special_tokens=False): + return list(text) + + class FakeAutoTokenizer: + @staticmethod + def from_pretrained(model, **kwargs): + calls.append((model, kwargs)) + return FakeTokenizer() + + monkeypatch.setitem( + sys.modules, "transformers", types.SimpleNamespace(AutoTokenizer=FakeAutoTokenizer), + ) + + counter, identity = _load_chunk_token_counter("local:C:/models/reader") + + assert calls == [("C:/models/reader", { + "trust_remote_code": False, "local_files_only": True, + })] + assert identity == "hf:C:/models/reader@unversioned" + assert counter("abc") == 3 + + def test_empty_or_whitespace_returns_nothing(): # engine.ingest treats [] as "extractor found nothing" and stores the raw text, # so an empty parse must not fabricate a chunk. diff --git a/tests/test_cli_entrypoints.py b/tests/test_cli_entrypoints.py index 4f284a61..eba708d1 100644 --- a/tests/test_cli_entrypoints.py +++ b/tests/test_cli_entrypoints.py @@ -4,6 +4,8 @@ import os import subprocess import sys +import threading +import time from pathlib import Path from types import SimpleNamespace @@ -457,8 +459,9 @@ def test_polling_watcher_untrack_causes_reappearance_on_next_poll(tmp_path): changed = watcher.poll() assert len(changed) == 1 - # Without untrack, next poll sees no change - assert watcher.poll() == [] + # Until the caller acknowledges a successful reindex, the change remains + # visible so a transient indexing failure can be retried. + assert watcher.poll() == [str(source)] # After untrack, the file reappears as changed watcher.untrack([str(source)]) @@ -489,7 +492,90 @@ def test_polling_watcher_retries_failed_deletions(tmp_path): source.write_text("x = 2\n", encoding="utf-8") assert watcher.poll() == [str(source)] + assert watcher.poll() == [str(source)] + watcher.acknowledge() + assert watcher.poll() == [] + + +def test_watchdog_retries_failed_reindex(monkeypatch, tmp_path): + source = tmp_path / "module.py" + source.write_text("value = 1\n", encoding="utf-8") + calls = [] + stop_event = threading.Event() + + class Event: + is_directory = False + src_path = str(source) + + class Observer: + def schedule(self, handler, _root, recursive): + assert recursive is True + self.handler = handler + + def start(self): + def emit(): + time.sleep(0.05) + self.handler.on_modified(Event()) + + self.thread = threading.Thread(target=emit) + self.thread.start() + + def stop(self): + stop_event.set() + + def join(self): + self.thread.join() + + monkeypatch.setitem(sys.modules, "watchdog", SimpleNamespace()) + monkeypatch.setitem(sys.modules, "watchdog.observers", SimpleNamespace(Observer=Observer)) + monkeypatch.setitem( + sys.modules, "watchdog.events", + SimpleNamespace(FileSystemEventHandler=object), + ) + + def reindex(paths): + calls.append(paths) + if len(calls) == 1: + return False + stop_event.set() + return True + + assert watch_repo._try_watchdog_watcher( + tmp_path, reindex, stop_event, lambda: True, + ) == 0 + assert calls == [[str(source)], [str(source)]] + + +def test_polling_watcher_retries_failed_changes_until_acknowledged(tmp_path): + source = tmp_path / "module.py" + source.write_text("value = 1\n", encoding="utf-8") + watcher = watch_repo._PollingWatcher(tmp_path) + + assert watcher.poll() == [] + source.write_text("value = 2\n", encoding="utf-8") + assert watcher.poll() == [str(source)] + assert watcher.poll() == [str(source)] + + watcher.acknowledge() + assert watcher.poll() == [] + + +def test_polling_watcher_prunes_codegraph_exclusions(tmp_path): + source = tmp_path / "src" / "keep.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + for directory in ("node_modules", ".venv", "target"): + ignored = tmp_path / directory + ignored.mkdir() + (ignored / "generated.py").write_text("value = 2\n", encoding="utf-8") + generated = tmp_path / "generated" + generated.mkdir() + (generated / "ignored.py").write_text("value = 3\n", encoding="utf-8") + (tmp_path / ".engraphisignore").write_text("generated\n", encoding="utf-8") + + watcher = watch_repo._PollingWatcher(tmp_path) assert watcher.poll() == [] + assert set(watcher._signatures) == {str(source)} def test_delete_namespace_is_atomic_and_records_one_batch_receipt( diff --git a/tests/test_config.py b/tests/test_config.py index 83cd0ed5..9ba795f3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -291,6 +291,47 @@ def test_explicit_owner_private_env_file_loads_without_overriding_process_env( assert overridden.stdout.strip() == "https://operator.example.test" +def test_trusted_env_parser_supports_documented_values_without_interpolation() -> None: + parsed = config._parse_trusted_env( + "# generated and operator-managed values\n" + "\n" + "ENGRAPHIS_DB_PATH=C:\\Users\\O'Brien\\Memory Vault\\engraphis.db\n" + "ENGRAPHIS_CSP=\"default-src 'self'; frame-ancestors 'none'\"\n" + "ENGRAPHIS_HSTS='max-age=31536000; includeSubDomains' # TLS only\n" + 'ENGRAPHIS_LLM_EXTRA_HEADERS={"X-Literal":"${HOME}","X-Title":"engraphis"}\n' + "ENGRAPHIS_DUPLICATE=first\n" + "export ENGRAPHIS_DUPLICATE=second\n" + ) + + assert parsed == { + "ENGRAPHIS_DB_PATH": r"C:\Users\O'Brien\Memory Vault\engraphis.db", + "ENGRAPHIS_CSP": "default-src 'self'; frame-ancestors 'none'", + "ENGRAPHIS_HSTS": "max-age=31536000; includeSubDomains", + "ENGRAPHIS_LLM_EXTRA_HEADERS": ( + '{"X-Literal":"${HOME}","X-Title":"engraphis"}' + ), + "ENGRAPHIS_DUPLICATE": "second", + } + + +@pytest.mark.parametrize( + "raw", + [ + "lowercase=value\n", + "ENGRAPHIS_BROKEN\n", + 'ENGRAPHIS_CSP="unterminated\n', + "ENGRAPHIS_CSP='unterminated\n", + 'ENGRAPHIS_CSP="valid" trailing\n', + "ENGRAPHIS_TOKEN=do-not-print\x00suffix\n", + ], +) +def test_trusted_env_parser_rejects_malformed_syntax_without_echoing_values(raw) -> None: + with pytest.raises(ValueError, match="trusted config contains invalid syntax") as caught: + config._parse_trusted_env(raw) + + assert "do-not-print" not in str(caught.value) + + def test_explicit_env_file_path_must_be_absolute(tmp_path) -> None: environment = dict(os.environ) environment["ENGRAPHIS_ENV_FILE"] = "relative.env" diff --git a/tests/test_core_store.py b/tests/test_core_store.py index 1f87ad08..c9aa2c56 100644 --- a/tests/test_core_store.py +++ b/tests/test_core_store.py @@ -33,6 +33,24 @@ def store(): s.close() +def test_append_event_does_not_commit_a_caller_owned_transaction(store): + workspace_id = store.get_or_create_workspace("events") + store.conn.execute( + "INSERT INTO audit(id, ts, actor, action, target, detail) VALUES (?,?,?,?,?,?)", + ("aud_pending", 1.0, "test", "pending", "target", "detail"), + ) + assert store.conn.transaction_owned_by_current_thread() + + event_id = store.append_event( + kind="test", content="event", workspace_id=workspace_id, + ) + + assert store.conn.transaction_owned_by_current_thread() + store.conn.rollback() + assert store.conn.execute("SELECT 1 FROM events WHERE id=?", (event_id,)).fetchone() is None + assert store.conn.execute("SELECT 1 FROM audit WHERE id='aud_pending'").fetchone() is None + + def test_schema_version(store): assert store.schema_version == SCHEMA_VERSION @@ -2898,4 +2916,4 @@ def test_add_memory_advances_hlc_only_for_real_local_descriptive_overwrite(store lattice_only = store.get_memory(memory_id) assert lattice_only is not None assert lattice_only.stability == idempotent.stability - assert lattice_only.modified_hlc == changed_clock \ No newline at end of file + assert lattice_only.modified_hlc == changed_clock diff --git a/tests/test_dashboard_v2.py b/tests/test_dashboard_v2.py index f3176a3c..73ae0c4f 100644 --- a/tests/test_dashboard_v2.py +++ b/tests/test_dashboard_v2.py @@ -54,7 +54,8 @@ def test_dashboard_serves_and_bootstraps_local_core(monkeypatch, tmp_path): page = client.get("/") assert page.status_code == 200 assert "Engraphis Ledger" in page.text - assert "Visible memories" in page.text + assert "Live memories" in page.text + assert "All versions, including history" in page.text assert "Live rows" not in page.text assert 'class="sidebar"' in page.text for area in ("Today", "Ask", "Library", "Graph & Relationships", "Provenance", "Manage"): @@ -82,11 +83,11 @@ def test_dashboard_serves_and_bootstraps_local_core(monkeypatch, tmp_path): assert "'/v2-assets/vendor/d3.min.js?v=20260727-final'" in ledger_js.text assert "'/v2-assets/vendor/force-graph.min.js?v=20260727-final'" in ledger_js.text assert "'/v2-assets/engraphis-graph.js?v=20260809-pin-only-physics'" in ledger_js.text - assert "/v2-assets/ledger.css?v=20260809-pin-only-physics" in page.text - assert "/v2-assets/ledger.js?v=20260809-pin-only-physics" in page.text + assert "/v2-assets/ledger.css?v=20260728-connected-memories" in page.text + assert "/v2-assets/ledger.js?v=20260728-connected-memories" in page.text classic_js = client.get("/classic-assets/dashboard.js") assert classic_js.status_code == 200 - assert "/static/vendor/force-graph.min.js?v=20260809-csp" in classic_js.text + assert "/static/vendor/force-graph.min.js" in classic_js.text assert "/v2-assets/engraphis-graph.js?v=20260809-pin-only-physics" in classic_js.text assert "graphLimit=GRAPH_FULL?20000:320" in classic_js.text assert "graphScope=GRAPH_FULL?'&full=true':(showUnlinked?'':'&connected_only=true')" in classic_js.text @@ -104,20 +105,506 @@ def test_dashboard_serves_and_bootstraps_local_core(monkeypatch, tmp_path): assert filtered.status_code == 200 assert filtered.json()["period"] == {"from_ts": 0, "to_ts": 9_999_999_999} assert "Estimated context saved" in page.text - ledger_css = client.get("/v2-assets/ledger.css") - assert ledger_css.status_code == 200 - assert ".savings-hero" in ledger_css.text - assert ".savings-progress" in ledger_css.text - assert "function savingsMetric(estimate)" in ledger_js.text - assert "tokens avoided" in ledger_js.text + assert 'class="content-section savings-overview-section" id="context-savings-summary"' in page.text + + +def test_dashboard_memory_reads_use_the_active_store_for_memory_databases(monkeypatch): + monkeypatch.setattr(settings, "db_path", ":memory:") + monkeypatch.setattr(settings, "embed_model", "") + monkeypatch.setattr(settings, "embed_dim", 384) + monkeypatch.setattr(settings, "allowed_workspaces", []) + monkeypatch.setattr(settings, "api_token", "") + from engraphis.dashboard_app import create_app + + with TestClient(create_app(), client=("127.0.0.1", 50000)) as client: + service = client.app.state.service + workspace_id = service.store.get_or_create_workspace("memory-db") + service.engine.remember( + "The dashboard must show this in-memory record.", + workspace_id=workspace_id, + scope=Scope.WORKSPACE, + title="Visible in memory", + ) + + listed = client.get("/api/memories", params={"workspace": "memory-db"}) + assert listed.status_code == 200 + assert listed.json()["count"] == 1 + assert listed.json()["memories"][0]["title"] == "Visible in memory" + + fallback = v2_api._keyword_search("memory-db", "dashboard in-memory") + assert len(fallback) == 1 + + +def test_dashboard_exposes_accessible_document_import_preview_and_job_contract(monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as client: + page = client.get("/") + assert page.status_code == 200 + for fragment in ( + 'id="obsidian-import-button"', + 'id="obsidian-import-dialog"', + 'aria-labelledby="obsidian-import-title"', + 'id="obsidian-import-files"', + 'id="obsidian-import-folder"', + 'webkitdirectory', + 'id="obsidian-source-mode"', + 'id="obsidian-workspace"', + 'id="obsidian-repo"', + 'id="obsidian-session"', + 'id="obsidian-scope"', + 'id="obsidian-memory-type"', + 'id="obsidian-conflict"', + 'id="obsidian-confirmed"', + 'id="obsidian-report-filter"', + 'id="obsidian-cancel"', + ): + assert fragment in page.text + # Folder selection must reveal every local file. In document mode the + # browser uploads only supported document bytes; Obsidian mode retains a + # content-free attachment manifest for link/report accuracy. + assert 'accept=".md"' not in page.text + script = client.get("/v2-assets/ledger.js").text + for endpoint in ( + '/workspaces/import-documents/sources?', + '/workspaces/import-documents/formats', + '/workspaces/import-documents/preview', + '/workspaces/import-documents/run', + '/workspaces/import-documents/jobs/', + ): + assert endpoint in script + assert "attachment_manifest" in script + assert "webkitRelativePath" in script + assert "documentExtensions" in script + assert "loadDocumentFormats" in script + assert "applySelectedDocumentSource" in script + assert "prefillNewSourceLabelFromFolder" in script + assert "requireNewSourceLabel" in script + assert "Enter a Source label before creating a new source." in script + assert "sourceMode === 'obsidian'" in script + assert "Confirm the selected scope before importing." in script + assert "review_token" in script + assert "reviewGeneration" in script + assert "invalidateDocumentImportPreview" in script + assert "selection.reviewToken = '';" in script + assert "if (obsidianImport.running) return;" in script + assert "pollObsidianImport(jobId, workspace)" in script + assert "dataset.workspace" in script + for material_control in ( + "obsidian-import-files", "obsidian-import-folder", "obsidian-workspace", + "obsidian-repo", "obsidian-session", "obsidian-scope", + "obsidian-memory-type", "obsidian-vault-label", "obsidian-conflict", + ): + assert material_control in script + + +def test_document_dashboard_endpoints_use_generic_service_and_reject_unknown_binary_uploads( + monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as client: + monkeypatch.setattr(settings, "api_token", "dashboard-owner-token") + session = client.post("/api/auth/session", json={"token": "dashboard-owner-token"}) + headers = { + "X-Engraphis-Browser-Session": "1", + "X-Engraphis-Review-CSRF": session.json()["review_csrf_token"], + } + seen = {} + + # The dashboard's generic bearer/API authority remains insufficient for + # document bytes, format disclosure, source metadata, or job control. + denied = client.get( + "/api/workspaces/import-documents/formats", + headers={"Authorization": "Bearer dashboard-owner-token"}, + ) + # The client still holds its session cookie, but the bearer cannot supply + # the required browser-session marker or the per-session CSRF nonce. + assert denied.status_code == 403 + + def preview_document_upload(**kwargs): + seen["preview"] = kwargs + return {"counts": {"documents": 1}, "files": [{"path": "notes/readme.txt", "status": "import"}]} + + def import_document_upload(**kwargs): + seen["run"] = kwargs + return {"job_id": "job_document", "state": "queued"} + + def preview_obsidian_upload(**kwargs): + seen["obsidian_preview"] = kwargs + return {"counts": {"markdown": 1}, "files": []} + + def import_obsidian_upload(**kwargs): + seen["obsidian_run"] = kwargs + return {"job_id": "job_obsidian_via_wizard", "state": "queued"} + + monkeypatch.setattr(client.app.state.service, "list_source_vaults", lambda workspace: [{"id": "vlt_1", "label": workspace}], raising=False) + monkeypatch.setattr(client.app.state.service, "preview_document_upload", preview_document_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "import_document_upload", import_document_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "preview_obsidian_upload", preview_obsidian_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "import_obsidian_upload", import_obsidian_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "get_document_import_job", lambda job_id, workspace: {"id": job_id, "workspace": workspace, "state": "completed"}, raising=False) + monkeypatch.setattr(client.app.state.service, "cancel_document_import_job", lambda job_id, workspace: {"id": job_id, "workspace": workspace, "cancel_requested": True}, raising=False) + payload = { + "workspace": "demo", "scope": "workspace", "memory_type": "semantic", + "source_id": "vlt_1", "source_label": "Notes", "on_conflict": "error", + "source_mode": "documents", "confirmed": "true", "attachment_manifest": "[]", + } + upload = [("files", ("notes/readme.txt", b"hello", "text/plain"))] + missing_label = {**payload, "source_id": "", "source_label": " "} + preview_missing_label = client.post( + "/api/workspaces/import-documents/preview", data=missing_label, + files=upload, headers=headers, + ) + assert preview_missing_label.status_code == 400 + assert preview_missing_label.json()["detail"]["error"] == "source label is required for a new source" + run_missing_label = client.post( + "/api/workspaces/import-documents/run", data=missing_label, + files=upload, headers=headers, + ) + assert run_missing_label.status_code == 400 + sources = client.get("/api/workspaces/import-documents/sources?workspace=demo", headers=headers) + assert sources.json()["sources"] == [{"id": "vlt_1", "label": "demo"}] + formats = client.get("/api/workspaces/import-documents/formats", headers=headers) + assert formats.status_code == 200 + assert ".png" in formats.json()["extensions"] + preview = client.post("/api/workspaces/import-documents/preview", data=payload, files=upload, headers=headers) + assert preview.status_code == 200 + review_token = preview.json()["review_token"] + assert preview.json()["review_expires_in"] == 300 + assert seen["preview"]["source_id"] == "vlt_1" + assert seen["preview"]["files"] == [("notes/readme.txt", b"hello")] + saved_source_without_label = client.post( + "/api/workspaces/import-documents/preview", + data={**payload, "source_label": ""}, files=upload, headers=headers, + ) + assert saved_source_without_label.status_code == 200 + image = client.post( + "/api/workspaces/import-documents/preview", data=payload, + files=[("files", ("notes/pic.png", b"png", "image/png"))], headers=headers, + ) + assert image.status_code == 200 + binary = client.post( + "/api/workspaces/import-documents/preview", data=payload, + files=[("files", ("notes/archive.bin", b"binary", "application/octet-stream"))], + headers=headers, + ) + assert binary.status_code == 400 + assert binary.json()["detail"]["error"] == "unsupported document format" + attachments = client.post( + "/api/workspaces/import-documents/preview", + data={**payload, "attachment_manifest": '[{"path":"notes/pic.png","size":3}]'}, + files=upload, headers=headers, + ) + assert attachments.status_code == 400 + assert client.post( + "/api/workspaces/import-documents/run", + data={**payload, "confirmed": "false", "review_token": review_token}, + files=upload, headers=headers, + ).status_code == 403 + run_payload = {**payload, "review_token": review_token} + run = client.post( + "/api/workspaces/import-documents/run", data=run_payload, + files=upload, headers=headers, + ) + assert run.status_code == 200 + assert seen["run"]["confirmed"] is True + replay = client.post( + "/api/workspaces/import-documents/run", data=run_payload, + files=upload, headers=headers, + ) + assert replay.status_code == 403 + assert replay.json()["detail"]["error"] == "a fresh matching import preview is required" + assert client.get("/api/workspaces/import-documents/jobs/job_document?workspace=demo", headers=headers).status_code == 200 + assert client.post( + "/api/workspaces/import-documents/jobs/job_document/cancel", + data={"workspace": "demo"}, headers=headers, + ).json()["cancel_requested"] is True + + # The source-neutral wizard keeps an explicit Obsidian mode. It must + # call the compatibility adapter so an ``obsidian`` vlt_ identity and + # its rich Markdown lineage remain resumable instead of being treated + # as a generic ``documents`` source. + obsidian_payload = { + **payload, "source_id": "vlt_obsidian", "source_label": "Vault", + "source_mode": "obsidian", + } + markdown = [("files", ("notes/readme.md", b"# Note\n", "text/markdown"))] + obsidian_preview = client.post( + "/api/workspaces/import-documents/preview", data=obsidian_payload, + files=markdown, headers=headers, + ) + assert obsidian_preview.status_code == 200 + assert seen["obsidian_preview"]["vault_id"] == "vlt_obsidian" + obsidian_run = client.post( + "/api/workspaces/import-documents/run", + data={**obsidian_payload, "review_token": obsidian_preview.json()["review_token"]}, + files=markdown, headers=headers, + ) + assert obsidian_run.status_code == 200 + assert seen["obsidian_run"]["vault_id"] == "vlt_obsidian" + + +def test_document_import_review_binds_owner_target_policy_manifest_and_exact_bytes( + monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as client: + monkeypatch.setattr(settings, "api_token", "dashboard-owner-token") + session = client.post("/api/auth/session", json={"token": "dashboard-owner-token"}) + headers = { + "X-Engraphis-Browser-Session": "1", + "X-Engraphis-Review-CSRF": session.json()["review_csrf_token"], + } + imported = [] + + monkeypatch.setattr( + client.app.state.service, + "preview_document_upload", + lambda **kwargs: {"counts": {"documents": len(kwargs["files"])}}, + raising=False, + ) + monkeypatch.setattr( + client.app.state.service, + "preview_obsidian_upload", + lambda **kwargs: {"counts": {"markdown": len(kwargs["files"])}}, + raising=False, + ) + + def import_document_upload(**kwargs): + imported.append(kwargs) + return {"job_id": "job_bound", "state": "queued"} + + monkeypatch.setattr( + client.app.state.service, + "import_document_upload", + import_document_upload, + raising=False, + ) + base = { + "workspace": "demo", "repo": "repo-a", "session_id": "session-a", + "scope": "workspace", "memory_type": "semantic", + "source_id": "vlt_1", "source_label": "Notes", + "on_conflict": "error", "source_mode": "documents", + "confirmed": "true", "attachment_manifest": "[]", + } + upload = [("files", ("notes/Welcome.md", b"# Welcome", "text/markdown"))] + + def preview_token(data=None, files=None): + response = client.post( + "/api/workspaces/import-documents/preview", + data=data or base, files=files or upload, headers=headers, + ) + assert response.status_code == 200 + return response.json()["review_token"] + + token = preview_token() + missing = client.post( + "/api/workspaces/import-documents/run", data=base, + files=upload, headers=headers, + ) + assert missing.status_code == 403 + assert missing.json()["detail"]["error"] == ( + "a fresh matching import preview is required" + ) + + mutations = { + "workspace": "beta", + "repo": "repo-b", + "session_id": "session-b", + "scope": "repo", + "memory_type": "episodic", + "source_id": "vlt_2", + "source_label": "Other notes", + "on_conflict": "update", + "source_mode": "obsidian", + } + for field, changed_value in mutations.items(): + token = preview_token() + changed = {**base, field: changed_value, "review_token": token} + rejected = client.post( + "/api/workspaces/import-documents/run", data=changed, + files=upload, headers=headers, + ) + assert rejected.status_code == 403, field + assert rejected.json()["detail"]["error"] == ( + "a fresh matching import preview is required" + ) + + for changed_upload in ( + [("files", ("notes/Renamed.md", b"# Welcome", "text/markdown"))], + [("files", ("notes/Welcome.md", b"# Changed", "text/markdown"))], + ): + token = preview_token() + rejected = client.post( + "/api/workspaces/import-documents/run", + data={**base, "review_token": token}, + files=changed_upload, headers=headers, + ) + assert rejected.status_code == 403 + + obsidian = { + **base, + "source_mode": "obsidian", + "attachment_manifest": '[{"path":"assets/pic.png","size":12}]', + } + token = preview_token(data=obsidian) + changed_manifest = { + **obsidian, + "attachment_manifest": '[{"path":"assets/pic.png","size":13}]', + "review_token": token, + } + assert client.post( + "/api/workspaces/import-documents/run", data=changed_manifest, + files=upload, headers=headers, + ).status_code == 403 + + token = preview_token() + with client.app.state.document_import_review_lock: + client.app.state.document_import_reviews[token]["expires_at"] = 0 + expired = client.post( + "/api/workspaces/import-documents/run", + data={**base, "review_token": token}, files=upload, headers=headers, + ) + assert expired.status_code == 403 + + token = preview_token() + replacement_session = client.post( + "/api/auth/session", json={"token": "dashboard-owner-token"}, + ) + replacement_headers = { + "X-Engraphis-Browser-Session": "1", + "X-Engraphis-Review-CSRF": replacement_session.json()["review_csrf_token"], + } + rebound = client.post( + "/api/workspaces/import-documents/run", + data={**base, "review_token": token}, files=upload, + headers=replacement_headers, + ) + assert rebound.status_code == 403 + assert imported == [] + + +def test_obsidian_dashboard_endpoints_require_browser_owner_csrf_and_preserve_relative_paths( + monkeypatch, tmp_path): + with _client(monkeypatch, tmp_path) as client: + monkeypatch.setattr(settings, "api_token", "dashboard-owner-token") + payload = { + "workspace": "demo", "scope": "workspace", "memory_type": "semantic", + "vault_id": "", "vault_label": "Notes", "on_conflict": "error", + "confirmed": "true", "attachment_manifest": '[{"path":"assets/pic.png","size":12}]', + } + upload = [("files", ("notes/Welcome.md", b"# Welcome", "text/markdown"))] + # The generic bearer passes the outer API gate, but vault imports themselves are + # intentionally owner-browser-only. + denied = client.post( + "/api/workspaces/import-obsidian/preview", data=payload, files=upload, + headers={"Authorization": "Bearer dashboard-owner-token"}, + ) + assert denied.status_code == 401 + + session = client.post("/api/auth/session", json={"token": "dashboard-owner-token"}) + assert session.status_code == 200 + headers = { + "X-Engraphis-Browser-Session": "1", + "X-Engraphis-Review-CSRF": session.json()["review_csrf_token"], + } + seen = {} + + def preview_obsidian_upload(**kwargs): + seen["preview"] = kwargs + return {"counts": {"markdown_files": 1}, "files": [{"path": "notes/Welcome.md", "status": "import"}]} + + def import_obsidian_upload(**kwargs): + seen["run"] = kwargs + return {"job_id": "job_obsidian", "state": "queued"} + + monkeypatch.setattr(client.app.state.service, "list_obsidian_vaults", lambda workspace: [{"id": "v_1", "label": workspace}], raising=False) + monkeypatch.setattr(client.app.state.service, "preview_obsidian_upload", preview_obsidian_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "import_obsidian_upload", import_obsidian_upload, raising=False) + monkeypatch.setattr(client.app.state.service, "get_obsidian_import_job", lambda job_id, workspace: {"id": job_id, "workspace": workspace, "state": "completed"}, raising=False) + monkeypatch.setattr(client.app.state.service, "cancel_obsidian_import_job", lambda job_id, workspace: {"id": job_id, "workspace": workspace, "cancel_requested": True}, raising=False) + + missing_label = client.post( + "/api/workspaces/import-obsidian/preview", + data={**payload, "vault_label": "\t"}, files=upload, headers=headers, + ) + assert missing_label.status_code == 400 + assert missing_label.json()["detail"]["error"] == "source label is required for a new source" + + vaults = client.get("/api/workspaces/import-obsidian/vaults?workspace=demo", headers=headers) + assert vaults.status_code == 200 + assert vaults.json()["vaults"] == [{"id": "v_1", "label": "demo"}] + preview = client.post("/api/workspaces/import-obsidian/preview", data=payload, files=upload, headers=headers) + assert preview.status_code == 200 + review_token = preview.json()["review_token"] + assert seen["preview"]["files"] == [("notes/Welcome.md", b"# Welcome")] + assert seen["preview"]["attachment_manifest"] == [{"path": "assets/pic.png", "size": 12}] + + duplicate = client.post( + "/api/workspaces/import-obsidian/preview", data=payload, + files=[ + ("files", ("notes/Dupe.md", b"one", "text/markdown")), + ("files", ("NOTES/DUPE.md", b"two", "text/markdown")), + ], headers=headers, + ) + assert duplicate.status_code == 400 + assert duplicate.json()["detail"]["error"] == "duplicate upload path" + for unsafe in ("C:/vault/Note.md", "//server/share/Note.md"): + rejected = client.post( + "/api/workspaces/import-obsidian/preview", data=payload, + files=[("files", (unsafe, b"unsafe", "text/markdown"))], + headers=headers, + ) + assert rejected.status_code == 400 + overlap_payload = { + **payload, + "attachment_manifest": '[{"path":"NOTES/welcome.MD","size":12}]', + } + overlap = client.post( + "/api/workspaces/import-obsidian/preview", data=overlap_payload, + files=upload, headers=headers, + ) + assert overlap.status_code == 400 + assert overlap.json()["detail"]["error"] == "upload and attachment paths overlap" + + markdown_alias = client.post( + "/api/workspaces/import-obsidian/preview", data=payload, + files=[("files", ("notes/Legacy.markdown", b"legacy", "text/markdown"))], + headers=headers, + ) + assert markdown_alias.status_code == 400 + assert markdown_alias.json()["detail"]["error"] == ( + "Obsidian mode accepts Markdown note bytes only" + ) + + payload["confirmed"] = "false" + assert client.post( + "/api/workspaces/import-obsidian/run", + data={**payload, "review_token": review_token}, files=upload, headers=headers, + ).status_code == 403 + payload["confirmed"] = "true" + run_payload = {**payload, "review_token": review_token} + run = client.post( + "/api/workspaces/import-obsidian/run", data=run_payload, + files=upload, headers=headers, + ) + assert run.status_code == 200 + assert seen["run"]["confirmed"] is True + assert client.post( + "/api/workspaces/import-obsidian/run", data=run_payload, + files=upload, headers=headers, + ).status_code == 403 + status = client.get("/api/workspaces/import-obsidian/jobs/job_obsidian?workspace=demo", headers=headers) + assert status.status_code == 200 + assert status.json()["state"] == "completed" + cancelled = client.post( + "/api/workspaces/import-obsidian/jobs/job_obsidian/cancel", + data={"workspace": "demo"}, headers=headers, + ) + assert cancelled.status_code == 200 + assert cancelled.json()["cancel_requested"] is True def test_dashboard_assets_revalidate_instead_of_pinning_old_visuals(monkeypatch, tmp_path): with _client(monkeypatch, tmp_path) as client: for path in ( "/v2-assets/engraphis-graph.js?v=20260809-pin-only-physics", - "/v2-assets/ledger.js?v=20260809-pin-only-physics", - "/v2-assets/ledger.css?v=20260809-pin-only-physics", + "/v2-assets/ledger.js?v=20260728-connected-memories", + "/v2-assets/ledger.css?v=20260728-connected-memories", "/classic-assets/dashboard.js?v=20260809-pin-only-physics", ): response = client.get(path) @@ -132,16 +619,6 @@ def test_classic_dashboard_script_mirrors_the_static_compatibility_asset(): ).read_bytes() -def test_memory_health_dashboard_uses_the_v2_analytics_contract(): - root = Path(__file__).parents[1] / "engraphis" - for relative in ("classic_assets/dashboard.js", "static/dashboard.js"): - source = (root / relative).read_text(encoding="utf-8") - assert "/analytics/health?workspace=" in source - assert "decay_distribution" in source - assert "conflict_frequency" in source - assert "/memory/health/overview" not in source - - def test_dashboard_and_mcp_recall_share_the_v2_service(monkeypatch, tmp_path): pytest.importorskip("mcp", reason="MCP extra not installed") import json diff --git a/tests/test_document_import_cli.py b/tests/test_document_import_cli.py new file mode 100644 index 00000000..6d3589be --- /dev/null +++ b/tests/test_document_import_cli.py @@ -0,0 +1,528 @@ +"""Universal local-document CLI and service contracts.""" +from __future__ import annotations + +import json +from pathlib import Path +import sqlite3 +from types import SimpleNamespace + +import pytest + +from engraphis.service import MemoryService, ValidationError +from engraphis.core.interfaces import Scope +from engraphis.document_import import scan_document_upload +from scripts import importer + + +def _scan(count: int = 2): + return SimpleNamespace( + source_id="a" * 64, + documents=[object() for _ in range(count)], + rejected=[], + skipped=[], + ) + + +def test_documents_dry_run_dispatches_generic_planner_with_zero_writes( + monkeypatch, tmp_path, capsys, +): + root = tmp_path / "Unsorted" + root.mkdir() + database = tmp_path / "missing.db" + scan = _scan() + captured = {} + + monkeypatch.setattr(importer, "scan_document_tree", lambda path, **kwargs: scan) + + def preview(args, selected_scan, workspace): + captured.update({"args": args, "scan": selected_scan, "workspace": workspace}) + return { + "state": "preview", + "adapter": "documents", + "counts": {"documents": 2}, + "summary": {"formats": {"markdown": 1, "pdf": 1}}, + "files": [], + } + + monkeypatch.setattr(importer, "_preview_documents", preview) + result = importer.main([ + "documents", str(root), "--db", str(database), "--workspace", "acme", + "--dry-run", "--json", + ]) + + assert result == 0 + assert captured == {"args": captured["args"], "scan": scan, "workspace": "acme"} + payload = json.loads(capsys.readouterr().out) + assert payload["adapter"] == "documents" + assert payload["summary"]["formats"] == {"markdown": 1, "pdf": 1} + assert not database.exists() + + +def test_documents_real_dry_run_creates_no_database_or_sidecars(tmp_path, capsys): + root = tmp_path / "Unsorted" + root.mkdir() + (root / "brief.txt").write_text("Local-only brief.", encoding="utf-8") + database = tmp_path / "absent.db" + + result = importer.main([ + "documents", str(root), "--db", str(database), "--workspace", "acme", + "--dry-run", "--json", + ]) + + assert result == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["state"] == "preview" + assert payload["adapter"] == "documents" + assert payload["counts"]["documents"] == 1 + assert not database.exists() + assert not Path(str(database) + "-wal").exists() + assert not Path(str(database) + "-shm").exists() + + +def test_documents_dry_run_new_workspace_cannot_reuse_foreign_source( + tmp_path, capsys, +): + root = tmp_path / "Documents" + root.mkdir() + (root / "brief.txt").write_text("Approved local brief.", encoding="utf-8") + database = tmp_path / "memory.db" + service = MemoryService.create( + str(database), embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + try: + imported = service.import_document_tree( + str(root), workspace="alpha", source_label=root.name, + confirmed=True, + ) + foreign_source_id = imported["source_id"] + finally: + service.close() + + result = importer.main([ + "documents", str(root), "--db", str(database), "--workspace", "beta", + "--dry-run", "--json", + ]) + + assert result == 0 + preview = json.loads(capsys.readouterr().out) + assert preview["source_id"] is None + assert preview["vault_id"] is None + assert preview["counts"]["imported"] == 1 + assert preview["counts"].get("skipped", 0) == 0 + assert foreign_source_id not in str(preview) + with sqlite3.connect(database) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM workspaces WHERE name='beta'" + ).fetchone()[0] == 0 + + +def test_documents_actual_run_uses_generic_service_and_source_names( + monkeypatch, tmp_path, capsys, +): + root = tmp_path / "Loose files" + root.mkdir() + scan = _scan(count=1) + calls = {} + + class Service: + def import_document_tree(self, path, **kwargs): + calls.update({"path": path, **kwargs}) + kwargs["progress"]({ + "status": "imported", "relative_path": "notes/brief.docx", + }) + return { + "state": "completed", "adapter": "documents", + "counts": {"documents": 1, "imported": 1}, + "summary": {"formats": {"docx": 1}}, "files": [], + } + + def close(self): + calls["closed"] = True + + monkeypatch.setattr(importer, "scan_document_tree", lambda path, **kwargs: scan) + monkeypatch.setattr(importer, "_preview_documents", lambda *args: { + "state": "preview", "counts": {"documents": 1}, + "summary": {"formats": {"docx": 1}}, "files": [], + }) + monkeypatch.setattr(importer, "_local_service", lambda path: Service()) + + result = importer.main([ + "documents", str(root), "--workspace", "acme", "--repo", "knowledge", + "--source-id", "vlt_registered", "--source-label", "Company archive", + "--yes", "--json", + ]) + + assert result == 0 + assert calls["path"] == str(root) + assert calls["workspace"] == "acme" + assert calls["repo"] == "knowledge" + assert calls["source_id"] == "vlt_registered" + assert calls["source_label"] == "Company archive" + assert calls["confirmed"] is True + assert calls["closed"] is True + assert json.loads(capsys.readouterr().out)["report"]["adapter"] == "documents" + + +def test_documents_noninteractive_write_requires_confirmation(monkeypatch, tmp_path, capsys): + root = tmp_path / "docs" + root.mkdir() + monkeypatch.setattr(importer, "scan_document_tree", lambda path, **kwargs: _scan()) + monkeypatch.setattr(importer, "_preview_documents", lambda *args: { + "counts": {"documents": 2}, "summary": {}, "files": [], + }) + monkeypatch.setattr(importer.sys.stdin, "isatty", lambda: False) + + assert importer.main(["documents", str(root), "--workspace", "acme"]) == 2 + assert "require --yes" in capsys.readouterr().err + + +def test_generic_upload_validation_accepts_mixed_files_and_rejects_traversal(tmp_path, monkeypatch): + service = MemoryService.create( + str(tmp_path / "memory.db"), embed_dim=32, extractor="none", + graph_extractor="none", retention_supervisor="none", + ) + try: + monkeypatch.setattr("engraphis.service.MAX_IMPORT_RESOURCE_BYTES", 10) + monkeypatch.setattr("engraphis.service.MAX_IMPORT_TOTAL_BYTES", 15) + uploads, attachments = service._document_upload_inputs( + [("notes/readme.md", b"# Hello"), ("reports/q1.pdf", b"%PDF")], + [{"path": "images/chart.png", "size": 10}], + ) + assert [path for path, _raw in uploads] == [ + "notes/readme.md", "reports/q1.pdf", + ] + assert attachments == [{"path": "images/chart.png", "size": 10}] + with pytest.raises(ValidationError, match="invalid path"): + service._document_upload_inputs([("../secret.txt", b"x")], []) + with pytest.raises(ValidationError, match="duplicate path"): + service._document_upload_inputs( + [("Notes/Readme.md", b"a"), ("notes/readme.md", b"b")], [], + ) + with pytest.raises(ValidationError, match="paths overlap"): + service._document_upload_inputs( + [("notes/readme.md", b"a")], + [{"path": "notes/readme.md", "size": 1}], + ) + with pytest.raises(ValidationError, match="invalid size"): + service._document_upload_inputs( + [("notes/readme.md", b"a")], + [{"path": "assets/big.bin", "size": 11}], + ) + with pytest.raises(ValidationError, match="too large"): + service._document_upload_inputs( + [("notes/big.md", b"x" * 11)], [], + ) + with pytest.raises(ValidationError, match="total size"): + service._document_upload_inputs( + [("notes/one.md", b"x" * 8), ("notes/two.md", b"y" * 8)], [], + ) + finally: + service.close() + + +def test_document_browser_label_identity_is_nfc() -> None: + files = [("note.txt", b"local note")] + assert scan_document_upload(files, source_label="Caf\u00e9").source_id == ( + scan_document_upload(files, source_label="Cafe\u0301").source_id + ) == scan_document_upload(files, source_label="CAF\u00c9").source_id + + +def test_import_help_presents_documents_as_primary_and_obsidian_as_compatibility(): + help_text = " ".join(importer._parser().format_help().split()) + assert "documents" in help_text + assert "mixed local folder" in help_text + assert "obsidian" in help_text + assert "compatibility command" in help_text + + +def test_session_id_defaults_to_session_scope_and_repo_must_match_help(): + args = importer._parser().parse_args([ + "documents", "C:/files", "--session", "ses_example", + ]) + assert importer._scope(args) == Scope.SESSION + help_text = importer._parser()._subparsers._group_actions[0].choices[ + "documents" + ].format_help() + assert "must match --session" in " ".join(help_text.split()) + + +def test_manifest_matching_infers_repo_from_session_when_repo_is_omitted(): + snapshot = { + "vaults": [{ + "id": "vlt_existing", + "kind": "documents", + "root_digest": "d" * 64, + "workspace_name": "acme", + "repo_name": "product", + "session_id": "ses_product", + "workspace_id": "ws_acme", + "repo_id": "repo_product", + }], + } + + effective_repo = importer._effective_manifest_repo( + snapshot, repo=None, session_id="ses_product", + ) + selected, workspace_id, repo_id = importer._snapshot_target( + snapshot, root_digest="d" * 64, workspace="acme", + repo=effective_repo, session_id="ses_product", vault_id=None, + source_kind="documents", + ) + + assert effective_repo == "product" + assert selected["id"] == "vlt_existing" + assert (workspace_id, repo_id) == ("ws_acme", "repo_product") + + +def test_first_import_infers_repo_from_session_lineage_without_a_vault(): + snapshot = { + "vaults": [], + "sessions": [{ + "id": "ses_product", + "workspace_name": "acme", + "repo_name": "product", + }], + } + + assert importer._effective_manifest_repo( + snapshot, repo=None, session_id="ses_product", + ) == "product" + + +def test_manifest_matching_normalizes_repo_scope_session_target(): + snapshot = { + "vaults": [ + { + "id": "vlt_session", + "kind": "documents", + "root_digest": "s" * 64, + "workspace_name": "acme", + "repo_name": "product", + "session_id": "ses_product", + }, + { + "id": "vlt_repo", + "kind": "documents", + "root_digest": "d" * 64, + "workspace_name": "acme", + "repo_name": "product", + "session_id": None, + "workspace_id": "ws_acme", + "repo_id": "repo_product", + }, + ], + } + + effective_repo, effective_session = importer._effective_manifest_target( + snapshot, repo=None, session_id="ses_product", scope=Scope.REPO, + ) + selected, workspace_id, repo_id = importer._snapshot_target( + snapshot, root_digest="d" * 64, workspace="acme", + repo=effective_repo, session_id=effective_session, vault_id=None, + source_kind="documents", + ) + + assert (effective_repo, effective_session) == ("product", None) + assert selected["id"] == "vlt_repo" + assert (workspace_id, repo_id) == ("ws_acme", "repo_product") + + +@pytest.mark.parametrize( + "extra", [["--repo", "api"], ["--session", "ses_example"]], +) +def test_cli_workspace_scope_rejects_narrower_target_ids(extra): + args = importer._parser().parse_args([ + "documents", "C:/files", "--scope", "workspace", *extra, + ]) + with pytest.raises(ValueError, match="requires --repo and --session to be omitted"): + importer._scope(args) + + +def test_service_workspace_scope_rejects_session_before_writes(): + service = MemoryService.create( + ":memory:", embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + try: + with pytest.raises( + ValidationError, match="requires repo and session_id to be omitted", + ): + service.preview_document_upload( + files=[("note.txt", b"Local note")], attachment_manifest=[], + workspace="must-not-exist", source_label="Loose files", + session_id="ses_example", scope="workspace", + ) + assert service._lookup_workspace("must-not-exist") is None + finally: + service.close() + + +def test_universal_service_imports_mixed_tree_idempotently(tmp_path): + root = tmp_path / "Archive" + (root / "notes").mkdir(parents=True) + (root / "notes" / "plan.md").write_text( + "# Plan\n\nShip the local importer.\n", encoding="utf-8", + ) + (root / "meeting.txt").write_text( + "Meeting notes\n\nThe launch is Tuesday.\n", encoding="utf-8", + ) + (root / "status.json").write_text( + '{"state": "ready", "owner": "local"}', encoding="utf-8", + ) + database = tmp_path / "memory.db" + service = MemoryService.create( + str(database), embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + try: + preview = service.preview_document_tree(str(root), workspace="acme") + assert preview["adapter"] == "documents" + assert preview["counts"]["documents"] == 3 + assert preview["summary"]["formats"] == { + "json": 1, "markdown": 1, "text": 1, + } + + first = service.import_document_tree( + str(root), workspace="acme", confirmed=True, + ) + assert first["state"] == "completed", { + key: first[key] for key in ("state", "counts", "files") + } + assert first["counts"]["imported"] == 3 + second = service.import_document_tree( + str(root), workspace="acme", source_id=first["source_id"], + confirmed=True, + ) + assert second["counts"]["skipped"] == 3 + finally: + service.close() + + with sqlite3.connect(database) as connection: + rows = connection.execute("SELECT metadata FROM memories").fetchall() + metadata = [json.loads(row[0]) for row in rows] + assert len(metadata) == 3 + assert {item["document"]["format"] for item in metadata} == { + "json", "markdown", "text", + } + assert all("obsidian" not in item for item in metadata) + + +def test_extensionless_document_links_remain_ambiguous(tmp_path): + root = tmp_path / "Ambiguous" + root.mkdir() + (root / "Source.md").write_text( + "# Source\n\nSee [[foo]].\n", encoding="utf-8", + ) + (root / "foo.md").write_text("# Markdown\n", encoding="utf-8") + (root / "foo.txt").write_text("Plain text\n", encoding="utf-8") + service = MemoryService.create( + str(tmp_path / "memory.db"), embed_dim=64, extractor="none", + graph_extractor="none", retention_supervisor="none", + ) + try: + report = service.import_document_tree( + str(root), workspace="acme", confirmed=True, + ) + assert report["counts"]["warning"] == 1 + assert any( + row["reason"] == "ambiguous_wikilink" + for row in report["files"] + ) + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE valid_to IS NULL" + ).fetchone()[0] == 0 + finally: + service.close() + + +def test_rejected_existing_document_is_not_reported_as_missing(tmp_path): + root = tmp_path / "Documents" + root.mkdir() + path = root / "note.txt" + path.write_text("Safe local note.", encoding="utf-8") + service = MemoryService.create( + str(tmp_path / "memory.db"), embed_dim=64, extractor="none", + graph_extractor="none", retention_supervisor="none", + ) + try: + first = service.import_document_tree( + str(root), workspace="acme", confirmed=True, + ) + path.write_text("api_key: very-secret-value", encoding="utf-8") + report = service.import_document_tree( + str(root), workspace="acme", source_id=first["source_id"], + confirmed=True, + ) + assert report["counts"]["rejected"] == 1 + assert report["counts"].get("missing", 0) == 0 + item = service.store.conn.execute( + "SELECT state FROM source_imports WHERE relative_path='note.txt'" + ).fetchone() + assert item["state"] == "imported" + finally: + service.close() + + +def test_universal_upload_job_reports_adapter_and_registered_source(tmp_path): + service = MemoryService.create( + str(tmp_path / "memory.db"), embed_dim=64, extractor="none", + graph_extractor="none", retention_supervisor="none", + ) + try: + with pytest.raises(ValidationError, match="source_label is required"): + service.import_document_upload( + files=[("brief.txt", b"Local release brief.")], + attachment_manifest=[], workspace="must-not-exist", confirmed=True, + ) + assert service._lookup_workspace("must-not-exist") is None + + started = service.import_document_upload( + files=[ + ("brief.txt", b"Local release brief."), + ("notes/context.md", b"# Context\n\nOffline and repeatable."), + ], + attachment_manifest=[], workspace="acme", + source_label="Caf\u00e9 documents", confirmed=True, + ) + assert started["adapter"] == "documents" + worker = service._obsidian_job_threads[started["job_id"]] + worker.join(30) + assert not worker.is_alive() + + job = service.get_document_import_job( + started["job_id"], workspace="acme", + ) + assert job["kind"] == "document_import" + assert job["state"] == "completed" + assert job["counts"]["documents"] == 2 + assert job["processed_items"] == job["total_items"] == 2 + assert {row["format"] for row in job["files"]} == {"markdown", "text"} + assert all("content" not in row for row in job["files"]) + sources = service.list_document_sources("acme") + assert len(sources) == 1 + assert sources[0]["id"] == started["source_id"] + assert sources[0]["label"] == "Caf\u00e9 documents" + assert sources[0]["kind"] == "documents" + assert sources[0]["adapter"] == "documents" + with pytest.raises(ValidationError, match="select its source_id"): + service.preview_document_upload( + files=[("other.txt", b"A separate collection")], + attachment_manifest=[], workspace="acme", + source_label="Cafe\u0301 DOCUMENTS", + ) + with pytest.raises(ValidationError, match="select its source_id"): + service.import_document_upload( + files=[("other.txt", b"A separate collection")], + attachment_manifest=[], workspace="acme", + source_label="Cafe\u0301 documents", confirmed=True, + ) + resumed_preview = service.preview_document_upload( + files=[("brief.txt", b"Local release brief.")], + attachment_manifest=[], workspace="acme", + source_id=started["source_id"], + ) + assert resumed_preview["source_id"] == started["source_id"] + assert resumed_preview["source_label"] == "Caf\u00e9 documents" + finally: + service.close() diff --git a/tests/test_document_importer.py b/tests/test_document_importer.py new file mode 100644 index 00000000..a77affc2 --- /dev/null +++ b/tests/test_document_importer.py @@ -0,0 +1,477 @@ +from __future__ import annotations + +import hashlib +import json +import socket +import sys +import types + +import pytest + +from engraphis.core.documents import ( + DOCUMENT_FORMATS, + DocumentFileIssue, + DocumentRecord, + DocumentScan, + parse_document, +) +from engraphis.core.interfaces import MemoryType, Scope, SearchFilter +from engraphis.document_import import ( + DocumentImporter, + local_document_adapter, + scan_document_upload, +) +from engraphis.obsidian_import import ObsidianImporter +from engraphis.service import MemoryService + + +def _service() -> MemoryService: + return MemoryService.create( + ":memory:", embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + + +def _scan(*files: tuple[str, bytes]) -> DocumentScan: + scan = DocumentScan(root_path="", source_id="d" * 64) + scan.documents.extend(parse_document(raw, path) for path, raw in files) + return scan + + +def test_mixed_document_import_is_temporal_repeatable_and_source_neutral(monkeypatch): + def no_network(*_args, **_kwargs): + raise AssertionError("document import attempted a network call") + + monkeypatch.setattr(socket, "create_connection", no_network) + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("mixed") + scan = _scan( + ("notes/readme.md", b"# Read me\nSee [plan](../plan.txt).\n"), + ("plan.txt", b"Ship the universal importer.\n"), + ("data/info.json", b'{"title":"Facts","enabled":true}'), + ) + importer = DocumentImporter(service) + before = { + table: service.store.conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + for table in ("memories", "source_vaults", "source_imports", "jobs", "operation_receipts") + } + preview = importer.preview( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_label="Loose files", + ) + after = { + table: service.store.conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + for table in before + } + assert after == before + assert preview["counts"]["documents"] == 3 + assert preview["summary"]["formats"] == {"json": 1, "markdown": 1, "text": 1} + + first = importer.import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_label="Loose files", confirmed=True, + ) + assert first["state"] == "completed" + assert first["counts"]["imported"] == 3 + assert service.store.list_source_vaults(kind="documents")[0]["id"] == first["source_id"] + memories = service.store.list_memories(SearchFilter(workspace_id=workspace_id)) + assert len(memories) == 3 + plan = next(memory for memory in memories if memory.title == "plan") + assert plan.metadata["document"]["relative_path"] == "plan.txt" + assert plan.metadata["document"]["format"] == "text" + assert plan.claim_kind == "source_document" + + again = importer.import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], source_label="Loose files", confirmed=True, + ) + assert again["counts"]["skipped"] == 3 + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 3 + + revised = _scan( + ("notes/readme.md", b"# Read me\nSee [plan](../plan.txt).\n"), + ("plan.txt", b"Ship the improved universal importer.\n"), + ("data/info.json", b'{"title":"Facts","enabled":true}'), + ) + changed = importer.import_scan( + revised, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], source_label="Loose files", confirmed=True, + ) + assert changed["counts"]["updated"] == 1 + history = service.store.conn.execute( + "SELECT valid_to FROM memories WHERE subject_key=? ORDER BY valid_from", + (plan.subject_key,), + ).fetchall() + assert len(history) == 2 + assert history[0]["valid_to"] is not None and history[1]["valid_to"] is None + assert service.store.conn.execute( + "SELECT COUNT(*) FROM operation_receipts WHERE operation='document_import'" + ).fetchone()[0] == 3 + finally: + service.close() + + +def test_pdf_uses_optional_local_resource_adapter(monkeypatch): + class Page: + @staticmethod + def extract_text(): + return "Quarterly report" + + class Reader: + def __init__(self, _stream): + self.pages = [Page()] + + monkeypatch.setitem(sys.modules, "pypdf", types.SimpleNamespace(PdfReader=Reader)) + record = parse_document( + b"%PDF-local", "reports/q1.pdf", adapter=local_document_adapter, + ) + assert record.format == "pdf" + assert record.body == "Quarterly report" + assert record.metadata["pages"] == 1 + + +def test_transcription_adapter_requires_an_existing_local_model(monkeypatch): + monkeypatch.delenv("ENGRAPHIS_WHISPER_MODEL", raising=False) + with pytest.raises(ValueError, match="local model file or directory"): + local_document_adapter(b"not-used", "meeting.mp3") + + +def test_transcription_adapter_accepts_a_local_model_directory(monkeypatch, tmp_path): + from engraphis.backends import resources + + model = tmp_path / "whisper-model" + model.mkdir() + monkeypatch.setenv("ENGRAPHIS_WHISPER_MODEL", str(model)) + monkeypatch.setattr( + resources, "get_resource_extractor", + lambda: types.SimpleNamespace(extract_bytes=lambda _name, _raw: types.SimpleNamespace( + text="Local transcript", media_type="audio/mpeg", title="Meeting", + kind="transcript", metadata={"duration": 3}, warnings=[], + )), + ) + + record = parse_document( + b"local audio bytes", "meeting.mp3", adapter=local_document_adapter, + ) + assert record.body == "Local transcript" + assert record.format == "audio" + + +def test_upload_rejects_casefold_and_nfc_path_collisions(): + with pytest.raises(ValueError, match="source_label is required"): + scan_document_upload([("note.txt", b"note")], source_label=" ") + + scan = scan_document_upload( + [ + ("Notes/Caf\u00e9.txt", b"first"), + ("notes/cafe\u0301.txt", b"second"), + ], + source_label="Portable paths", + ) + assert [item.relative_path for item in scan.documents] == ["Notes/Caf\u00e9.txt"] + assert [(item.relative_path, item.reason) for item in scan.rejected] == [ + ("notes/caf\u00e9.txt", "duplicate upload path"), + ] + + +def test_document_links_normalize_parent_paths_and_link_imported_attachments(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("links") + source = parse_document( + b"[plan](../plan.txt) [local](#section) " + b"[web](https://example.test/x) ![report](../reports/q1.pdf)", + "notes/readme.txt", + ) + plan = parse_document(b"The release plan.", "plan.txt") + raw_pdf = b"%PDF-local" + pdf = DocumentRecord( + relative_path="reports/q1.pdf", format="pdf", + media_type="application/pdf", title="Quarterly report", + content="Quarterly report", body="Quarterly report", + raw_sha256=hashlib.sha256(raw_pdf).hexdigest(), + canonical_sha256=hashlib.sha256(b"Quarterly report").hexdigest(), + source_size=len(raw_pdf), title_source="extracted", + ) + scan = DocumentScan(root_path="", source_id="e" * 64) + scan.documents.extend((source, plan, pdf)) + report = DocumentImporter(service).import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_label="Linked files", confirmed=True, + ) + + assert report["state"] == "completed" + rows = service.store.conn.execute( + "SELECT a.title AS source_title, b.title AS target_title, l.relation " + "FROM mem_links l JOIN memories a ON a.id=l.a " + "JOIN memories b ON b.id=l.b WHERE l.valid_to IS NULL " + "ORDER BY b.title" + ).fetchall() + assert [(row["source_title"], row["target_title"], row["relation"]) for row in rows] == [ + ("readme", "Quarterly report", "embeds"), + ("readme", "plan", "references"), + ] + assert report["counts"].get("warning", 0) == 0 + finally: + service.close() + + +def test_failed_document_revision_stays_seen_and_resumes(monkeypatch): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("resume") + importer = DocumentImporter(service) + first_scan = _scan(("note.txt", b"Durable version one.")) + first = importer.import_scan( + first_scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, + ) + changed = _scan(("note.txt", b"Durable version two.")) + original = service.store.upsert_source_import_item + + def fail_finalizer(**kwargs): + if not kwargs.get("commit", True): + raise RuntimeError("injected finalizer failure") + return original(**kwargs) + + monkeypatch.setattr(service.store, "upsert_source_import_item", fail_finalizer) + failed = importer.import_scan( + changed, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], confirmed=True, + ) + assert failed["counts"]["error"] == 1 + manifest = service.store.list_source_import_items(vault_id=first["source_id"])[0] + assert manifest["state"] == "error" + assert manifest["missing_at"] is None + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 1 + + monkeypatch.setattr(service.store, "upsert_source_import_item", original) + resumed = importer.import_scan( + changed, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], confirmed=True, + ) + assert resumed["counts"]["updated"] == 1 + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 2 + finally: + service.close() + + +def test_cancelled_import_reports_planned_missing_rows_as_pending(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("cancel-missing") + importer = DocumentImporter(service) + first = importer.import_scan( + _scan(("keep.txt", b"Keep me."), ("gone.txt", b"Gone later.")), + workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, + ) + cancelled = importer.import_scan( + _scan(("keep.txt", b"Keep me.")), + workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], confirmed=True, + cancel_check=lambda: True, + ) + assert cancelled["state"] == "cancelled" + assert cancelled["counts"].get("missing", 0) == 0 + assert cancelled["counts"]["pending"] == 2 + assert { + row["relative_path"] for row in cancelled["files"] + if row["status"] == "pending" + } == {"keep.txt", "gone.txt"} + item = service.store.conn.execute( + "SELECT state FROM source_imports WHERE relative_path='gone.txt'" + ).fetchone() + assert item["state"] == "imported" + finally: + service.close() + + +def test_cancelled_import_reports_unprocessed_plans_as_pending(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("cancel-plans") + importer = DocumentImporter(service) + cancelled = importer.import_scan( + _scan(("first.txt", b"First."), ("second.txt", b"Second.")), + workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, cancel_check=lambda: True, + ) + assert cancelled["state"] == "cancelled" + assert cancelled["counts"]["documents"] == 2 + assert cancelled["counts"]["pending"] == 2 + assert { + row["relative_path"] for row in cancelled["files"] + if row["status"] == "pending" + } == {"first.txt", "second.txt"} + job_items = service.store.conn.execute( + "SELECT relative_path, result_state FROM source_import_items " + "WHERE job_id=? ORDER BY relative_path", + (cancelled["job_id"],), + ).fetchall() + assert [(row["relative_path"], row["result_state"]) for row in job_items] == [ + ("first.txt", "pending"), ("second.txt", "pending"), + ] + finally: + service.close() + + +def test_unreadable_directory_does_not_finalize_descendants_as_missing(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("unreadable-dir") + importer = DocumentImporter(service) + first = importer.import_scan( + _scan(("blocked/gone.txt", b"Keep me.")), + workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, + ) + scan = DocumentScan(root_path="", source_id="d" * 64) + scan.skipped.append(DocumentFileIssue("blocked", "unreadable directory")) + report = importer.import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], confirmed=True, + ) + assert report["state"] == "partial" + assert report["counts"].get("missing", 0) == 0 + assert report["counts"]["skipped"] == 1 + item = service.store.conn.execute( + "SELECT state FROM source_imports WHERE relative_path='blocked/gone.txt'" + ).fetchone() + assert item["state"] == "imported" + finally: + service.close() + + +def test_incomplete_document_scan_defers_missing_reconciliation(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("incomplete-scan") + importer = DocumentImporter(service) + first = importer.import_scan( + _scan(("keep.txt", b"Keep me."), ("gone.txt", b"Gone later.")), + workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, + ) + scan = _scan(("keep.txt", b"Keep me.")) + scan.complete = False + report = importer.import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=first["source_id"], confirmed=True, + ) + assert report["state"] == "partial" + assert report["counts"].get("missing", 0) == 0 + assert report["counts"].get("pending", 0) == 1 + item = service.store.conn.execute( + "SELECT state FROM source_imports WHERE relative_path='gone.txt'" + ).fetchone() + assert item["state"] == "imported" + finally: + service.close() + + +def test_document_importer_rejects_an_obsidian_source_identity(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("identity") + scan = _scan(("note.txt", b"Source neutral.")) + wrong_id = service.store.register_source_vault( + kind="obsidian", root_digest=scan.source_id, + workspace_id=workspace_id, repo_id=None, session_id=None, + display_name="Wrong adapter", scope="workspace", + memory_type="semantic", importer_version="1", + ) + with pytest.raises(ValueError, match="different import adapter"): + DocumentImporter(service).preview( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_id=wrong_id, + ) + # The same identity remains valid for its compatibility adapter. + ObsidianImporter(service).preview( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + vault_id=wrong_id, + ) + finally: + service.close() + + +def test_format_metadata_and_rejected_items_stay_bounded_and_complete(): + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("bounds") + note = parse_document(b"Bounded metadata.", "bounded.txt") + note.metadata = {f"field_{index}": "x" * 1_000 for index in range(64)} + envelope = DocumentImporter._metadata( + note, vault_id="vlt_preview", source_id="src_preview", + imported_at=1.0, actor="test", branch="", + ) + assert len(json.dumps(envelope, ensure_ascii=False).encode("utf-8")) < 14_500 + assert envelope["document"]["omitted_counts"]["format_metadata"] > 0 + + scan = DocumentScan(root_path="", source_id="f" * 64) + scan.documents.append(note) + scan.rejected.append(DocumentFileIssue("broken.pdf", "source rejected")) + report = DocumentImporter(service).import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + confirmed=True, + ) + assert report["state"] == "partial" + job = service.store.conn.execute( + "SELECT total_items, processed_items FROM jobs WHERE id=?", + (report["job_id"],), + ).fetchone() + assert (job["total_items"], job["processed_items"]) == (2, 2) + persisted = service.store.list_source_import_job_items(job_id=report["job_id"]) + imported = next(row for row in persisted if row["relative_path"] == "bounded.txt") + assert imported["source_format"] == "text" + finally: + service.close() + + +def test_every_registered_format_survives_the_generic_import_handoff(): + """Dispatch is parser-tested; this covers the shared v2 persistence contract.""" + service = _service() + try: + workspace_id = service.store.get_or_create_workspace("all-formats") + scan = DocumentScan(root_path="", source_id="a" * 64) + for index, spec in enumerate(DOCUMENT_FORMATS.values()): + raw = ("source-%d" % index).encode("ascii") + text = "Readable %s source" % spec.name + scan.documents.append(DocumentRecord( + relative_path="format_%d%s" % (index, spec.extensions[0]), + format=spec.name, media_type=spec.media_type, title=spec.name, + content=text, body=text, + raw_sha256=hashlib.sha256(raw).hexdigest(), + canonical_sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(), + source_size=len(raw), title_source="fixture", + )) + report = DocumentImporter(service).import_scan( + scan, workspace_id=workspace_id, repo_id=None, session_id=None, + scope=Scope.WORKSPACE, memory_type=MemoryType.SEMANTIC, + source_label="Format contract", confirmed=True, + ) + assert report["state"] == "completed" + assert report["counts"]["imported"] == len(DOCUMENT_FORMATS) + rows = service.store.list_source_import_job_items(job_id=report["job_id"]) + assert {row["source_format"] for row in rows} == set(DOCUMENT_FORMATS) + finally: + service.close() diff --git a/tests/test_documentation_contracts.py b/tests/test_documentation_contracts.py index ed9cf59d..09a46dc7 100644 --- a/tests/test_documentation_contracts.py +++ b/tests/test_documentation_contracts.py @@ -232,10 +232,10 @@ def test_schema_and_erasure_docs_match_live_export_policy() -> None: erasure = _read("docs/SECURE_ERASURE.md") schema = _read("engraphis/core/schema.py") - assert "SCHEMA_VERSION = 15" in schema - assert agents.count("`SCHEMA_VERSION = 15`") == 2 - assert "schema 15" in readme - assert "schema 15" in changelog + assert "SCHEMA_VERSION = 16" in schema + assert agents.count("`SCHEMA_VERSION = 16`") == 2 + assert "schema 16" in readme + assert "schema 16" in changelog for document in (agents, readme, changelog, sync, erasure): normalized = " ".join(document.split()) @@ -249,6 +249,29 @@ def test_schema_and_erasure_docs_match_live_export_policy() -> None: assert "only a non-secret workspace/repo record" in erasure +def test_document_import_docs_describe_the_source_neutral_contract() -> None: + readme = _read("README.md") + agents = _read("AGENTS.md") + guide = _read("docs/DOCUMENT_IMPORT.md") + obsidian = _read("docs/OBSIDIAN_IMPORT.md") + + for document in (readme, guide): + assert "engraphis import documents" in document + assert "--dry-run" in document + assert "--yes" in document + for format_name in ( + "Markdown", "reStructuredText", "HTML", "JSON", "CSV", "DOCX", "ODT", + "RTF", "XLSX", "ODS", "PPTX", "ODP", "EPUB", "Source code", + ): + assert format_name in guide + for safety_term in ("symlink", "secret", "unsupported", "resumable", "temporal", "conflict"): + assert safety_term in guide + assert "SCHEMA_VERSION = 16" in agents + assert "source-neutral" in agents + assert "rich Markdown adapter" in obsidian + assert "DOCUMENT_IMPORT.md" in obsidian + + def test_consolidation_docs_expose_only_live_public_options() -> None: readme = _read("README.md") tools = _read("skills/engraphis-memory/references/TOOLS.md") diff --git a/tests/test_documents.py b/tests/test_documents.py new file mode 100644 index 00000000..2aa8bb4b --- /dev/null +++ b/tests/test_documents.py @@ -0,0 +1,822 @@ +"""Focused coverage for the dependency-free universal document parser.""" +from __future__ import annotations + +import hashlib +import io +from pathlib import Path +import zipfile +from typing import Union + +import pytest + +from engraphis.core.documents import ( + DOCUMENT_FORMATS, + MAX_DOCUMENT_CHARS, + MAX_DOCUMENT_WARNINGS, + DocumentRecord, + DocumentParseError, + document_format_for_path, + normalize_document_path, + parse_document, + scan_document_tree, +) + + +def _zip(parts: dict[str, Union[str, bytes]]) -> bytes: + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as archive: + for path, content in parts.items(): + archive.writestr(path, content) + return output.getvalue() + + +def test_markdown_uses_obsidian_adapter_and_masks_code_discovery(): + record = parse_document( + b"---\ntags: [project]\n---\n# Design\n[[Roadmap]] #active\n```md\n[[hidden]] #nope\n```\n", + "notes/design.md", + ) + assert record.format == "markdown" + assert record.title == "Design" + assert record.tags == ["project", "active"] + assert [link.target for link in record.links] == ["Roadmap"] + assert record.metadata["adapter"] == "obsidian-markdown" + + +def test_markdown_uses_detected_utf16_encoding_for_obsidian_parser(): + raw = "---\ntitle: Café\n---\n# Roadmap\n[[Launch]]\n".encode("utf-16") + + record = parse_document(raw, "notes/utf16.md") + + assert record.title == "Café" + assert record.body == "# Roadmap\n[[Launch]]\n" + assert [link.target for link in record.links] == ["Launch"] + assert record.source_size == len(raw) + assert record.raw_sha256 == hashlib.sha256(raw).hexdigest() + assert "\x00" not in record.content + + +@pytest.mark.parametrize( + ("name", "raw", "expected"), + [ + ("note.txt", b"# kept literal\nplain #tag", "text"), + ("guide.rst", b"Guide\n=====\n\nA link https://example.test.", "rst"), + ("page.html", b"Page

Hello world

", "html"), + ("data.json", b'{"title":"Inventory", "items":[1,2]}', "json"), + ("data.csv", b"name,role\nAda,Engineer\n", "csv"), + ("data.tsv", b"name\trole\nAda\tEngineer\n", "tsv"), + ], +) +def test_common_text_formats_preserve_readable_content(name, raw, expected): + record = parse_document(raw, name) + assert record.format == expected + assert record.content + assert record.body + assert len(record.raw_sha256) == len(record.canonical_sha256) == 64 + if expected == "html": + assert "Hello world" in record.body and "bad" not in record.body + assert record.title == "Page" + if expected == "csv": + assert record.metadata["columns"] == ["name", "role"] + assert record.metadata["rows"] == 1 + + +def test_json_lines_are_parsed_as_independent_records(): + record = parse_document( + b'{"id":1,"name":"one"}\n{"id":2,"name":"two"}\n', + "records.jsonl", + ) + assert record.metadata == {"json_kind": "jsonl", "records": 2} + assert '"name": "two"' in record.body + assert record.warnings == [] + + +def test_stdlib_container_formats_extract_docx_odt_and_epub(): + docx = _zip({ + "word/document.xml": ( + '' + "Deployment plan" + ), + }) + odt = _zip({ + "content.xml": ( + '' + "ODT body" + "" + ), + }) + epub = _zip({ + "META-INF/container.xml": ( + '' + '' + ), + "EPUB/book.opf": ( + 'Book' + '' + "" + ), + "EPUB/one.xhtml": "

Chapter

EPUB body

", + }) + assert parse_document(docx, "plan.docx").body == "Deployment plan" + assert parse_document(odt, "plan.odt").body == "ODT body" + epub_record = parse_document(epub, "book.epub") + assert epub_record.title == "Book" and "EPUB body" in epub_record.body + + +def test_epub_decodes_manifest_urls_before_archive_lookup(): + epub = _zip({ + "META-INF/container.xml": ( + '' + '' + ), + "EPUB/book.opf": ( + 'Book' + '' + '' + ), + "EPUB/one chapter.xhtml": "

Encoded chapter path

", + }) + + record = parse_document(epub, "book.epub") + + assert record.body == "Encoded chapter path" + + +def test_epub_titles_are_checked_for_secrets(): + epub = _zip({ + "META-INF/container.xml": ( + '' + '' + ), + "EPUB/book.opf": ( + '' + 'api_key=sk-proj-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + '' + '' + ), + "EPUB/one.xhtml": "

Safe chapter

", + }) + with pytest.raises(DocumentParseError, match="secret"): + parse_document(epub, "secret-title.epub") + + +def test_epub_titles_are_bounded_before_record_creation(): + oversized = "T" * (MAX_DOCUMENT_CHARS + 1) + epub = _zip({ + "META-INF/container.xml": ( + '' + '' + ), + "EPUB/book.opf": ( + '' + f"{oversized}" + '' + '' + ), + "EPUB/one.xhtml": "

Safe chapter

", + }) + + record = parse_document(epub, "oversized-title.epub") + + assert len(record.title) == MAX_DOCUMENT_CHARS + assert len(record.metadata["title"]) == MAX_DOCUMENT_CHARS + + +def test_ods_extracts_attribute_backed_cells(): + ods = _zip({ + "content.xml": ( + '' + '' + '' + '' + '' + '' + ), + }) + record = parse_document(ods, "values.ods") + assert record.body == "42\ttrue" + + +def test_ods_repeated_cells_are_bounded_before_materialization(): + oversized = "x" * 20_000 + ods = _zip({ + "content.xml": ( + '' + '' + f'' + '' + ), + }) + + with pytest.raises(DocumentParseError, match="exceeds"): + parse_document(ods, "repeated.ods") + + +def test_ods_repeated_rows_are_preserved_and_bounded(): + ods = _zip({ + "content.xml": ( + '' + '' + '' + 'Repeated' + '' + '' + ), + }) + + record = parse_document(ods, "repeated-rows.ods") + + assert record.body == "Repeated\nRepeated\nRepeated" + assert record.metadata == {"rows": 3, "cells": 3} + + +def test_scan_is_safe_and_continues_after_per_file_errors(tmp_path): + (tmp_path / "notes").mkdir() + (tmp_path / "notes" / "good.txt").write_text("good", encoding="utf-8") + (tmp_path / "notes" / "bad.bin").write_bytes(b"\0\1") + (tmp_path / "secret.txt").write_text("api_key: very-secret-value", encoding="utf-8") + (tmp_path / ".hidden.txt").write_text("skip", encoding="utf-8") + outside = tmp_path.parent / "outside.txt" + outside.write_text("outside", encoding="utf-8") + try: + (tmp_path / "linked.txt").symlink_to(outside) + except (NotImplementedError, OSError): + pass + scan = scan_document_tree(tmp_path) + assert [item.relative_path for item in scan.documents] == ["notes/good.txt"] + assert ("notes/bad.bin", "unsupported document format") in [(x.relative_path, x.reason) for x in scan.skipped] + assert {item.relative_path for item in scan.rejected} == {"secret.txt"} + assert ".hidden.txt" in {item.relative_path for item in scan.skipped} + + +@pytest.mark.parametrize("path", ["../x.txt", "/x.txt", "C:/x.txt", "a/x.txt:stream", " x.txt", ""]) +def test_paths_and_unsupported_or_dangerous_containers_fail_closed(path): + with pytest.raises(DocumentParseError): + normalize_document_path(path) + with pytest.raises(DocumentParseError, match="unsupported"): + parse_document(b"text", "unknown.xyz") + with pytest.raises(DocumentParseError, match="binary"): + parse_document(b"visible\x00hidden", "mislabelled.txt") + archive = _zip({"../outside.xml": "oops", "word/document.xml": ""}) + with pytest.raises(DocumentParseError, match="unsafe member"): + parse_document(archive, "unsafe.docx") + + +@pytest.mark.parametrize( + ("name", "raw"), + [ + ("empty.txt", b" \t\r\n"), + ("empty.rst", b"\n\n"), + ("empty.html", b"Metadata only"), + ("frontmatter.md", b"---\ntitle: Metadata only\ntags: [empty]\n---\n\n"), + ], +) +def test_blank_documents_are_rejected_before_import_preview(name, raw): + with pytest.raises(DocumentParseError, match="produced no readable text"): + parse_document(raw, name) + + +@pytest.mark.parametrize( + ("name", "raw", "expected_title"), + [ + ("settings.yaml", b"title: YAML title\nitems:\n - one\n", "YAML title"), + ("settings.toml", b'title = "TOML title"\n[build]\nvalue = 1\n', "TOML title"), + ("settings.ini", b"[app]\nname = INI title\n", "settings"), + ("config.xml", b'readablelink', "XML title"), + ("program.py", b"# https://example.test/should-not-link\n#tag\ndef run():\n return 1\n", "program"), + ], +) +def test_config_xml_and_source_formats_are_safe_and_readable(name, raw, expected_title): + record = parse_document(raw, name) + assert record.title == expected_title + assert record.content + if name.endswith(".xml"): + assert "readable" in record.body + assert [link.target for link in record.links] == ["https://example.test/a"] + if name.endswith(".py"): + assert record.tags == [] and record.links == [] and record.attachments == [] + + +def test_xml_honors_the_declared_encoding(): + raw = ( + '' + 'café' + ).encode("iso-8859-1") + record = parse_document(raw, "accented.xml") + assert record.title == "Café" + assert record.body == "café" + + +def test_xml_attributes_are_preserved_and_secrets_are_rejected(): + record = parse_document( + b'readable', + "config.xml", + ) + assert {item["name"] for item in record.metadata["xml_attributes"]} == { + "title", "host", + } + with pytest.raises(DocumentParseError, match="secret"): + parse_document( + b'ok', + "secret.xml", + ) + + +def test_adapter_metadata_and_title_secrets_are_rejected(): + raw = b"%PDF-local" + + def make_adapter(*, title="Report", metadata=None): + def adapter(data, path, mtime): + text = "safe extracted document" + return DocumentRecord( + relative_path=path, format="pdf", media_type="application/pdf", + title=title, content=text, body=text, + raw_sha256=hashlib.sha256(data).hexdigest(), + canonical_sha256=hashlib.sha256(text.encode()).hexdigest(), + source_size=len(data), source_mtime_ns=mtime, + metadata=metadata or {}, + ) + return adapter + + with pytest.raises(DocumentParseError, match="secret"): + parse_document( + raw, "report.pdf", + adapter=make_adapter(title="api_key=sk-proj-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + ) + with pytest.raises(DocumentParseError, match="secret"): + parse_document( + raw, "report.pdf", + adapter=make_adapter(metadata={"endpoint": "token=secret-value-123456789"}), + ) + + +@pytest.mark.parametrize( + "warnings", + [ + ["api_key=sk-proj-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"], + ["x" * (MAX_DOCUMENT_CHARS + 1)], + ["warning"] * (MAX_DOCUMENT_WARNINGS + 1), + ], +) +def test_adapter_warnings_are_bounded_and_checked(warnings): + raw = b"%PDF-local" + + def adapter(data, path, mtime): + text = "safe extracted document" + return DocumentRecord( + relative_path=path, format="pdf", media_type="application/pdf", + title="Report", content=text, body=text, + raw_sha256=hashlib.sha256(data).hexdigest(), + canonical_sha256=hashlib.sha256(text.encode()).hexdigest(), + source_size=len(data), source_mtime_ns=mtime, warnings=warnings, + ) + + with pytest.raises(DocumentParseError, match="warnings"): + parse_document(raw, "invalid-warnings.pdf", adapter=adapter) + + +def test_rtf_and_additional_office_containers_are_dependency_free(): + rtf = parse_document(b"{\\rtf1\\ansi Hello\\par world}", "notes.rtf") + assert "Hello" in rtf.body and "world" in rtf.body + unicode_rtf = parse_document( + b"{\\rtf1\\ansi\\uc1 Caf\\u233? {\\uc0\\u945} \\u233? Smile \\u-10179?\\u-8704?}", + "unicode.rtf", + ) + assert unicode_rtf.body == "Café α é Smile 😀" + hex_fallback_rtf = parse_document( + b"{\\rtf1\\ansi\\uc1 \\u945\\'3f}", + "hex-fallback.rtf", + ) + assert hex_fallback_rtf.body == "α" + cyrillic_rtf = parse_document( + b"{\\rtf1\\ansi\\ansicpg1251 \\'cf\\'f0\\'e8\\'ec\\'e5\\'f0}", + "cyrillic.rtf", + ) + assert cyrillic_rtf.body == "Пример" + literal_rtf = parse_document( + b"{\\rtf1\\ansi\\ansicpg1252 Caf\xe9}", + "literal.rtf", + ) + assert literal_rtf.body == "Café" + + # \binN skips N raw binary bytes (braces/backslashes inside are never + # parsed, so the group stack stays balanced) without emitting them. + bin_rtf = parse_document( + b"{\\rtf1\\ansi hello\\par\\pict\\bin8 ab{\\x7f world}", + "binary.rtf", + ) + assert bin_rtf.body == "hello" + assert "world" not in bin_rtf.body + assert bin_rtf.body == "hello" # \pict destination suppression drops trailing text + # An oversized \binN is rejected instead of trusting N. + with pytest.raises(DocumentParseError, match="invalid RTF"): + parse_document( + b"{\\rtf1\\ansi\\bin999999999}", + "oversized-binary.rtf", + ) + + xlsx = _zip({ + "xl/sharedStrings.xml": "Revenue", + "xl/worksheets/sheet1.xml": ( + "07" + "North" + " America" + ), + }) + pptx = _zip({ + "ppt/slides/slide1.xml": 'Release deck', + }) + ods = _zip({ + "content.xml": 'ODS cell', + }) + odp = _zip({ + "content.xml": 'ODP slide', + }) + assert "Revenue\t7\tNorth America" in parse_document(xlsx, "book.xlsx").body + assert parse_document(pptx, "slides.pptx").body == "Release deck" + assert parse_document(ods, "book.ods").body == "ODS cell" + assert parse_document(odp, "slides.odp").body == "ODP slide" + + +def test_markup_and_markdown_code_cannot_create_discovery_records(): + html = parse_document( + b'Title only

Visible heading

live
# hidden https://example.test/code
', + "page.html", + ) + assert [link.target for link in html.links] == ["https://example.test/live"] + assert html.headings == ["Visible heading"] + assert "Title only" not in html.body + assert "template-tag" not in html.body + markdown = parse_document( + b"```python\n[[Hidden]] [hidden](secret.md) #hidden\n````\n" + b"[[Visible]] [Table](../metrics.csv) ![Diagram](images/plot.png) #visible\n" + b"`[[also-hidden]] [hidden](private.md) #bad`\n", + "note.md", + ) + assert [link.target for link in markdown.links] == ["Visible", "../metrics.csv"] + assert [item.path for item in markdown.attachments] == ["images/plot.png"] + assert markdown.tags == ["visible"] + + +def test_mismatched_ignored_html_closing_tags_stay_hidden(): + html = parse_document( + b"

visible

", + "page.html", + ) + assert html.body == "visible" + + +def test_html_uses_declared_non_utf8_charset_before_parsing(): + html = parse_document( + b'Caf\xe9

Caf\xe9

', + "page.html", + ) + assert html.title == "Café" + assert html.body == "Café" + + +def test_html_charset_detection_ignores_comments_and_script_text(): + html = parse_document( + b'' + b'' + b'

Caf\xc3\xa9

', + "page.html", + ) + + assert html.body == "Café" + + +def test_xhtml_uses_xml_prolog_encoding_before_meta_charset(): + # Latin-1 declared in the XML prolog with no present. + xhtml = parse_document( + b'' + b"Caf\xe9", + "page.xhtml", + ) + assert xhtml.body == "Café" + # UTF-8 XHTML with a prolog still decodes correctly. + utf8_xhtml = parse_document( + b'' + b"Caf\xc3\xa9", + "page.xhtml", + ) + assert utf8_xhtml.body == "Café" + # The prolog may be preceded by whitespace. + spaced_xhtml = parse_document( + b' \n' + b"Caf\xe9", + "page.xhtml", + ) + assert spaced_xhtml.body == "Café" + + +def test_unreadable_directory_is_reported_and_marks_scan_incomplete(monkeypatch, tmp_path): + blocked = tmp_path / "blocked" + blocked.mkdir() + (blocked / "note.txt").write_text("durable", encoding="utf-8") + original_iterdir = Path.iterdir + + def fail_blocked(path): + if path == blocked: + raise OSError("permission denied") + return original_iterdir(path) + + monkeypatch.setattr(Path, "iterdir", fail_blocked) + scan = scan_document_tree(tmp_path) + assert scan.complete is False + assert any( + issue.relative_path == "blocked" and issue.reason == "unreadable directory" + for issue in scan.skipped + ) + + +def test_unreadable_root_is_reported_and_marks_scan_incomplete(monkeypatch, tmp_path): + original_iterdir = Path.iterdir + + def fail_root(path): + if path == tmp_path: + raise OSError("permission denied") + return original_iterdir(path) + + monkeypatch.setattr(Path, "iterdir", fail_root) + scan = scan_document_tree(tmp_path) + assert scan.complete is False + assert any( + issue.relative_path == "." and issue.reason == "unreadable directory" + for issue in scan.skipped + ) + + +def test_xml_and_container_attacks_and_invalid_rtf_fail_closed(): + with pytest.raises(DocumentParseError, match="entities"): + parse_document(b']>&boom;', "unsafe.xml") + with pytest.raises(DocumentParseError, match="invalid RTF"): + parse_document(b"not rtf", "unsafe.rtf") + encrypted = _zip({"word/document.xml": ""}) + payload = bytearray(encrypted) + flags_offset = payload.find(b"PK\x01\x02") + 8 + if flags_offset >= 8: + payload[flags_offset] |= 1 + with pytest.raises(DocumentParseError, match="encrypted"): + parse_document(bytes(payload), "encrypted.docx") + duplicate = io.BytesIO() + with zipfile.ZipFile(duplicate, "w") as archive: + archive.writestr("word/document.xml", "") + archive.writestr("word/document.xml", "") + with pytest.raises(DocumentParseError, match="duplicate"): + parse_document(duplicate.getvalue(), "duplicate.docx") + + +def test_adapter_contract_is_bounded_and_redacts_failures(): + raw = b"%PDF-local" + with pytest.raises(DocumentParseError, match="requires an optional"): + parse_document(raw, "report.pdf") + + def broken(*_args): + raise RuntimeError("secret-local-path") + + with pytest.raises(DocumentParseError, match="adapter failed"): + parse_document(raw, "report.pdf", adapter=broken) + + def adapter(data, path, mtime): + text = "extracted document" + return DocumentRecord( + relative_path=path, format="pdf", media_type="application/pdf", title="Report", + content=text, body=text, raw_sha256=__import__("hashlib").sha256(data).hexdigest(), + canonical_sha256=__import__("hashlib").sha256(text.encode()).hexdigest(), + source_size=len(data), source_mtime_ns=mtime, + ) + + assert parse_document(raw, "report.pdf", adapter=adapter).body == "extracted document" + + +def test_malformed_adapter_text_is_rejected_without_an_internal_exception(): + def adapter(data, path, mtime): + return DocumentRecord( + relative_path=path, format="pdf", media_type="application/pdf", title="Report", + content=object(), body="text", # type: ignore[arg-type] + raw_sha256=__import__("hashlib").sha256(data).hexdigest(), + canonical_sha256="not-reached", source_size=len(data), source_mtime_ns=mtime, + ) + + with pytest.raises(DocumentParseError, match="adapter returned invalid text"): + parse_document(b"%PDF", "report.pdf", adapter=adapter) + + def invalid_unicode_adapter(data, path, mtime): + content = "otherwise valid" + return DocumentRecord( + relative_path=path, format="pdf", media_type="application/pdf", title="Report", + content=content, body="bad \ud800 body", + raw_sha256=__import__("hashlib").sha256(data).hexdigest(), + canonical_sha256=__import__("hashlib").sha256(content.encode()).hexdigest(), + source_size=len(data), source_mtime_ns=mtime, + ) + + with pytest.raises(DocumentParseError, match="adapter returned invalid text"): + parse_document(b"%PDF", "report.pdf", adapter=invalid_unicode_adapter) + + +def test_every_registered_extension_has_a_stable_dispatch_and_adapter_contract(): + for spec in DOCUMENT_FORMATS.values(): + for extension in spec.extensions: + assert document_format_for_path("folder/document" + extension) == spec + + def adapter(data, path, mtime): + spec = document_format_for_path(path) + assert spec is not None + text = spec.name + " text" + import hashlib + return DocumentRecord( + relative_path=path, format=spec.name, media_type=spec.media_type, + title=spec.name, content=text, body=text, + raw_sha256=hashlib.sha256(data).hexdigest(), + canonical_sha256=hashlib.sha256(text.encode()).hexdigest(), + source_size=len(data), source_mtime_ns=mtime, + ) + + for spec in DOCUMENT_FORMATS.values(): + if spec.requires_adapter: + record = parse_document(b"local-adapter-input", "item" + spec.extensions[0], adapter=adapter) + assert record.format == spec.name + + +def test_text_and_container_bounds_apply_before_unbounded_materialization(monkeypatch): + from engraphis.core import documents + + def should_not_parse(_value): + raise AssertionError("oversized JSON reached the parser") + + monkeypatch.setattr(documents.json, "loads", should_not_parse) + with pytest.raises(DocumentParseError, match="100000 character"): + parse_document(b"{" + (b"x" * 100_000) + b"}", "huge.json") + + monkeypatch.setattr(documents, "parse_obsidian_note", should_not_parse) + with pytest.raises(DocumentParseError, match="100000 character"): + parse_document(b"#" + (b"x" * 100_000), "huge.md") + + monkeypatch.setattr(documents, "MAX_CONTAINER_TEXT_CHARS", 5) + docx = _zip({ + "word/document.xml": 'sixsix', + }) + with pytest.raises(DocumentParseError, match="100000 character"): + parse_document(docx, "huge.docx") + + xlsx = _zip({ + "xl/worksheets/sheet1.xml": ( + "" + "sixsix" + ), + }) + with pytest.raises(DocumentParseError, match="100000 character"): + parse_document(xlsx, "huge.xlsx") + + +def test_deep_json_is_preserved_without_unbounded_pretty_printing(): + raw = (b"[" * 256) + b"0" + (b"]" * 256) + record = parse_document(raw, "deep.json") + assert record.body == raw.decode("ascii") + assert any("nesting exceeds" in warning for warning in record.warnings) + + +def test_unreadable_directory_is_reported(monkeypatch, tmp_path): + blocked = tmp_path / "blocked" + blocked.mkdir() + (blocked / "note.txt").write_text("durable", encoding="utf-8") + original_iterdir = Path.iterdir + + def fail_blocked(path): + if path == blocked: + raise OSError("permission denied") + return original_iterdir(path) + + monkeypatch.setattr(Path, "iterdir", fail_blocked) + scan = scan_document_tree(tmp_path) + assert any( + issue.relative_path == "blocked" and issue.reason == "unreadable directory" + for issue in scan.skipped + ) + + +def test_oversized_directory_is_bounded_and_marks_scan_incomplete(monkeypatch, tmp_path): + """A single directory with more entries than MAX_DOCUMENT_FILES must not be + materialized into an unbounded sorted list: the walk emits a + "directory exceeds safety limit" issue for that directory and the scan result + is marked incomplete.""" + import engraphis.core.documents as documents_module + + monkeypatch.setattr(documents_module, "MAX_DOCUMENT_FILES", 5) + oversized = tmp_path / "oversized" + oversized.mkdir() + for index in range(10): + (oversized / f"note-{index}.md").write_text(f"# Note {index}\n", encoding="utf-8") + scan = scan_document_tree(tmp_path) + assert scan.complete is False + assert any( + issue.relative_path == "oversized" + and issue.reason == "directory exceeds safety limit" + for issue in scan.skipped + ) + + +def test_normalized_paths_are_bounded_and_portable(tmp_path): + assert normalize_document_path("notes/cafe\u0301.txt") == "notes/café.txt" + with pytest.raises(DocumentParseError, match="4096"): + normalize_document_path(("a" * 4093) + ".txt") + (tmp_path / "A.txt").write_text("one", encoding="utf-8") + (tmp_path / "a.txt").write_text("two", encoding="utf-8") + scan = scan_document_tree(tmp_path) + assert len(scan.documents) == 1 + if len(list(tmp_path.iterdir())) == 2: + assert any(item.reason == "duplicate normalized source path" for item in scan.rejected) + + +def _worksheet(marker: str) -> str: + return ( + '' + + marker + + "" + ) + + +def test_xlsx_extraction_follows_workbook_sheet_order(): + xlsx = _zip({ + "xl/workbook.xml": ( + '' + "" + '' + '' + '' + "" + ), + "xl/_rels/workbook.xml.rels": ( + '' + '' + '' + "" + ), + "xl/worksheets/sheet1.xml": _worksheet("First sheet"), + "xl/worksheets/sheet2.xml": _worksheet("Second sheet"), + "xl/worksheets/sheet3.xml": _worksheet("Third sheet"), + }) + record = parse_document(xlsx, "reordered.xlsx") + assert record.body == "Second sheet\nFirst sheet\nThird sheet" + assert record.metadata["sheets"] == 3 + assert record.metadata["rows"] == 3 + + +def test_xlsx_extraction_falls_back_to_numeric_order_without_workbook(): + xlsx = _zip({ + "xl/worksheets/sheet1.xml": _worksheet("First sheet"), + "xl/worksheets/sheet2.xml": _worksheet("Second sheet"), + "xl/worksheets/sheet10.xml": _worksheet("Tenth sheet"), + }) + record = parse_document(xlsx, "plain.xlsx") + assert record.body == "First sheet\nSecond sheet\nTenth sheet" + assert record.metadata["sheets"] == 3 + assert record.metadata["rows"] == 3 + + +def _slide(marker: str) -> str: + return '' + marker + "" + + +def test_pptx_extraction_follows_presentation_slide_order(): + pptx = _zip({ + "ppt/presentation.xml": ( + '' + "" + '' + '' + "" + ), + "ppt/_rels/presentation.xml.rels": ( + '' + '' + '' + '' + "" + ), + "ppt/slides/slide1.xml": _slide("First slide"), + "ppt/slides/slide2.xml": _slide("Second slide"), + "ppt/slides/slide3.xml": _slide("Third slide"), + }) + record = parse_document(pptx, "reordered.pptx") + assert record.body == "Second slide\n\nFirst slide\n\nThird slide" + assert record.metadata["slides"] == 3 + + +def test_pptx_extraction_falls_back_to_numeric_order_without_presentation(): + pptx = _zip({ + "ppt/slides/slide1.xml": _slide("First slide"), + "ppt/slides/slide2.xml": _slide("Second slide"), + "ppt/slides/slide10.xml": _slide("Tenth slide"), + }) + record = parse_document(pptx, "plain.pptx") + assert record.body == "First slide\n\nSecond slide\n\nTenth slide" + assert record.metadata["slides"] == 3 diff --git a/tests/test_encrypted_store.py b/tests/test_encrypted_store.py index 1796cc46..d503990c 100644 --- a/tests/test_encrypted_store.py +++ b/tests/test_encrypted_store.py @@ -7,6 +7,7 @@ key fails loudly, keys load from env or file, and the default (no key) path stays plaintext. """ import sqlite3 +from pathlib import Path import pytest @@ -24,6 +25,7 @@ def _require_sqlcipher(): pytest.importorskip("sqlcipher3", reason="encryption extra not installed") from engraphis.backends import encrypted_db # noqa: E402 +from engraphis.core.store import Store # noqa: E402 from engraphis.service import MemoryService # noqa: E402 KEY = "b3" * 32 # 64 hex chars → raw-key form @@ -82,6 +84,101 @@ def test_wrong_key_is_rejected(monkeypatch, tmp_path): MemoryService.create(db) +def test_existing_encrypted_database_opens_read_only_without_sidecar_mutation( + monkeypatch, tmp_path, +): + monkeypatch.setenv("ENGRAPHIS_DB_KEY", KEY) + db_path = tmp_path / "read-only-encrypted.db" + service = MemoryService.create(str(db_path)) + stored = service.remember( + "Encrypted immutable evidence.", workspace="demo", scope="workspace", + ) + service.engine.store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + service.engine.store.close() + tracked = [ + db_path, + tmp_path / "read-only-encrypted.db-wal", + tmp_path / "read-only-encrypted.db-shm", + tmp_path / "read-only-encrypted.db-journal", + ] + + def state(path): + return ( + path.exists(), + path.stat().st_size if path.exists() else None, + path.stat().st_mtime_ns if path.exists() else None, + ) + + before = {path.name: state(path) for path in tracked} + read_only = Store( + str(db_path), connect=encrypted_db.make_connector(KEY), read_only=True, + ) + try: + record = read_only.get_memory(stored["id"]) + assert record is not None and record.content == "Encrypted immutable evidence." + assert read_only.conn.execute("PRAGMA query_only").fetchone()[0] == 1 + with pytest.raises(sqlite3.OperationalError): + read_only.conn.execute( + "UPDATE memories SET content='changed' WHERE id=?", (stored["id"],) + ) + finally: + read_only.close() + assert {path.name: state(path) for path in tracked} == before + + +def test_encrypted_read_only_open_rejects_wrong_key(monkeypatch, tmp_path): + monkeypatch.setenv("ENGRAPHIS_DB_KEY", KEY) + db_path = tmp_path / "wrong-read-only-key.db" + service = MemoryService.create(str(db_path)) + service.engine.store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + service.engine.store.close() + tracked = [ + db_path, + tmp_path / "wrong-read-only-key.db-wal", + tmp_path / "wrong-read-only-key.db-shm", + tmp_path / "wrong-read-only-key.db-journal", + ] + + def state(path): + return ( + path.exists(), + path.stat().st_size if path.exists() else None, + path.stat().st_mtime_ns if path.exists() else None, + ) + + before = {path.name: state(path) for path in tracked} + + with pytest.raises(encrypted_db.EncryptionError): + Store( + str(db_path), connect=encrypted_db.make_connector("aa" * 32), + read_only=True, + ) + assert {path.name: state(path) for path in tracked} == before + + +def test_encrypted_read_only_open_rejects_active_wal_before_connector( + monkeypatch, tmp_path, +): + monkeypatch.setenv("ENGRAPHIS_DB_KEY", KEY) + db_path = tmp_path / "active-encrypted-wal.db" + writable = Store(str(db_path), connect=encrypted_db.make_connector(KEY)) + try: + writable.conn.execute("PRAGMA wal_autocheckpoint=0") + writable.get_or_create_workspace("active-encrypted-wal") + wal_path = tmp_path / "active-encrypted-wal.db-wal" + assert wal_path.is_file() and wal_path.stat().st_size > 0 + before = (wal_path.stat().st_size, wal_path.stat().st_mtime_ns) + + with pytest.raises(RuntimeError, match="active WAL found"): + Store( + str(db_path), connect=encrypted_db.make_connector(KEY), + read_only=True, + ) + assert (wal_path.stat().st_size, wal_path.stat().st_mtime_ns) == before + finally: + writable.close() + + def test_key_from_file(monkeypatch, tmp_path): keyfile = tmp_path / "db.key" keyfile.write_text(KEY + "\n") @@ -95,6 +192,44 @@ def test_key_from_file(monkeypatch, tmp_path): sqlite3.connect(db).execute("SELECT * FROM memories").fetchone() +def test_encrypted_manifest_snapshot_is_immutable_and_read_only(monkeypatch, tmp_path): + monkeypatch.setenv("ENGRAPHIS_DB_KEY", KEY) + db = str(tmp_path / "manifest.db") + service = MemoryService.create(db) + workspace_id = service.store.get_or_create_workspace("encrypted-manifest") + vault_id = service.store.register_source_vault( + kind="obsidian", root_digest="a" * 64, workspace_id=workspace_id, + display_name="Encrypted vault", + ) + source_id = service.store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="Notes/One.md", + content_sha256="c" * 64, importer_version="1", seen_at=10, + ) + service.store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + service.close() + tracked = [ + Path(db), Path(db + "-wal"), Path(db + "-shm"), Path(db + "-journal"), + ] + + def state(path): + return ( + path.exists(), + path.stat().st_size if path.exists() else None, + path.stat().st_mtime_ns if path.exists() else None, + ) + + before = {path.name: state(path) for path in tracked} + + snapshot = Store.snapshot_source_import_manifest( + db, connect=encrypted_db.connector_from_env(), + ) + + assert snapshot["schema_version"] >= 15 + assert [vault["id"] for vault in snapshot["vaults"]] == [vault_id] + assert [item["id"] for item in snapshot["items"]] == [source_id] + assert {path.name: state(path) for path in tracked} == before + + def test_passphrase_key_non_hex(monkeypatch, tmp_path): monkeypatch.setenv("ENGRAPHIS_DB_KEY", "correct horse battery staple") # → passphrase (KDF) db = str(tmp_path / "m.db") diff --git a/tests/test_init.py b/tests/test_init.py index 220402ec..204b3da6 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -163,7 +163,6 @@ def test_generated_encryption_key_is_private(tmp_path, monkeypatch): def test_installed_config_loads_the_trusted_env_from_an_unrelated_cwd( tmp_path, monkeypatch): """The wheel must consume the exact trusted file ``engraphis-init`` writes.""" - pytest.importorskip("dotenv") monkeypatch.chdir(tmp_path) target = tmp_path / "preserved.db" main(["--db", str(target)]) diff --git a/tests/test_memory_routes_fixes.py b/tests/test_memory_routes_fixes.py index c8ca30d5..11ce39a8 100644 --- a/tests/test_memory_routes_fixes.py +++ b/tests/test_memory_routes_fixes.py @@ -6,7 +6,6 @@ - POST /memory/conversations validates content and forwards the complete grounded history. - POST /memory/interactions recorded signals that never reinforced any memory. """ -import os import threading import numpy as np @@ -172,120 +171,101 @@ def test_upload_filename_stem_is_lexical_only(filename, expected): assert _filename_stem(filename) == expected -def test_import_folder_preserves_relative_case_and_uses_lexical_stem( +def test_legacy_folder_import_contains_path_before_filesystem_access( monkeypatch, tmp_path, ): + import os + from engraphis.routes import vault as vault_routes - root = tmp_path / "MixedRoot" - nested = root / "NestedDir" - nested.mkdir(parents=True) - (nested / "notes.v1.md").write_text("body", encoding="utf-8") - fake_home = tmp_path / "fake-home" - fake_home.mkdir() - monkeypatch.setenv("ENGRAPHIS_IMPORT_ROOTS", str(root)) - monkeypatch.setattr( - vault_routes.Path, - "home", - classmethod(lambda _cls: fake_home), - ) + client = _client(monkeypatch, tmp_path) + home = tmp_path / "decoy-home" + allowed = tmp_path / "allowed" + sibling = tmp_path / "allowed-sibling" + home.mkdir() + allowed.mkdir() + sibling.mkdir() + (allowed / "safe.md").write_text("# Safe\nAllowed content.\n", encoding="utf-8") + (sibling / "secret.md").write_text("must not be read", encoding="utf-8") + + monkeypatch.setattr(vault_routes.Path, "home", lambda: home) + monkeypatch.setenv("ENGRAPHIS_IMPORT_ROOTS", str(allowed)) + + original_exists = vault_routes.Path.exists + original_is_dir = vault_routes.Path.is_dir + original_rglob = vault_routes.Path.rglob + forbidden = os.path.normcase(str(sibling)) + + def reject_forbidden(path): + if os.path.normcase(str(path)) == forbidden: + pytest.fail("rejected import path reached the filesystem") + + def guarded_exists(path): + reject_forbidden(path) + return original_exists(path) + + def guarded_is_dir(path): + reject_forbidden(path) + return original_is_dir(path) + + def guarded_rglob(path, pattern): + reject_forbidden(path) + return original_rglob(path, pattern) + + monkeypatch.setattr(vault_routes.Path, "exists", guarded_exists) + monkeypatch.setattr(vault_routes.Path, "is_dir", guarded_is_dir) + monkeypatch.setattr(vault_routes.Path, "rglob", guarded_rglob) monkeypatch.setattr( vault_routes.ingest_engine, "ingest_document", - lambda **_kwargs: {"document_id": "notes-v1"}, + lambda **_kwargs: "doc", ) - with _client(monkeypatch, tmp_path) as client: - response = client.post( + with client: + rejected = client.post( "/memory/vaults/import-folder", - json={ - "path": str(root), - "namespace": "ns", - "file_pattern": "*.md", - }, + json={"path": str(sibling), "namespace": "ns"}, ) + assert rejected.status_code == 403 - assert response.status_code == 200 - result = response.json()["data"] - assert result["imported"] == 1 - assert result["files"] == [ - {"path": "NestedDir/notes.v1.md", "title": "notes.v1", "status": "ok"} - ] + accepted = client.post( + "/memory/vaults/import-folder", + json={"path": str(allowed), "namespace": "ns"}, + ) + assert accepted.status_code == 200 + assert accepted.json()["data"]["folder"] == str(allowed) + assert accepted.json()["data"]["imported"] == 1 -def test_import_folder_rejects_paths_outside_allowed_roots(monkeypatch, tmp_path): +def test_legacy_folder_import_skips_symlink_escape(monkeypatch, tmp_path): from engraphis.routes import vault as vault_routes + client = _client(monkeypatch, tmp_path) + home = tmp_path / "decoy-home" allowed = tmp_path / "allowed" outside = tmp_path / "outside" - fake_home = tmp_path / "fake-home" + home.mkdir() allowed.mkdir() outside.mkdir() - fake_home.mkdir() - (outside / "secret.md").write_text("secret", encoding="utf-8") + secret = outside / "secret.md" + secret.write_text("must not be imported", encoding="utf-8") + try: + (allowed / "escape.md").symlink_to(secret) + except (OSError, NotImplementedError): + pytest.skip("symlinks not supported in this environment") + + monkeypatch.setattr(vault_routes.Path, "home", lambda: home) monkeypatch.setenv("ENGRAPHIS_IMPORT_ROOTS", str(allowed)) - monkeypatch.setattr( - vault_routes.Path, - "home", - classmethod(lambda _cls: fake_home), - ) - with _client(monkeypatch, tmp_path) as client: + with client: response = client.post( "/memory/vaults/import-folder", - json={"path": str(outside), "namespace": "ns"}, + json={"path": str(allowed), "namespace": "ns"}, ) - assert response.status_code == 403 - assert response.json()["detail"].startswith("Import path must be under an allowed root") - -def test_import_folder_accepts_files_under_any_configured_root(monkeypatch, tmp_path): - """When ENGRAPHIS_IMPORT_ROOTS points outside home, files under ANY configured - root must be accepted (not rejected because they're not under ALL roots). - Regression for review thread #20: any(not _path_within_root(...)) required - every root to be an ancestor — should accept files under ANY configured root.""" - from engraphis.routes import vault as vault_routes - - # Two disjoint allowed roots - root_a = tmp_path / "data" / "root_a" - root_b = tmp_path / "data" / "root_b" - fake_home = tmp_path / "fake-home" - root_a.mkdir(parents=True) - root_b.mkdir(parents=True) - fake_home.mkdir() - - # File under root_b (not under root_a or fake_home) - (root_b / "note.md").write_text("# Disjoint Root Test", encoding="utf-8") - - # Configure both roots (disjoint from each other and from home) - monkeypatch.setenv("ENGRAPHIS_IMPORT_ROOTS", f"{root_a}{os.pathsep}{root_b}") - monkeypatch.setattr( - vault_routes.Path, - "home", - classmethod(lambda _cls: fake_home), - ) - monkeypatch.setattr( - vault_routes.ingest_engine, - "ingest_document", - lambda **_kwargs: {"document_id": "disjoint-test"}, - ) - - with _client(monkeypatch, tmp_path) as client: - response = client.post( - "/memory/vaults/import-folder", - json={ - "path": str(root_b), - "namespace": "ns", - "file_pattern": "*.md", - }, - ) - - # Must succeed (file is under root_b, one of the configured roots) assert response.status_code == 200 - result = response.json()["data"] - assert result["imported"] == 1 - assert result["files"][0]["status"] == "ok" + assert response.json()["data"]["imported"] == 0 def test_safe_call_classifies_and_sanitizes_legacy_failures(): diff --git a/tests/test_migration.py b/tests/test_migration.py index 61c189bc..b2ced0ab 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -131,6 +131,43 @@ def test_migration_writes_scoped_v2(tmp_path): store.close() +def test_migration_preserves_legacy_event_payload_entity_and_timestamp(tmp_path): + old = tmp_path / "engraphis_v1.db" + new = tmp_path / "engraphis_v2.db" + _build_v1_db(str(old)) + with sqlite3.connect(old) as connection: + connection.execute( + "CREATE TABLE events (id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, " + "entity_name TEXT NOT NULL, event_type TEXT NOT NULL, description TEXT, " + "payload TEXT NOT NULL, timestamp REAL NOT NULL)" + ) + connection.execute( + "INSERT INTO events(namespace, entity_name, event_type, description, payload, timestamp) " + "VALUES (?,?,?,?,?,?)", + ("infra", "PostgreSQL", "deploy", "release observed", '{"version":16}', 1234.5), + ) + + migrate(str(old), str(new)) + store = Store(str(new)) + event = store.conn.execute( + "SELECT content, refs, ts FROM events WHERE kind='deploy'" + ).fetchone() + assert event is not None + refs = json.loads(event["refs"]) + assert event["content"] == "release observed" + assert event["ts"] == 1234.5 + assert {item["kind"] for item in refs} == { + "v1_event_id", "v1_entity", "v1_payload" + } + assert {item["name"] for item in refs if item["kind"] == "v1_entity"} == { + "PostgreSQL" + } + assert [item["value"] for item in refs if item["kind"] == "v1_payload"] == [ + {"version": 16} + ] + store.close() + + def test_migration_preserves_same_name_entities_with_distinct_types(tmp_path): old = tmp_path / "engraphis_v1.db" new = tmp_path / "engraphis_v2.db" diff --git a/tests/test_obsidian_cli.py b/tests/test_obsidian_cli.py new file mode 100644 index 00000000..c501c712 --- /dev/null +++ b/tests/test_obsidian_cli.py @@ -0,0 +1,222 @@ +"""Console, compatibility, and packaging contracts for the v2 Obsidian importer.""" +from __future__ import annotations + +import json +import io +from pathlib import Path +import sqlite3 +from types import SimpleNamespace + +import pytest + +from engraphis.service import MemoryService +from scripts import entry, importer, seed_from_obsidian, smoke_entry_points + + +ROOT = Path(__file__).resolve().parents[1] + + +def _vault(tmp_path: Path, count: int = 2) -> Path: + vault = tmp_path / "Vault" + vault.mkdir() + for index in range(count): + (vault / f"Note-{index}.md").write_text( + f"# Note {index}\nLocal content.\n", encoding="utf-8", + ) + return vault + + +def test_dry_run_rejects_invalid_scope_without_creating_database(tmp_path, capsys): + vault = _vault(tmp_path) + database = tmp_path / "missing.db" + + result = importer.main([ + "obsidian", str(vault), "--db", str(database), "--workspace", "acme", + "--scope", "session", "--dry-run", + ]) + + assert result == 2 + assert "session scope requires --session" in capsys.readouterr().err + assert not database.exists() + + +def test_dry_run_new_workspace_cannot_reuse_another_workspaces_vault( + tmp_path, capsys, +): + vault = _vault(tmp_path, count=1) + database = tmp_path / "memory.db" + service = MemoryService.create( + str(database), embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + try: + imported = service.import_obsidian_vault( + str(vault), workspace="alpha", vault_label=vault.name, + confirmed=True, + ) + foreign_vault_id = imported["vault_id"] + finally: + service.close() + + result = importer.main([ + "obsidian", str(vault), "--db", str(database), "--workspace", "beta", + "--dry-run", "--json", + ]) + + assert result == 0 + preview = json.loads(capsys.readouterr().out) + assert preview["vault_id"] is None + assert preview["source_id"] is None + assert preview["counts"]["imported"] == 1 + assert preview["counts"].get("skipped", 0) == 0 + assert foreign_vault_id not in str(preview) + with sqlite3.connect(database) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM workspaces WHERE name='beta'" + ).fetchone()[0] == 0 + + +def test_limit_uses_service_cancellation_boundary_and_reports_partial(monkeypatch, tmp_path, capsys): + vault = _vault(tmp_path, count=3) + scan = SimpleNamespace(notes=[object(), object(), object()]) + calls = {} + + class Service: + def import_obsidian_vault(self, _path, **kwargs): + calls.update(kwargs) + assert kwargs["cancel_check"]() is False + kwargs["progress"]({"status": "imported", "relative_path": "Note-0.md"}) + assert kwargs["cancel_check"]() is True + return {"state": "cancelled", "counts": {"imported": 1}} + + def close(self): + calls["closed"] = True + + monkeypatch.setattr(importer, "scan_obsidian_vault", lambda _path: scan) + monkeypatch.setattr(importer, "_preview", lambda *_args: { + "state": "preview", "counts": {"markdown": 3}, "summary": {}, "files": [], + }) + monkeypatch.setattr(importer, "_local_service", lambda _path: Service()) + + result = importer.main([ + "obsidian", str(vault), "--workspace", "acme", "--yes", "--json", + "--limit", "1", + ]) + + assert result == 3 + payload = json.loads(capsys.readouterr().out) + assert payload["limit"] == {"processed": 1, "reached": True, "requested": 1} + assert calls["confirmed"] is True + assert calls["closed"] is True + + +def test_confirmed_cli_import_writes_through_v2_service(monkeypatch, tmp_path, capsys): + vault = _vault(tmp_path, count=1) + database = tmp_path / "memory.db" + monkeypatch.setattr( + importer, + "_local_service", + lambda path: MemoryService.create( + path, embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ), + ) + + result = importer.main([ + "obsidian", str(vault), "--db", str(database), "--workspace", "acme", + "--yes", "--json", + ]) + + assert result == 0 + assert json.loads(capsys.readouterr().out)["report"]["state"] == "completed" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 1 + + +def test_confirmed_cli_import_uses_the_exact_previewed_scan(monkeypatch, tmp_path, capsys): + vault = _vault(tmp_path, count=1) + note = vault / "Note-0.md" + note.write_text("# Approved\nPREVIEWED_BYTES\n", encoding="utf-8") + database = tmp_path / "memory.db" + monkeypatch.setattr( + importer, + "_local_service", + lambda path: MemoryService.create( + path, embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ), + ) + + def mutate_after_preview(_args): + note.write_text("# Changed\nUNPREVIEWED_BYTES\n", encoding="utf-8") + return True + + monkeypatch.setattr(importer, "_confirm", mutate_after_preview) + result = importer.main([ + "obsidian", str(vault), "--db", str(database), "--workspace", "acme", + "--yes", "--json", + ]) + + assert result == 0 + assert json.loads(capsys.readouterr().out)["report"]["state"] == "completed" + with sqlite3.connect(database) as connection: + content = connection.execute("SELECT content FROM memories").fetchone()[0] + assert "PREVIEWED_BYTES" in content + assert "UNPREVIEWED_BYTES" not in content + + +def test_legacy_wrapper_maps_namespace_limit_and_confirmation(monkeypatch, capsys): + forwarded = [] + monkeypatch.setattr(importer, "main", lambda argv: forwarded.extend(argv) or 3) + + result = seed_from_obsidian.main([ + "C:/vault", "--namespace", "legacy", "--limit", "4", "--json", + ]) + + assert result == 3 + assert forwarded[:4] == ["obsidian", "C:/vault", "--workspace", "legacy"] + assert ["--limit", "4"] == forwarded[forwarded.index("--limit"):][:2] + assert "--yes" in forwarded + assert "deprecated" in capsys.readouterr().err.casefold() + + +def test_front_door_dispatches_import_without_a_second_implementation(monkeypatch): + captured = [] + module = SimpleNamespace(main=lambda: captured.append(list(entry.sys.argv)) or 0) + monkeypatch.setattr(entry, "import_module", lambda name: module) + + assert entry.main(["import", "obsidian", "C:/vault", "--dry-run"]) == 0 + assert captured == [["engraphis import", "obsidian", "C:/vault", "--dry-run"]] + + +def test_import_console_alias_matches_distribution_and_artifact_manifest(): + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + assert 'engraphis-import = "scripts.importer:main"' in pyproject + assert smoke_entry_points.EXPECTED_ENTRY_POINTS["engraphis-import"] == ( + "scripts.importer:main" + ) + + +def test_reports_survive_windows_charmap_and_escape_terminal_controls(monkeypatch): + sink = io.BytesIO() + stream = io.TextIOWrapper(sink, encoding="cp1252") + monkeypatch.setattr(importer.sys, "stdout", stream) + + importer._json({"path": "Notes/進捗.md"}) + importer._console("Notes/line\nbreak-進捗.md") + stream.flush() + + output = sink.getvalue() + assert b"Notes/\\u9032\\u6357.md" in output + assert b"Notes/line\\x0abreak-\\u9032\\u6357.md" in output + + +@pytest.mark.parametrize("value", ["-1", "not-an-integer"]) +def test_limit_rejects_invalid_values_without_scanning(monkeypatch, value): + monkeypatch.setattr( + importer, "scan_obsidian_vault", + lambda _path: pytest.fail("invalid input must fail before scanning"), + ) + with pytest.raises(SystemExit) as exc: + importer.main(["obsidian", "C:/vault", "--limit", value]) + assert exc.value.code == 2 diff --git a/tests/test_obsidian_import_schema.py b/tests/test_obsidian_import_schema.py new file mode 100644 index 00000000..5ec1ec29 --- /dev/null +++ b/tests/test_obsidian_import_schema.py @@ -0,0 +1,694 @@ +import os +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from engraphis.core import ids +from engraphis.core.schema import SCHEMA_SQL, SCHEMA_VERSION +from engraphis.core.store import Store + + +def _prepare_v13_database(path: Path) -> str: + store = Store(str(path)) + workspace_id = store.get_or_create_workspace("preserved-v13") + store.close() + conn = sqlite3.connect(path) + conn.execute("DROP TABLE source_import_items") + conn.execute("DROP TABLE source_imports") + conn.execute("DROP TABLE source_vaults") + conn.execute("DELETE FROM schema_migrations") + conn.execute("INSERT INTO schema_migrations(version, applied_at) VALUES (13, 0)") + conn.commit() + conn.close() + return workspace_id + + +def test_import_source_ids_have_typed_prefixes(): + assert ids.new_id("vault").startswith("vlt_") + assert ids.new_id("source").startswith("src_") + + +def test_source_manifest_is_scoped_idempotent_and_marks_missing(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("obsidian") + vault_id = store.register_source_vault( + kind="obsidian", root_digest="a" * 64, workspace_id=workspace_id, + display_name="Personal", + ) + assert vault_id == store.register_source_vault( + kind="obsidian", root_digest="a" * 64, workspace_id=workspace_id, + display_name="Renamed", + ) + item_id = store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="Notes/One.md", + content_sha256="c" * 64, + importer_version="1", seen_at=10, + ) + assert item_id.startswith("src_") + assert store.mark_source_import_items_missing(vault_id=vault_id, seen_before=11) == 1 + item = store.get_source_import_item(vault_id=vault_id, source_key="b" * 64) + assert item["state"] == "missing" + store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="Archive/One.md", + state="imported", seen_at=12, + ) + assert store.get_source_import_item(vault_id=vault_id, source_key="b" * 64)["missing_at"] is None + finally: + store.close() + + +def test_source_vault_validates_scope_and_memory_type(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("scopes") + with pytest.raises(ValueError, match="workspace source vault"): + store.register_source_vault( + kind="obsidian", root_digest="a" * 64, workspace_id=workspace_id, + repo_id="repo_forbidden", scope="workspace", + ) + with pytest.raises(ValueError, match="repo source vault"): + store.register_source_vault( + kind="obsidian", root_digest="b" * 64, workspace_id=workspace_id, + scope="repo", + ) + with pytest.raises(ValueError, match="memory_type"): + store.register_source_vault( + kind="obsidian", root_digest="c" * 64, workspace_id=workspace_id, + memory_type="unknown", + ) + other_workspace = store.get_or_create_workspace("other-scopes") + foreign_repo = store.get_or_create_repo(other_workspace, "foreign") + with pytest.raises(ValueError, match="does not belong"): + store.register_source_vault( + kind="obsidian", root_digest="d" * 64, + workspace_id=workspace_id, repo_id=foreign_repo, scope="repo", + ) + finally: + store.close() + + +def test_source_import_item_allows_explicit_conflict_state(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("conflict") + vault_id = store.register_source_vault( + kind="obsidian", root_digest="d" * 64, workspace_id=workspace_id, + ) + store.upsert_source_import_item( + vault_id=vault_id, source_key="e" * 64, relative_path="One.md", + state="conflict", + ) + assert store.get_source_import_item(vault_id=vault_id, source_key="e" * 64)["state"] == "conflict" + finally: + store.close() + + +def test_source_lineage_and_per_job_results_are_separate_content_free_tables(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("jobs") + vault_id = store.register_source_vault( + kind="obsidian", root_digest="f" * 64, workspace_id=workspace_id, + ) + source_id = store.upsert_source_import_item( + vault_id=vault_id, source_key="1" * 64, relative_path="One.md", + content_sha256="2" * 64, canonical_sha256="3" * 64, + file_size=12, importer_version="1", + ) + job_id = ids.new_id("job") + store.conn.execute( + "INSERT INTO jobs(id, workspace_id, kind, state, created_at) " + "VALUES (?,?,'obsidian_import','running',0)", + (job_id, workspace_id), + ) + store.conn.commit() + item_id = store.record_source_import_job_item( + job_id=job_id, source_id=source_id, relative_path="One.md", + source_format="markdown", planned_action="imported", + result_state="imported", warning_count=2, + ) + assert item_id.startswith("src_") + lineage = store.get_source_import(source_id) + assert lineage["relative_path"] == "One.md" + assert lineage["content_sha256"] == "2" * 64 + result = store.list_source_import_job_items(job_id=job_id)[0] + assert result["result_state"] == "imported" + assert result["source_format"] == "markdown" + assert result["warning_count"] == 2 + assert "content" not in result and "content_sha256" not in result + with pytest.raises(ValueError, match="format"): + store.record_source_import_job_item( + job_id=job_id, relative_path="Bad.md", source_format="text/private path", + planned_action="rejected", result_state="rejected", + ) + finally: + store.close() + + +def test_source_vault_null_identity_and_cross_scope_lineage_are_durable(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("vault-owner") + other_workspace = store.get_or_create_workspace("vault-other") + vault_id = store.register_source_vault( + kind="obsidian", root_digest="a" * 64, + workspace_id=workspace_id, + ) + with pytest.raises(sqlite3.IntegrityError): + store.conn.execute( + "INSERT INTO source_vaults(id,kind,root_digest,workspace_id,scope," + "memory_type,created_at,updated_at) " + "VALUES (?,?,?,?,'workspace','semantic',0,0)", + (ids.new_id("vault"), "obsidian", "a" * 64, workspace_id), + ) + source_id = store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="One.md", + ) + foreign_job = ids.new_id("job") + store.conn.execute( + "INSERT INTO jobs(id,workspace_id,kind,state,created_at) " + "VALUES (?,?,'obsidian_import','running',0)", + (foreign_job, other_workspace), + ) + store.conn.commit() + with pytest.raises(sqlite3.IntegrityError, match="scope mismatch"): + store.record_source_import_job_item( + job_id=foreign_job, source_id=source_id, + relative_path="One.md", planned_action="imported", + ) + with pytest.raises(sqlite3.IntegrityError, match="scope mismatch"): + store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, + relative_path="One.md", import_id=foreign_job, + ) + finally: + store.close() + + +def test_source_manifest_rejects_cross_adapter_jobs(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("adapter-binding") + document_vault = store.register_source_vault( + kind="documents", root_digest="7" * 64, workspace_id=workspace_id, + ) + source_id = store.upsert_source_import_item( + vault_id=document_vault, source_key="8" * 64, + relative_path="notes/One.txt", + ) + document_job = ids.new_id("job") + obsidian_job = ids.new_id("job") + store.conn.executemany( + "INSERT INTO jobs(id,workspace_id,kind,state,created_at) VALUES (?,?,?,'running',0)", + ( + (document_job, workspace_id, "document_import"), + (obsidian_job, workspace_id, "obsidian_import"), + ), + ) + store.conn.commit() + store.upsert_source_import_item( + vault_id=document_vault, source_key="8" * 64, + relative_path="notes/One.txt", import_id=document_job, + ) + with pytest.raises(sqlite3.IntegrityError, match="seen-job scope mismatch"): + store.upsert_source_import_item( + vault_id=document_vault, source_key="8" * 64, + relative_path="notes/One.txt", import_id=obsidian_job, + ) + with pytest.raises(sqlite3.IntegrityError, match="job scope mismatch"): + store.record_source_import_job_item( + job_id=obsidian_job, source_id=source_id, + relative_path="notes/One.txt", planned_action="imported", + ) + finally: + store.close() + + +def test_source_manifest_rejects_cross_session_jobs_and_keeps_generic_jobs_compatible(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("session-job-binding") + repo_id = store.get_or_create_repo(workspace_id, "repo") + session_a = store.start_session(workspace_id, repo_id) + session_b = store.start_session(workspace_id, repo_id) + vault_id = store.register_source_vault( + kind="documents", root_digest="a" * 64, workspace_id=workspace_id, + repo_id=repo_id, session_id=session_a, scope="session", + ) + job_a, job_b, generic = ids.new_id("job"), ids.new_id("job"), ids.new_id("job") + store.conn.executemany( + "INSERT INTO jobs(id,workspace_id,repo_id,session_id,kind,state,created_at) " + "VALUES (?,?,?,?,?,'running',0)", + ( + (job_a, workspace_id, repo_id, session_a, "document_import"), + (job_b, workspace_id, repo_id, session_b, "document_import"), + (generic, workspace_id, repo_id, None, "graph_index"), + ), + ) + store.conn.commit() + source_id = store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="One.txt", + import_id=job_a, + ) + with pytest.raises(sqlite3.IntegrityError, match="seen-job scope mismatch"): + store.upsert_source_import_item( + vault_id=vault_id, source_key="b" * 64, relative_path="One.txt", + import_id=job_b, + ) + with pytest.raises(sqlite3.IntegrityError, match="job scope mismatch"): + store.record_source_import_job_item( + job_id=job_b, source_id=source_id, relative_path="One.txt", + planned_action="imported", + ) + with pytest.raises(sqlite3.IntegrityError, match="job session scope mismatch"): + store.conn.execute( + "INSERT INTO jobs(id,workspace_id,repo_id,session_id,kind,state,created_at) " + "VALUES (?,?,?,?,?,'running',0)", + (ids.new_id("job"), workspace_id, None, session_b, "document_import"), + ) + finally: + store.close() + + +def test_concurrent_null_scoped_vault_registration_has_one_winner(tmp_path): + db = tmp_path / "vault-race.db" + initial = Store(str(db)) + workspace_id = initial.get_or_create_workspace("race") + initial.close() + + def register(_index): + candidate = Store(str(db)) + try: + return candidate.register_source_vault( + kind="obsidian", root_digest="e" * 64, + workspace_id=workspace_id, + ) + finally: + candidate.close() + + with ThreadPoolExecutor(max_workers=2) as pool: + vault_ids = list(pool.map(register, range(2))) + assert len(set(vault_ids)) == 1 + verifier = Store(str(db)) + try: + assert len(verifier.list_source_vaults(workspace_id=workspace_id)) == 1 + finally: + verifier.close() + + +def test_source_manifest_store_methods_enforce_workspace_allowlist(tmp_path): + db = tmp_path / "tenants.db" + owner = Store(str(db)) + allowed_id = owner.get_or_create_workspace("allowed") + denied_id = owner.get_or_create_workspace("denied") + allowed_vault = owner.register_source_vault( + kind="obsidian", root_digest="c" * 64, workspace_id=allowed_id, + ) + denied_vault = owner.register_source_vault( + kind="obsidian", root_digest="d" * 64, workspace_id=denied_id, + ) + owner.close() + + scoped = Store(str(db), allowed_workspaces={"allowed"}) + try: + assert scoped.get_source_vault(allowed_vault)["workspace_id"] == allowed_id + with pytest.raises(ValueError, match="not permitted"): + scoped.get_source_vault(denied_vault) + assert [row["id"] for row in scoped.list_source_vaults()] == [allowed_vault] + finally: + scoped.close() + + +def test_manifest_snapshot_handles_absent_and_v13_database_without_writing(tmp_path): + absent = tmp_path / "absent.db" + assert Store.snapshot_source_import_manifest(str(absent)) == { + "schema_version": 0, "vaults": [], "items": [], + } + legacy = tmp_path / "legacy.db" + conn = sqlite3.connect(legacy) + conn.execute("CREATE TABLE schema_migrations(version INTEGER, applied_at REAL)") + conn.execute("INSERT INTO schema_migrations VALUES (13, 0)") + conn.commit() + conn.close() + assert Store.snapshot_source_import_manifest(str(legacy)) == { + "schema_version": 13, "vaults": [], "items": [], + } + + +def test_manifest_snapshot_uses_injected_immutable_read_connector(tmp_path): + db = tmp_path / "manifest.db" + store = Store(str(db)) + store.close() + seen = [] + + class Connector: + def __call__(self, _path): + raise AssertionError("writable connector must not be used by a snapshot") + + def open_read_only(self, path): + snapshot_path = Path(path) + seen.append({ + "path": snapshot_path, + "exists": snapshot_path.is_file(), + "mode": snapshot_path.stat().st_mode, + }) + uri = Path(path).as_uri() + "?mode=ro&immutable=1" + connection = sqlite3.connect(uri, uri=True) + connection.row_factory = sqlite3.Row + return connection + + snapshot = Store.snapshot_source_import_manifest(str(db), connect=Connector()) + + assert snapshot["schema_version"] == SCHEMA_VERSION + assert len(seen) == 1 + assert seen[0]["exists"] is True + assert seen[0]["path"] != db.resolve() + assert seen[0]["path"].name == "manifest.db" + if os.name != "nt": + assert seen[0]["mode"] & 0o077 == 0 + assert not seen[0]["path"].exists() + assert not seen[0]["path"].parent.exists() + assert not Path(str(db) + "-wal").exists() + assert not Path(str(db) + "-shm").exists() + assert not (tmp_path / "legacy.db-wal").exists() + + +def test_manifest_snapshot_includes_repo_session_lineage_for_first_import(tmp_path): + db = tmp_path / "lineage.db" + owner = Store(str(db)) + workspace_id = owner.get_or_create_workspace("acme") + repo_id = owner.get_or_create_repo(workspace_id, "product") + session_id = owner.start_session(workspace_id, repo_id) + owner.close() + + snapshot = Store.snapshot_source_import_manifest(str(db)) + + assert snapshot["repos"] == [{ + "id": repo_id, + "workspace_id": workspace_id, + "name": "product", + "workspace_name": "acme", + }] + assert snapshot["sessions"] == [{ + "id": session_id, + "workspace_id": workspace_id, + "repo_id": repo_id, + "workspace_name": "acme", + "repo_name": "product", + }] + + +def test_manifest_snapshot_rejects_path_replacement_between_lstat_and_open( + monkeypatch, tmp_path, +): + db = tmp_path / "manifest.db" + owner = Store(str(db)) + owner.close() + replacement = tmp_path / "replacement.db" + foreign = Store(str(replacement)) + foreign.close() + backup = tmp_path / "manifest-owner.db" + real_open = os.open + swapped = False + + def swapping_open(path, flags, mode=0o777): + nonlocal swapped + if not swapped and Path(path) == db.resolve(): + swapped = True + os.replace(db, backup) + os.replace(replacement, db) + if flags & os.O_CREAT: + return real_open(path, flags, mode) + return real_open(path, flags) + + monkeypatch.setattr(os, "open", swapping_open) + try: + with pytest.raises(RuntimeError, match="changed while it was opened"): + Store.snapshot_source_import_manifest(str(db)) + finally: + if swapped: + os.replace(db, replacement) + os.replace(backup, db) + + +def test_manifest_snapshot_cleans_private_copy_when_connector_fails(tmp_path): + db = tmp_path / "manifest.db" + store = Store(str(db)) + store.close() + seen = [] + + class FailingConnector: + def __call__(self, _path): + raise AssertionError("writable connector must not be used by a snapshot") + + def open_read_only(self, path): + seen.append(Path(path)) + assert seen[-1].is_file() + raise RuntimeError("synthetic connector failure") + + with pytest.raises(RuntimeError, match="synthetic connector failure"): + Store.snapshot_source_import_manifest( + str(db), connect=FailingConnector(), + ) + + assert len(seen) == 1 + assert not seen[0].exists() + assert not seen[0].parent.exists() + + +def test_manifest_snapshot_rejects_bare_connector_but_keeps_empty_semantics(tmp_path): + db = tmp_path / "manifest.db" + store = Store(str(db)) + store.close() + calls = [] + + def writable_only(path): + calls.append(path) + return sqlite3.connect(path) + + with pytest.raises(TypeError, match="open_read_only"): + Store.snapshot_source_import_manifest( + str(db), connect=writable_only, # type: ignore[arg-type] + ) + assert calls == [] + + empty = {"schema_version": 0, "vaults": [], "items": []} + assert Store.snapshot_source_import_manifest( + str(tmp_path / "missing.db"), + connect=writable_only, # type: ignore[arg-type] + ) == empty + assert Store.snapshot_source_import_manifest( + ":memory:", connect=writable_only, # type: ignore[arg-type] + ) == empty + assert calls == [] + + +def test_manifest_snapshot_refuses_active_wal(tmp_path): + db = tmp_path / "active.db" + store = Store(str(db)) + try: + assert store.schema_version == SCHEMA_VERSION + store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + store.conn.execute("BEGIN IMMEDIATE") + store.conn.execute("INSERT INTO workspaces(id, name, created_at, settings) VALUES (?,?,?,?)", + ("ws_test", "snapshot", 0, "{}")) + store.conn.commit() + with pytest.raises(RuntimeError, match="active WAL"): + Store.snapshot_source_import_manifest(str(db)) + finally: + store.close() + + +def test_manifest_snapshot_refuses_active_rollback_journal(tmp_path): + db = tmp_path / "active-journal.db" + store = Store(str(db)) + store.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + store.close() + journal = Path(f"{db}-journal") + journal.write_bytes(b"simulated hot rollback journal") + before = (journal.stat().st_size, journal.stat().st_mtime_ns) + + with pytest.raises(RuntimeError, match="active rollback journal"): + Store.snapshot_source_import_manifest(str(db)) + + assert (journal.stat().st_size, journal.stat().st_mtime_ns) == before + + +def test_v13_writable_upgrade_creates_durable_current_manifest_schema(tmp_path): + db = tmp_path / "v13.db" + workspace_id = _prepare_v13_database(db) + upgraded = Store(str(db)) + try: + assert upgraded.schema_version == SCHEMA_VERSION == 16 + assert upgraded.conn.execute( + "SELECT id FROM workspaces WHERE id=?", (workspace_id,) + ).fetchone() is not None + objects = { + row["name"] for row in upgraded.conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table','index','trigger')" + ).fetchall() + } + assert { + "source_vaults", "source_imports", "source_import_items", + "idx_source_vaults_identity", "trg_source_vault_scope_insert", + "trg_source_import_scope_insert", "trg_source_import_job_insert", + }.issubset(objects) + assert Path(f"{db}.pre-migration-v14.bak").is_file() + finally: + upgraded.close() + + +def test_v13_to_current_failure_rolls_back_schema_and_version(monkeypatch, tmp_path): + db = tmp_path / "rollback-v13.db" + _prepare_v13_database(db) + original = Store._apply_schema + + def fail_after_schema(self, previous_version): + original(self, previous_version) + raise RuntimeError("injected current migration failure") + + monkeypatch.setattr(Store, "_apply_schema", fail_after_schema) + with pytest.raises(RuntimeError, match="injected current"): + Store(str(db)) + conn = sqlite3.connect(db) + try: + assert conn.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 13 + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='source_vaults'" + ).fetchone() is None + finally: + conn.close() + + +def test_v13_read_only_refuses_without_writing_then_accepts_upgraded_db(tmp_path): + db = tmp_path / "readonly-v13.db" + _prepare_v13_database(db) + with pytest.raises(RuntimeError, match="complete current schema"): + Store(str(db), read_only=True) + assert not Path(f"{db}.pre-migration-v14.bak").exists() + + writable = Store(str(db)) + writable.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + writable.close() + readonly = Store(str(db), read_only=True) + try: + assert readonly.schema_version == 16 + with pytest.raises(sqlite3.OperationalError): + readonly.conn.execute( + "INSERT INTO workspaces(id,name) VALUES ('ws_nope','nope')" + ) + finally: + readonly.close() + + +def test_v14_manifest_upgrade_preserves_lineage_and_accepts_documents(tmp_path): + db = tmp_path / "v14-manifest.db" + conn = sqlite3.connect(db) + legacy_sql = SCHEMA_SQL.replace( + "CHECK(kind IN ('documents','obsidian'))", "CHECK(kind IN ('obsidian'))", + ) + conn.executescript(legacy_sql) + # A v14 database predates the exact-session source-job triggers that are + # installed during this upgrade; leave the legacy rows insertable so the + # migration itself exercises the backfill path. + for trigger in ( + "trg_source_import_seen_job_insert", + "trg_source_import_seen_job_update", + "trg_source_import_job_insert", + "trg_source_import_job_update", + ): + conn.execute(f"DROP TRIGGER {trigger}") + workspace_id = ids.new_id("workspace") + session_id = ids.new_id("session") + job_id = ids.new_id("job") + vault_id = ids.new_id("vault") + source_id = ids.new_id("source") + conn.execute( + "INSERT INTO schema_migrations(version,applied_at) VALUES (14,0)" + ) + conn.execute( + "INSERT INTO workspaces(id,name,created_at,settings) VALUES (?,?,0,'{}')", + (workspace_id, "v14-owner"), + ) + conn.execute( + "INSERT INTO sessions(id,workspace_id,repo_id,agent,status,started_at) " + "VALUES (?,?,NULL,'legacy','active',0)", + (session_id, workspace_id), + ) + conn.execute( + "INSERT INTO jobs(id,workspace_id,kind,state,created_at) " + "VALUES (?,?,'obsidian_import','completed',0)", + (job_id, workspace_id), + ) + conn.execute( + "INSERT INTO source_vaults(id,kind,root_digest,display_name,workspace_id," + "session_id,scope,memory_type,importer_version,created_at,updated_at) " + "VALUES (?,'obsidian',?,'Legacy',?,?, 'session','semantic','1',0,0)", + (vault_id, "a" * 64, workspace_id, session_id), + ) + conn.execute( + "INSERT INTO source_imports(id,vault_id,source_key,relative_path," + "importer_version,state,last_seen_job_id) " + "VALUES (?,?,?,?, '1','imported',?)", + (source_id, vault_id, "b" * 64, "Legacy.md", job_id), + ) + conn.execute( + "INSERT INTO source_import_items(id,job_id,source_id,relative_path," + "planned_action,result_state,created_at) VALUES (?,?,?,?, 'imported','imported',0)", + (ids.new_id("source"), job_id, source_id, "Legacy.md"), + ) + conn.commit() + conn.close() + + upgraded = Store(str(db)) + try: + assert upgraded.schema_version == 16 + assert upgraded.conn.execute( + "SELECT session_id FROM jobs WHERE id=?", (job_id,) + ).fetchone()["session_id"] == session_id + assert upgraded.get_source_vault(vault_id)["kind"] == "obsidian" + assert upgraded.get_source_import(source_id)["relative_path"] == "Legacy.md" + assert upgraded.list_source_import_job_items(job_id=job_id)[0]["source_id"] == source_id + assert upgraded.list_source_import_job_items(job_id=job_id)[0]["source_format"] == "" + documents_id = upgraded.register_source_vault( + kind="documents", root_digest="c" * 64, workspace_id=workspace_id, + display_name="Mixed documents", + ) + assert upgraded.get_source_vault(documents_id)["kind"] == "documents" + assert upgraded.conn.execute("PRAGMA foreign_key_check").fetchall() == [] + assert upgraded.conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + assert Path(f"{db}.pre-migration-v15.bak").is_file() + finally: + upgraded.close() + + +def test_v15_upgrade_adds_nullable_job_session_scope(tmp_path): + db = tmp_path / "v15-jobs.db" + conn = sqlite3.connect(db) + conn.executescript( + "CREATE TABLE schema_migrations(version INTEGER PRIMARY KEY, applied_at REAL);" + "INSERT INTO schema_migrations(version, applied_at) VALUES (15, 0);" + "CREATE TABLE jobs(" + "id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, repo_id TEXT, kind TEXT NOT NULL," + "state TEXT NOT NULL DEFAULT 'queued', dry_run INTEGER NOT NULL DEFAULT 1," + "total_items INTEGER NOT NULL DEFAULT 0, processed_items INTEGER NOT NULL DEFAULT 0," + "counts TEXT NOT NULL DEFAULT '{}', errors TEXT NOT NULL DEFAULT '[]'," + "request TEXT NOT NULL DEFAULT '{}', cancel_requested INTEGER NOT NULL DEFAULT 0," + "runner_id TEXT, heartbeat_at REAL, created_at REAL NOT NULL, started_at REAL, finished_at REAL" + ");" + ) + conn.commit() + conn.close() + + upgraded = Store(str(db)) + try: + assert upgraded.schema_version == 16 + assert "session_id" in { + row["name"] for row in upgraded.conn.execute("PRAGMA table_info(jobs)") + } + assert Path(f"{db}.pre-migration-v16.bak").is_file() + finally: + upgraded.close() diff --git a/tests/test_obsidian_importer.py b/tests/test_obsidian_importer.py new file mode 100644 index 00000000..d89ba8c1 --- /dev/null +++ b/tests/test_obsidian_importer.py @@ -0,0 +1,772 @@ +from __future__ import annotations + +import json +import threading +from pathlib import Path + +import pytest + +from engraphis.core.interfaces import Scope, SearchFilter +from engraphis.core.obsidian import ( + ObsidianFileIssue, ObsidianNote, ObsidianVaultScan, scan_obsidian_vault, +) +from engraphis.obsidian_import import ObsidianImportCancelled, ObsidianImporter +from engraphis.service import MemoryService +from scripts import importer as importer_cli + + +def _vault(root: Path) -> Path: + vault = root / "Knowledge" + (vault / "projects").mkdir(parents=True) + (vault / "Home.md").write_text( + "---\ntitle: Home base\naliases: [Start]\ntags: [index, private]\n" + "created: 2026-01-02\n---\n# Home base\nSee [[projects/Plan|the plan]].\n", + encoding="utf-8", + ) + (vault / "projects" / "Plan.md").write_text( + "# Plan\n\nShip locally. Link back to [[Home]].\n", + encoding="utf-8", + ) + (vault / ".obsidian").mkdir() + (vault / ".obsidian" / "app.json").write_text("{}", encoding="utf-8") + return vault + + +def _service(path: Path) -> MemoryService: + return MemoryService.create( + str(path), embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + + +def _live(service: MemoryService, workspace: str = "acme"): + wid = service._lookup_workspace(workspace) + return service.store.list_memories(SearchFilter(workspace_id=wid)) + + +def test_unreadable_directory_is_reported_and_marks_vault_scan_incomplete(monkeypatch, tmp_path: Path): + blocked = tmp_path / "blocked" + blocked.mkdir() + (blocked / "note.md").write_text("# note", encoding="utf-8") + original_iterdir = Path.iterdir + + def fail_blocked(path): + if path == blocked: + raise OSError("permission denied") + return original_iterdir(path) + + monkeypatch.setattr(Path, "iterdir", fail_blocked) + scan = scan_obsidian_vault(tmp_path) + assert scan.complete is False + assert any( + item.relative_path == "blocked" and item.reason == "unreadable directory" + for item in scan.skipped + ) + + +def test_unreadable_root_is_reported_and_marks_vault_scan_incomplete(monkeypatch, tmp_path: Path): + original_iterdir = Path.iterdir + + def fail_root(path): + if path == tmp_path: + raise OSError("permission denied") + return original_iterdir(path) + + monkeypatch.setattr(Path, "iterdir", fail_root) + scan = scan_obsidian_vault(tmp_path) + + assert scan.complete is False + assert any( + item.relative_path == "." and item.reason == "unreadable directory" + for item in scan.skipped + ) + + +def test_unreadable_vault_path_marks_scan_incomplete(monkeypatch, tmp_path: Path): + blocked = tmp_path / "blocked.md" + blocked.write_text("# blocked", encoding="utf-8") + original_lstat = Path.lstat + + def fail_blocked(path): + if path == blocked: + raise OSError("permission denied") + return original_lstat(path) + + monkeypatch.setattr(Path, "lstat", fail_blocked) + scan = scan_obsidian_vault(tmp_path) + + assert scan.complete is False + assert any( + item.relative_path == "blocked.md" and item.reason == "unreadable path" + for item in scan.skipped + ) + + +def test_import_reimport_revision_rename_and_missing(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + preview = service.preview_obsidian_vault(str(vault), workspace="acme") + assert preview["counts"]["markdown"] == 2 + assert preview["counts"]["imported"] == 2 + assert preview["summary"]["wikilinks"] == 2 + + first = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert first["state"] == "completed" + assert first["counts"]["imported"] == 2 + memories = _live(service) + assert len(memories) == 2 + home = next(memory for memory in memories if memory.title == "Home base") + assert home.content.startswith("# Home base") + assert home.metadata["obsidian"]["relative_path"] == "Home.md" + assert home.metadata["obsidian"]["aliases"] == ["Start"] + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE relation='references' AND valid_to IS NULL" + ).fetchone()[0] == 1 + + again = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert again["counts"]["skipped"] == 3 # two unchanged notes + .obsidian exclusion + assert len(_live(service)) == 2 + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 2 + + home_path = vault / "Home.md" + home_path.write_text( + "---\ntitle: Home base\naliases: [Start]\ntags: [index]\n---\n" + "# Home base\nThe durable revision. See [[projects/Plan]].\n", + encoding="utf-8", + ) + revised = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert revised["counts"]["updated"] == 1 + history = service.store.conn.execute( + "SELECT valid_to, metadata FROM memories WHERE subject_key=? ORDER BY valid_from", + (home.subject_key,), + ).fetchall() + assert len(history) == 2 + assert history[0]["valid_to"] is not None + assert history[1]["valid_to"] is None + link_history = service.store.conn.execute( + "SELECT valid_to FROM mem_links WHERE relation='references' ORDER BY valid_from" + ).fetchall() + assert len(link_history) == 2 + assert link_history[0]["valid_to"] is not None + assert link_history[1]["valid_to"] is None + + moved = vault / "Welcome.md" + home_path.rename(moved) + renamed = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert renamed["counts"]["renamed"] == 1 + item = service.store.conn.execute( + "SELECT relative_path FROM source_imports WHERE subject_key=?", + (home.subject_key,), + ).fetchone() + assert item["relative_path"] == "Welcome.md" + + moved.unlink() + missing = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert missing["counts"]["missing"] == 1 + manifest = service.store.conn.execute( + "SELECT state FROM source_imports WHERE subject_key=?", + (home.subject_key,), + ).fetchone() + assert manifest["state"] == "missing" + assert service.store.conn.execute( + "SELECT COUNT(*) FROM memories WHERE subject_key=?", (home.subject_key,), + ).fetchone()[0] == 3 + finally: + service.close() + + +def test_metadata_only_revision_and_conflict_policies(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + source = service.store.conn.execute( + "SELECT * FROM source_imports WHERE relative_path='Home.md'" + ).fetchone() + old = service.store.get_memory(source["memory_id"]) + assert old is not None + + # A metadata-only frontmatter change is still a temporal source revision. + path = vault / "Home.md" + path.write_text(path.read_text(encoding="utf-8").replace("index, private", "index"), + encoding="utf-8") + report = service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + assert report["counts"]["updated"] == 1 + current = service.store.conn.execute( + "SELECT * FROM source_imports WHERE relative_path='Home.md' AND state='imported'" + ).fetchone() + assert current["memory_id"] != old.id + + # A local correction closes the importer's current record. The default importer + # reports a conflict and does not silently supersede that divergent lineage. + imported = service.store.get_memory(current["memory_id"]) + correction = service.engine.remember_with_resolution( + "# Home base\nOwner correction.\n", + workspace_id=imported.workspace_id, mtype=imported.mtype, + scope=imported.scope, title=imported.title, + subject_key=imported.subject_key, claim_kind=imported.claim_kind, + valid_from=(imported.valid_from or 0) + 1, + ) + assert correction["op"] == "invalidate" + path.write_text("# Home base\nSource changed after correction.\n", encoding="utf-8") + conflict = service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + assert conflict["state"] == "partial" + assert conflict["counts"]["conflict"] == 1 + assert service.store.get_memory(correction["id"]).valid_to is None + + replaced = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, on_conflict="replace", + ) + assert replaced["counts"]["updated"] == 1 + assert service.store.get_memory(correction["id"]).valid_to is not None + finally: + service.close() + + +def test_cancelled_import_resumes_without_duplicates(tmp_path: Path): + vault = _vault(tmp_path) + db = tmp_path / "memory.db" + service = _service(db) + checks = 0 + + def cancel_after_one() -> bool: + nonlocal checks + checks += 1 + return checks > 1 + + try: + cancelled = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + cancel_check=cancel_after_one, + ) + assert cancelled["state"] == "cancelled" + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 1 + resumed = service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + assert resumed["state"] == "completed" + assert resumed["counts"]["skipped"] == 2 # one unchanged note + .obsidian exclusion + assert resumed["counts"]["imported"] == 1 + assert service.store.conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] == 2 + finally: + service.close() + + +def test_ambiguous_wikilink_retires_previous_derived_edge(tmp_path: Path): + vault = _vault(tmp_path) + home = vault / "Home.md" + home.write_text( + home.read_text(encoding="utf-8").replace("projects/Plan|the plan", "Plan"), + encoding="utf-8", + ) + # Keep this case focused on the only supporting reference. The default fixture's + # backlink is covered by the independent-support regression below. + (vault / "projects" / "Plan.md").write_text("# Plan\n\nNo backlink.\n", encoding="utf-8") + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + home_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", + ("Home base",), + ).fetchone()[0] + plan_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", + ("Plan",), + ).fetchone()[0] + initial_links = service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] + assert initial_links >= 1 + + archive = vault / "archive" + archive.mkdir() + (archive / "Plan.md").write_text("# Another Plan\n", encoding="utf-8") + + second = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert any( + row["reason"] == "ambiguous_wikilink" for row in second["files"] + ) + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] == 0 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? AND valid_to IS NOT NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] == initial_links + finally: + service.close() + + +def test_ambiguous_wikilink_preserves_independent_derived_edge(tmp_path: Path): + vault = _vault(tmp_path) + home = vault / "Home.md" + home.write_text( + home.read_text(encoding="utf-8").replace("projects/Plan|the plan", "Plan"), + encoding="utf-8", + ) + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + home_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Home base",) + ).fetchone()[0] + plan_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Plan",) + ).fetchone()[0] + archive = vault / "archive" + archive.mkdir() + (archive / "Plan.md").write_text("# Another Plan\n", encoding="utf-8") + + second = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert any(row["reason"] == "ambiguous_wikilink" for row in second["files"]) + # Plan.md still points back to Home.md, so its independent support keeps the + # aggregate undirected edge live even though Home.md is now ambiguous. + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] == 1 + finally: + service.close() + + +def test_concurrent_imports_leave_one_live_source_revision(tmp_path: Path): + vault = _vault(tmp_path) + db = tmp_path / "memory.db" + initial = _service(db) + initial.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + initial.close() + + home = vault / "Home.md" + home.write_text( + home.read_text(encoding="utf-8").replace( + "See [[projects/Plan|the plan]].", "The concurrent revision." + ), + encoding="utf-8", + ) + first = _service(db) + second = _service(db) + barrier = threading.Barrier(2) + states = threading.local() + errors: list[BaseException] = [] + + def gate_first_write(original): + def gated(*args, **kwargs): + if not getattr(states, "waited", False): + states.waited = True + barrier.wait(timeout=10) + return original(*args, **kwargs) + return gated + + first.engine.remember_with_resolution = gate_first_write( # type: ignore[method-assign] + first.engine.remember_with_resolution + ) + second.engine.remember_with_resolution = gate_first_write( # type: ignore[method-assign] + second.engine.remember_with_resolution + ) + + def run(service: MemoryService) -> None: + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + except BaseException as exc: # pragma: no cover - assertion below reports it + errors.append(exc) + + workers = [threading.Thread(target=run, args=(service,)) for service in (first, second)] + try: + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=30) + assert not any(worker.is_alive() for worker in workers) + assert not errors + subject_key = first.store.conn.execute( + "SELECT subject_key FROM source_imports WHERE relative_path='Home.md' " + "ORDER BY last_seen_at DESC LIMIT 1" + ).fetchone()[0] + live = first.store.conn.execute( + "SELECT id FROM memories WHERE subject_key=? AND valid_to IS NULL " + "AND expired_at IS NULL", + (subject_key,), + ).fetchall() + assert len(live) == 1 + finally: + first.close() + second.close() + + +def test_missing_wikilink_retires_previous_derived_edge(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + home_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Home base",) + ).fetchone()[0] + plan_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Plan",) + ).fetchone()[0] + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] >= 1 + + (vault / "projects" / "Plan.md").unlink() + second = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + assert any(row["reason"] == "unresolved_wikilink" for row in second["files"]) + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] == 0 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links " + "WHERE reason=? AND a=? AND b=? AND valid_to IS NOT NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] >= 1 + finally: + service.close() + + +def test_exact_target_retires_previous_basename_fallback_link(tmp_path: Path): + vault = _vault(tmp_path) + (vault / "projects" / "Plan.md").write_text("# Plan\n\nNo backlink.\n", encoding="utf-8") + home = vault / "Home.md" + home.write_text( + home.read_text(encoding="utf-8").replace("projects/Plan|the plan", "Plan"), + encoding="utf-8", + ) + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + home_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Home base",) + ).fetchone()[0] + fallback_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Plan",) + ).fetchone()[0] + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, fallback_id), + ).fetchone()[0] == 1 + + (vault / "Plan.md").write_text("# Exact Plan\n", encoding="utf-8") + second = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + exact_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Exact Plan",) + ).fetchone()[0] + assert second["state"] == "completed" + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, fallback_id), + ).fetchone()[0] == 0 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, exact_id), + ).fetchone()[0] == 1 + finally: + service.close() + + +def test_incomplete_scan_preserves_derived_links(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + first = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + home_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Home base",) + ).fetchone()[0] + plan_id = service.store.conn.execute( + "SELECT id FROM memories WHERE title=? LIMIT 1", ("Plan",) + ).fetchone()[0] + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] >= 1 + + complete = scan_obsidian_vault(str(vault)) + partial = ObsidianVaultScan( + vault_path=complete.vault_path, vault_id=complete.vault_id, + notes=[note for note in complete.notes if note.relative_path == "Home.md"], + skipped=[ObsidianFileIssue("projects", "unreadable directory")], + complete=False, + ) + second = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, _scan=partial, + ) + assert second["state"] == "partial" + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE reason=? AND a=? AND b=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ObsidianImporter.LINK_REASON, home_id, plan_id), + ).fetchone()[0] == 1 + assert service.get_obsidian_import_job( + second["job_id"], workspace="acme", + )["counts"].get("warning", 0) == 0 + assert first["state"] == "completed" + finally: + service.close() + + +def test_link_warnings_are_visible_in_durable_job_rows(tmp_path: Path): + vault = _vault(tmp_path) + home = vault / "Home.md" + home.write_text( + home.read_text(encoding="utf-8").replace("projects/Plan|the plan", "Plan"), + encoding="utf-8", + ) + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + archive = vault / "archive" + archive.mkdir() + (archive / "Plan.md").write_text("# Another Plan\n", encoding="utf-8") + report = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, + ) + job = service.get_obsidian_import_job(report["job_id"], workspace="acme") + home_row = next(row for row in job["files"] if row["relative_path"] == "Home.md") + assert home_row["warning_count"] == 1 + assert home_row["reason"] == "ambiguous_wikilink" + finally: + service.close() + + +def test_link_reconciliation_cancels_and_rolls_back_only_the_open_batch(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + report = service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + service.store.conn.execute("DELETE FROM mem_links") + service.store.conn.commit() + checks = 0 + + def cancel_after_first_source() -> bool: + nonlocal checks + checks += 1 + return checks >= 2 + + with pytest.raises(ObsidianImportCancelled): + ObsidianImporter(service)._reconcile_links( + scan_obsidian_vault(str(vault)), vault_id=report["vault_id"], + job_id=report["job_id"], cancel_check=cancel_after_first_source, + ) + assert service.store.conn.execute("SELECT COUNT(*) FROM mem_links").fetchone()[0] == 0 + finally: + service.close() + + +def test_rename_planner_indexes_hashes_once_for_large_sources(): + class CountingItems(list): + iterations = 0 + + def __iter__(self): + self.iterations += 1 + return super().__iter__() + + notes = [ + ObsidianNote( + relative_path=f"current/{index}.md", title=str(index), content="body", body="body", + raw_sha256=f"{index:064x}", canonical_sha256=f"{index:064x}", + ) + for index in range(128) + ] + items = CountingItems([ + { + "id": f"src_{index}", "source_key": f"{index + 1000:064x}", + "relative_path": f"archived/{index}.md", "content_sha256": f"{index:064x}", + "state": "imported", "last_seen_at": index, + } + for index in range(128) + ]) + plans, missing = ObsidianImporter()._plan( + ObsidianVaultScan(vault_path="", vault_id="a" * 64, notes=notes), + "a" * 64, items, inspect_memories=False, + ) + assert not missing + assert {plan.action for plan in plans} == {"renamed"} + # The prior implementation iterated the full historical item collection once + # per unmatched note. The indexed planner has only setup/final missing passes. + assert items.iterations <= 3 + + +def test_conflict_new_branch_and_atomic_note_failure(tmp_path: Path, monkeypatch): + vault = tmp_path / "Vault" + vault.mkdir() + note = vault / "Note.md" + note.write_text("# Note\nInitial.\n", encoding="utf-8") + service = _service(tmp_path / "memory.db") + try: + service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + item = service.store.conn.execute("SELECT * FROM source_imports").fetchone() + current = service.store.get_memory(item["memory_id"]) + service.engine.remember_with_resolution( + "# Note\nOwner version.\n", workspace_id=current.workspace_id, + scope=current.scope, mtype=current.mtype, title=current.title, + subject_key=current.subject_key, claim_kind=current.claim_kind, + valid_from=(current.valid_from or 0) + 1, + ) + note.write_text("# Note\nIndependent source branch.\n", encoding="utf-8") + branched = service.import_obsidian_vault( + str(vault), workspace="acme", confirmed=True, on_conflict="new", + ) + assert branched["state"] == "completed" + assert branched["counts"]["imported"] == 1 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM source_imports WHERE relative_path='Note.md'" + ).fetchone()[0] == 2 + + # A manifest failure raised by the transactional finalizer rolls back the + # canonical memory/FTS/vector mirrors for that note as one unit. + second = vault / "Atomic.md" + second.write_text("# Atomic\nMust be all or nothing.\n", encoding="utf-8") + original = service.store.upsert_source_import_item + + def fail_atomic(**kwargs): + if kwargs.get("relative_path") == "Atomic.md" and not kwargs.get("commit", True): + raise RuntimeError("injected finalizer failure") + return original(**kwargs) + + monkeypatch.setattr(service.store, "upsert_source_import_item", fail_atomic) + failed = service.import_obsidian_vault(str(vault), workspace="acme", confirmed=True) + assert failed["state"] == "partial", failed + assert failed["counts"]["error"] == 1 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM memories WHERE title='Atomic'" + ).fetchone()[0] == 0 + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_fts WHERE title='Atomic'" + ).fetchone()[0] == 0 + finally: + service.close() + + +def test_cli_dry_run_is_strictly_write_free(tmp_path: Path, capsys): + vault = _vault(tmp_path) + db = tmp_path / "does-not-exist.db" + result = importer_cli.main([ + "obsidian", str(vault), "--db", str(db), "--workspace", "acme", + "--dry-run", "--json", + ]) + assert result == 0 + assert not db.exists() + assert not Path(str(db) + "-wal").exists() + payload = json.loads(capsys.readouterr().out) + assert payload["state"] == "preview" + assert payload["counts"]["markdown"] == 2 + + +def test_repo_and_session_scope_mapping(tmp_path: Path): + vault = _vault(tmp_path) + service = _service(tmp_path / "memory.db") + try: + repo_report = service.import_obsidian_vault( + str(vault), workspace="acme", repo="product", scope="repo", confirmed=True, + ) + assert repo_report["target"]["scope"] == Scope.REPO.value + assert all(memory.scope == Scope.REPO for memory in _live(service)) + + session_vault = tmp_path / "Session" + session_vault.mkdir() + (session_vault / "Only.md").write_text("# Only\nSession memory.\n", encoding="utf-8") + wid = service._lookup_workspace("acme") + rid = service._lookup_repo(wid, "product") + sid = service.store.start_session(wid, rid) + session_report = service.import_obsidian_vault( + str(session_vault), workspace="acme", repo="product", + session_id=sid, scope="session", confirmed=True, + ) + assert session_report["target"]["session_id"] == sid + assert service.store.conn.execute( + "SELECT session_id FROM jobs WHERE id=?", (session_report["job_id"],) + ).fetchone()["session_id"] == sid + record = service.store.get_memory(next( + row["memory_id"] for row in service.store.list_source_import_items( + vault_id=session_report["vault_id"] + ) + )) + assert record.scope == Scope.SESSION + assert record.session_id == sid + + repo_from_session_vault = tmp_path / "RepoFromSession" + repo_from_session_vault.mkdir() + (repo_from_session_vault / "Repo.md").write_text( + "# Repo\nRepo-scoped memory.\n", encoding="utf-8", + ) + repo_from_session_report = service.import_obsidian_vault( + str(repo_from_session_vault), workspace="acme", session_id=sid, + scope="repo", confirmed=True, + ) + assert repo_from_session_report["target"]["workspace"] == "acme" + assert repo_from_session_report["target"]["scope"] == Scope.REPO.value + assert repo_from_session_report["target"]["session_id"] is None + assert repo_from_session_report["target"]["repo_id"] == rid + repo_record = service.store.get_memory(next( + row["memory_id"] for row in service.store.list_source_import_items( + vault_id=repo_from_session_report["vault_id"] + ) + )) + assert repo_record.scope == Scope.REPO + assert repo_record.session_id is None + finally: + service.close() + + +def test_browser_upload_runs_as_resumable_job_without_upload_copy(tmp_path: Path): + service = _service(tmp_path / "memory.db") + try: + started = service.import_obsidian_upload( + files=[("Folder/Browser.md", b"# Browser\nSee [[Second]].\n"), + ("Second.md", b"# Second\nLocal only.\n")], + attachment_manifest=[{"path": "assets/photo.png", "size": 123}], + workspace="acme", vault_label="Browser vault", confirmed=True, + ) + job_id = started["job_id"] + worker = service._obsidian_job_threads.get(job_id) + if worker is not None: + worker.join(10) + status = service.get_obsidian_import_job(job_id, workspace="acme") + assert status["state"] == "completed" + assert status["counts"]["imported"] == 2 + assert not any(tmp_path.glob("**/*Browser.md")) + assert service.store.conn.execute( + "SELECT COUNT(*) FROM mem_links WHERE relation='references' AND valid_to IS NULL" + ).fetchone()[0] == 1 + finally: + service.close() diff --git a/tests/test_obsidian_parser.py b/tests/test_obsidian_parser.py new file mode 100644 index 00000000..53b913db --- /dev/null +++ b/tests/test_obsidian_parser.py @@ -0,0 +1,193 @@ +from pathlib import Path + +import pytest + +from engraphis.core.obsidian import ( + normalize_obsidian_path, + parse_obsidian_note, + scan_obsidian_vault, +) + + +def test_parses_frontmatter_title_tags_dates_and_headings(): + note = parse_obsidian_note( + b"\xef\xbb\xbf---\ntitle: Vault title\naliases: [One, 'Two']\ntags:\n - projects\n - #python\ncreated: 2024-01-02\n---\n# Body title\n## Details\n#inline-tag\n", + "Projects/Note.md", + ) + assert note.title == "Vault title" + assert note.title_source == "frontmatter" + assert note.aliases == ["One", "Two"] + assert note.tags == ["projects", "python", "inline-tag"] + assert note.dates == {"created": "2024-01-02"} + assert note.headings == ["Body title", "Details"] + assert note.relative_path == "Projects/Note.md" + assert len(note.raw_sha256) == len(note.canonical_sha256) == 64 + + +def test_title_falls_back_to_h1_then_stem(): + heading = parse_obsidian_note(b"# H1\n", "one.md") + fallback = parse_obsidian_note(b"No title\n", "Folder/two.md") + assert (heading.title, heading.title_source) == ("H1", "heading") + assert (fallback.title, fallback.title_source) == ("two", "filename") + + +def test_discovers_wikilinks_embeds_blocks_and_attachments_but_not_code(): + note = parse_obsidian_note( + b"[[Note|Read this]] [[Note#Heading]] [[Note^block]] ![[image.png]]\n" + b"![alt](docs/file.pdf)\n`[[inline]]`\n```md\n![[hidden.jpg]]\n```\n", + "links.md", + ) + assert [(link.target, link.display_text, link.heading, link.block_id, link.embedded) for link in note.links] == [ + ("Note", "Read this", None, None, False), ("Note", None, "Heading", None, False), + ("Note", None, None, "block", False), ("image.png", None, None, None, True), + ] + assert [attachment.path for attachment in note.attachments] == ["image.png", "docs/file.pdf"] + + +def test_unclosed_and_variable_length_fences_cannot_create_graph_metadata(): + closed = parse_obsidian_note( + b"```md\n[[Hidden]] #hidden\n````\n[[Visible]] #visible\n", + "closed.md", + ) + assert [link.target for link in closed.links] == ["Visible"] + assert closed.tags == ["visible"] + + unclosed = parse_obsidian_note( + b"# Visible heading\n[[Visible]] #visible\n````md\n" + b"[[Injected]] #injected\n## Forged heading\n", + "unclosed.md", + ) + assert [link.target for link in unclosed.links] == ["Visible"] + assert unclosed.tags == ["visible"] + assert unclosed.headings == ["Visible heading"] + + +def test_malformed_frontmatter_is_a_warning_and_is_not_fatal(): + note = parse_obsidian_note(b"---\ntitle: unfinished\n# Body\n", "bad.md") + assert note.title == "Body" + assert note.warnings == ["unclosed YAML frontmatter treated as Markdown"] + + +def test_invalid_utf8_is_repaired_with_a_warning(): + note = parse_obsidian_note(b"# Title\n\xff", "bad-utf8.md") + assert "invalid UTF-8" in note.warnings[0] + assert "\ufffd" in note.content + + +def test_frontmatter_only_note_is_not_a_memory_candidate(): + with pytest.raises(ValueError, match="note produced no readable text"): + parse_obsidian_note(b"---\ntitle: Metadata only\n---\n\n", "empty.md") + + +@pytest.mark.parametrize("body", [ + b"-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----", + b"api_key: this-is-a-long-secret-value", +]) +def test_secret_content_is_rejected_without_echoing_it(body): + with pytest.raises(ValueError, match="source appears to contain a secret"): + parse_obsidian_note(body, "note.md") + + +def test_vault_scan_skips_hidden_config_symlinks_and_rejects_secrets(tmp_path: Path): + (tmp_path / "Notes").mkdir() + (tmp_path / "Notes" / "ok.md").write_text("# Safe", encoding="utf-8") + (tmp_path / ".obsidian").mkdir() + (tmp_path / ".obsidian" / "config.md").write_text("# Ignore", encoding="utf-8") + (tmp_path / ".env.md").write_text("# Ignore", encoding="utf-8") + (tmp_path / "secret.md").write_text("password: very-secret-value", encoding="utf-8") + outside = tmp_path.parent / "outside.md" + outside.write_text("# Outside", encoding="utf-8") + try: + (tmp_path / "linked.md").symlink_to(outside) + except (NotImplementedError, OSError): + pytest.skip("symlinks unavailable on this platform") + report = scan_obsidian_vault(tmp_path) + assert [note.relative_path for note in report.notes] == ["Notes/ok.md"] + assert {issue.relative_path for issue in report.rejected} == {"secret.md"} + assert {issue.relative_path for issue in report.skipped} >= {".obsidian", ".env.md", "linked.md"} + + +def test_large_notes_are_rejected(tmp_path: Path): + (tmp_path / "large.md").write_text("x" * 100_001, encoding="utf-8") + report = scan_obsidian_vault(tmp_path) + assert report.notes == [] + assert report.rejected[0].reason == "note exceeds 100000 character safety limit" + + +def test_oversized_note_bytes_are_rejected_before_read(tmp_path: Path): + (tmp_path / "large.md").write_bytes(b"x" * 2_000_001) + report = scan_obsidian_vault(tmp_path) + assert report.notes == [] + assert report.rejected[0].reason == "note exceeds 2000000 byte safety limit" + + +def test_direct_parser_enforces_its_byte_and_type_contract_before_decode(): + with pytest.raises(ValueError, match="note data must be bytes"): + parse_obsidian_note("# not bytes", "note.md") # type: ignore[arg-type] + with pytest.raises(ValueError, match="2000000 byte safety limit"): + parse_obsidian_note(b"\xff" * 2_000_001, "note.md") + + +@pytest.mark.parametrize("path", [ + "../note.md", "/note.md", "C:/note.md", r"C:\note.md", "C:note.md", + r"\\server\share\note.md", "//server/share/note.md", + "folder/note.md:stream", " note.md", "note.md\x00", "", +]) +def test_uploaded_relative_paths_reject_traversal_and_absolute_paths(path): + with pytest.raises(ValueError, match="safe vault-relative"): + normalize_obsidian_path(path) + + +def test_relative_path_allows_dots_inside_a_filename(): + assert normalize_obsidian_path("notes/version..history.md") == "notes/version..history.md" + + +def test_relative_path_is_nfc_and_bounded(): + assert normalize_obsidian_path("notes/cafe\u0301.md") == "notes/café.md" + with pytest.raises(ValueError, match="4096"): + normalize_obsidian_path(("a" * 4094) + ".md") + + +def test_vault_scan_rejects_portable_case_collisions(tmp_path: Path): + (tmp_path / "A.md").write_text("first", encoding="utf-8") + (tmp_path / "a.md").write_text("second", encoding="utf-8") + report = scan_obsidian_vault(tmp_path) + assert len(report.notes) == 1 + if len(list(tmp_path.iterdir())) == 2: + assert any(issue.reason == "duplicate normalized source path" for issue in report.rejected) + + +def test_vault_scan_rejects_a_file_changed_during_read(monkeypatch, tmp_path: Path): + note = tmp_path / "racing.md" + note.write_text("# original", encoding="utf-8") + from engraphis.core import obsidian + + original_read = obsidian.os.read + changed = False + + def racing_read(fd, size): + nonlocal changed + chunk = original_read(fd, size) + if chunk and not changed: + changed = True + note.write_text("# replacement with a different size", encoding="utf-8") + return chunk + + monkeypatch.setattr(obsidian.os, "read", racing_read) + report = scan_obsidian_vault(tmp_path) + assert report.notes == [] + assert [(issue.relative_path, issue.reason) for issue in report.rejected] == [ + ("racing.md", "file changed during scan"), + ] + + +def test_vault_root_symlink_is_rejected(tmp_path: Path): + vault = tmp_path / "vault" + vault.mkdir() + linked = tmp_path / "linked-vault" + try: + linked.symlink_to(vault, target_is_directory=True) + except (NotImplementedError, OSError): + pytest.skip("directory symlinks unavailable on this platform") + with pytest.raises(ValueError, match="root cannot be a symlink"): + scan_obsidian_vault(linked) diff --git a/tests/test_obsidian_service.py b/tests/test_obsidian_service.py new file mode 100644 index 00000000..bfcdc810 --- /dev/null +++ b/tests/test_obsidian_service.py @@ -0,0 +1,339 @@ +"""Real-service coverage for the owner-only Obsidian import facade.""" +from __future__ import annotations + +import time + +import pytest + +from engraphis.service import MemoryService, ValidationError +from engraphis.obsidian_import import scan_obsidian_upload + + +_TERMINAL_STATES = {"completed", "partial", "failed", "cancelled"} +_SECRET = "sk-proj-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + +def _service() -> MemoryService: + return MemoryService.create( + ":memory:", embed_dim=64, extractor="none", graph_extractor="none", + retention_supervisor="none", + ) + + +def _await_job(service: MemoryService, started: dict) -> dict: + job_id = started["job_id"] + worker = service._obsidian_job_threads.get(job_id) + if worker is not None: + worker.join(10) + deadline = time.monotonic() + 10 + while True: + result = service.get_obsidian_import_job(job_id, workspace=started["workspace"]) + if result["state"] in _TERMINAL_STATES: + return result + if time.monotonic() >= deadline: + raise AssertionError(f"Obsidian job {job_id} did not finish: {result}") + time.sleep(0.01) + + +def _import(service: MemoryService, files: list[tuple[str, bytes]], **kwargs) -> tuple[dict, dict]: + started = service.import_obsidian_upload( + files=files, attachment_manifest=[], workspace="alpha", + vault_label="Team notes", confirmed=True, **kwargs, + ) + return started, _await_job(service, started) + + +def test_preview_is_write_free_and_service_enforces_confirmation_and_upload_guards(monkeypatch): + service = _service() + try: + with pytest.raises(ValueError, match="vault_label is required"): + scan_obsidian_upload([("One.md", b"# One\n")], vault_label=" ") + collision = scan_obsidian_upload( + [ + ("Notes/Caf\u00e9.md", b"# First\n"), + ("notes/cafe\u0301.md", b"# Second\n"), + ], + vault_label="Portable vault", + ) + assert [note.relative_path for note in collision.notes] == ["Notes/Caf\u00e9.md"] + assert [(item.relative_path, item.reason) for item in collision.rejected] == [ + ("notes/caf\u00e9.md", "duplicate upload path"), + ] + + preview = service.preview_obsidian_upload( + files=[ + ("notes/Welcome.md", b"# Welcome\nPRIVATE_BODY_MARKER\n"), + ("../escape.md", b"# Escape\n"), + ], + attachment_manifest=[{"path": "assets/photo.png", "size": 12}], + workspace="alpha", vault_label="Team notes", + ) + + assert preview["state"] == "preview" + assert preview["counts"]["imported"] == 1 + assert preview["counts"]["rejected"] == 1 + assert "PRIVATE_BODY_MARKER" not in str(preview) + for table in ( + "workspaces", "source_vaults", "source_imports", "jobs", + "memories", "operation_receipts", "audit", + ): + count = service.store.conn.execute( + f"SELECT COUNT(*) FROM {table}" + ).fetchone() + assert count is not None and count[0] == 0 + + with pytest.raises(ValidationError, match="vault_label is required"): + service.import_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="must-not-exist", confirmed=True, + ) + assert service._lookup_workspace("must-not-exist") is None + + with pytest.raises(ValidationError, match="confirmation"): + service.import_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="alpha", vault_label="Team notes", confirmed="true", # type: ignore[arg-type] + ) + with pytest.raises(ValidationError, match="invalid path"): + service.preview_obsidian_upload( + files=[("One.md", b"# One\n")], + attachment_manifest=[{"path": "../outside.png", "size": 1}], + workspace="alpha", vault_label="Team notes", + ) + with pytest.raises(ValidationError, match="OpenAI API key"): + service.preview_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="alpha", vault_label=_SECRET, + ) + monkeypatch.setattr("engraphis.core.obsidian.MAX_NOTE_BYTES", 10) + monkeypatch.setattr("engraphis.core.obsidian.MAX_VAULT_BYTES", 15) + with pytest.raises(ValidationError, match="duplicate path"): + service._obsidian_upload_inputs( + [("Notes/One.md", b"a"), ("notes/one.md", b"b")], [], + ) + with pytest.raises(ValidationError, match="paths overlap"): + service._obsidian_upload_inputs( + [("notes/one.md", b"a")], + [{"path": "notes/one.md", "size": 1}], + ) + with pytest.raises(ValidationError, match="invalid size"): + service._obsidian_upload_inputs( + [("notes/one.md", b"a")], + [{"path": "assets/large.png", "size": 100_000_001}], + ) + with pytest.raises(ValidationError, match="too large"): + service._obsidian_upload_inputs([("notes/big.md", b"x" * 11)], []) + with pytest.raises(ValidationError, match="total size"): + service._obsidian_upload_inputs( + [("notes/one.md", b"x" * 8), ("notes/two.md", b"y" * 8)], [], + ) + finally: + service.close() + + +def test_obsidian_browser_label_identity_is_nfc_and_requires_registered_vault(): + service = _service() + try: + first = service.import_obsidian_upload( + files=[("One.md", b"# One\nLocal note.\n")], attachment_manifest=[], + workspace="alpha", vault_label="Caf\u00e9", confirmed=True, + ) + assert _await_job(service, first)["state"] == "completed" + assert scan_obsidian_upload( + [("One.md", b"# One\n")], vault_label="Caf\u00e9", + ).vault_id == scan_obsidian_upload( + [("One.md", b"# One\n")], vault_label="Cafe\u0301", + ).vault_id + with pytest.raises(ValidationError, match="select its source_id"): + service.preview_obsidian_upload( + files=[("One.md", b"# One\nChanged selection.\n")], + attachment_manifest=[], workspace="alpha", vault_label="Cafe\u0301", + ) + resumed = service.preview_obsidian_upload( + files=[("One.md", b"# One\nLocal note.\n")], + attachment_manifest=[], workspace="alpha", vault_id=first["vault_id"], + ) + assert resumed["vault_id"] == first["vault_id"] + assert resumed["vault_label"] == "Caf\u00e9" + finally: + service.close() + + +def test_new_workspace_previews_cannot_reuse_another_workspaces_source(tmp_path): + service = _service() + try: + started, job = _import( + service, [("One.md", b"# One\nApproved upload.\n")], + ) + assert job["state"] == "completed" + upload_preview = service.preview_obsidian_upload( + files=[("One.md", b"# One\nApproved upload.\n")], + attachment_manifest=[], workspace="upload-target-does-not-exist", + vault_label="Team notes", + ) + assert upload_preview["vault_id"] is None + assert upload_preview["counts"]["imported"] == 1 + assert upload_preview["counts"].get("skipped", 0) == 0 + assert upload_preview["vault_id"] != started["vault_id"] + assert service._lookup_workspace("upload-target-does-not-exist") is None + + vault = tmp_path / "DiskVault" + vault.mkdir() + (vault / "One.md").write_text( + "# One\nApproved disk note.\n", encoding="utf-8", + ) + imported = service.import_obsidian_vault( + str(vault), workspace="alpha", vault_label="Disk notes", + confirmed=True, + ) + assert imported["state"] == "completed" + disk_preview = service.preview_obsidian_vault( + str(vault), workspace="disk-target-does-not-exist", + vault_label="Disk notes", + ) + assert disk_preview["vault_id"] is None + assert disk_preview["counts"]["imported"] == 1 + assert disk_preview["counts"].get("skipped", 0) == 0 + assert disk_preview["vault_id"] != imported["vault_id"] + assert service._lookup_workspace("disk-target-does-not-exist") is None + finally: + service.close() + + +def test_import_job_vault_listing_receipt_and_workspace_isolation_are_content_free(): + service = _service() + try: + started, job = _import( + service, + [("Folder/One.md", b"# One\nUPLOAD_BODY_MARKER\nSee [[Two]].\n"), + ("Two.md", b"# Two\nSecond note.\n")], + ) + assert job["state"] == "completed" + assert job["counts"]["imported"] == 2 + assert job["processed_items"] == 2 + assert {row["relative_path"] for row in job["files"]} == { + "Folder/One.md", "Two.md", + } + assert "UPLOAD_BODY_MARKER" not in str(job) + + vaults = service.list_obsidian_vaults("alpha") + assert len(vaults) == 1 + assert set(vaults[0]) == { + "id", "label", "workspace", "repo", "session_id", "scope", + "memory_type", "importer_version", + } + assert "root_digest" not in str(vaults) + resumed_preview = service.preview_obsidian_upload( + files=[("Folder/One.md", b"# One\nUPLOAD_BODY_MARKER\nSee [[Two]].\n")], + attachment_manifest=[], workspace="alpha", vault_id=started["vault_id"], + ) + assert resumed_preview["vault_id"] == started["vault_id"] + assert resumed_preview["vault_label"] == "Team notes" + workspace_id = service._lookup_workspace("alpha") + assert workspace_id is not None + receipt = service.store.list_receipts(workspace_id=workspace_id, limit=10)[0] + assert receipt["operation"] == "obsidian_import" + assert receipt["target_count"] == 2 + assert "One.md" not in str(receipt) + assert "UPLOAD_BODY_MARKER" not in str(receipt) + + service.create_workspace("beta") + assert service.list_obsidian_vaults("beta") == [] + with pytest.raises(KeyError): + service.get_obsidian_import_job(started["job_id"], workspace="beta") + with pytest.raises(ValidationError, match="that target"): + service.preview_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="beta", vault_id=started["vault_id"], + vault_label="Team notes", + ) + with pytest.raises(ValidationError, match="not found"): + service.import_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="must-not-be-created", vault_id=started["vault_id"], + vault_label="Team notes", confirmed=True, + ) + assert service._lookup_workspace("must-not-be-created") is None + with pytest.raises(ValueError, match="different import defaults"): + service.preview_obsidian_upload( + files=[("One.md", b"# One\n")], attachment_manifest=[], + workspace="alpha", vault_id=started["vault_id"], + vault_label="Team notes", memory_type="procedural", + ) + finally: + service.close() + + +def test_upload_reimport_is_idempotent_and_tracks_rename_and_missing(): + service = _service() + try: + initial_files = [ + ("One.md", b"# One\nStable note.\n"), + ("Folder/Two.md", b"# Two\nWill be removed.\n"), + ] + started, first = _import(service, initial_files) + assert first["counts"]["imported"] == 2 + memory_count_row = service.store.conn.execute( + "SELECT COUNT(*) FROM memories" + ).fetchone() + assert memory_count_row is not None + memory_count = memory_count_row[0] + + _again, second = _import( + service, initial_files, vault_id=started["vault_id"], + ) + assert second["counts"]["skipped"] == 2 + current_count = service.store.conn.execute( + "SELECT COUNT(*) FROM memories" + ).fetchone() + assert current_count is not None and current_count[0] == memory_count + + _moved, third = _import( + service, [("Archive/One.md", initial_files[0][1])], + vault_id=started["vault_id"], + ) + assert third["counts"]["renamed"] == 1 + assert third["counts"]["missing"] == 1 + manifest = service.store.list_source_import_items(vault_id=started["vault_id"]) + assert {(row["relative_path"], row["state"]) for row in manifest} == { + ("Archive/One.md", "renamed"), ("Folder/Two.md", "missing"), + } + finally: + service.close() + + +def test_upload_conflict_error_and_replace_policies_preserve_temporal_lineage(): + service = _service() + try: + started, _first = _import(service, [("One.md", b"# One\nImported.\n")]) + source = service.store.list_source_import_items(vault_id=started["vault_id"])[0] + imported = service.store.get_memory(source["memory_id"]) + assert imported is not None + assert imported.workspace_id is not None + correction = service.engine.remember_with_resolution( + "# One\nOwner correction.\n", workspace_id=imported.workspace_id, + repo_id=imported.repo_id, session_id=imported.session_id, + scope=imported.scope, mtype=imported.mtype, title=imported.title, + subject_key=imported.subject_key, claim_kind=imported.claim_kind, + valid_from=(imported.valid_from or 0) + 1, + ) + + _conflict, reported = _import( + service, [("One.md", b"# One\nSource changed.\n")], + vault_id=started["vault_id"], on_conflict="error", + ) + assert reported["state"] == "partial" + assert reported["counts"]["conflict"] == 1 + corrected = service.store.get_memory(correction["id"]) + assert corrected is not None and corrected.valid_to is None + + _replace, replaced = _import( + service, [("One.md", b"# One\nSource changed.\n")], + vault_id=started["vault_id"], on_conflict="replace", + ) + assert replaced["state"] == "completed" + assert replaced["counts"]["updated"] == 1 + superseded = service.store.get_memory(correction["id"]) + assert superseded is not None and superseded.valid_to is not None + finally: + service.close() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index ee1456d3..8a9b925c 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -69,17 +69,26 @@ def test_http_mcp_cli_configures_the_packaged_transport(monkeypatch): from engraphis import mcp_http_cli calls = [] + security_calls = [] + transport_security = object() fake_mcp = types.SimpleNamespace( settings=types.SimpleNamespace(host=None, port=None), run=lambda *, transport: calls.append(transport), ) monkeypatch.setattr(mcp_http_cli, "_dependency_error", lambda: "") + monkeypatch.setattr( + mcp_http_cli, + "_transport_security", + lambda host, port: security_calls.append((host, port)) or transport_security, + ) monkeypatch.setitem(sys.modules, "engraphis.mcp_server", types.SimpleNamespace(mcp=fake_mcp)) mcp_http_cli.main(["--host", "::1", "--port", "9876", "--transport", "sse"]) assert fake_mcp.settings.host == "::1" assert fake_mcp.settings.port == 9876 + assert fake_mcp.settings.transport_security is transport_security + assert security_calls == [("::1", 9876)] assert calls == ["sse"] @@ -214,6 +223,7 @@ def test_distribution_configuration_includes_public_evidence_tools(): for rule in ( "include LICENSE NOTICE README.md CHANGELOG.md BENCHMARKS.md", "include docs/RECALL_RECOVERY.md", + "include docs/DOCUMENT_IMPORT.md docs/OBSIDIAN_IMPORT.md", "include docs/images/context-efficiency.svg", "include docker-entrypoint.sh Dockerfile docker-compose.yml docker-compose.lan.yml", "recursive-include eval *.py", @@ -385,18 +395,48 @@ def test_manual_release_dispatch_cannot_publish(): def test_source_tree_version_matches_pyproject(): - """The ``PackageNotFoundError`` fallback in ``engraphis/__init__.py`` must equal the - ``[project] version``. It only shows up in an uninstalled source tree, so a stale - value survives every test run on an installed checkout and then leaks into the API - index and ``--version`` output of anyone running from a clone.""" + """Both source-tree versions must equal the ``[project] version``.""" import re pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") init = (ROOT / "engraphis" / "__init__.py").read_text(encoding="utf-8") declared = re.search(r'^version = "([^"]+)"', pyproject, re.M) + source = re.search(r'^_SOURCE_VERSION = "([^"]+)"', init, re.M) fallback = re.search(r'^ __version__ = "([^"]+)"', init, re.M) - assert declared and fallback, "version declarations moved — update this test" - assert declared.group(1) == fallback.group(1) + assert declared and source and fallback, "version declarations moved — update this test" + assert declared.group(1) == source.group(1) == fallback.group(1) + + +def test_release_version_surfaces_are_synchronized(): + """Repo-distributed integrations and release filters track the package version.""" + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + declared = re.search(r'^version = "([^"]+)"', pyproject, re.M) + assert declared, "project version declaration moved — update this test" + version = declared.group(1) + + commercial = json.loads( + (ROOT / "engraphis" / "commercial_manifest.json").read_text(encoding="utf-8") + ) + assert commercial["version"] == version + + hermes = (ROOT / "integrations" / "hermes" / "engraphis" / "plugin.yaml").read_text( + encoding="utf-8" + ) + hermes_version = re.search(r"^version:\s*(\S+)\s*$", hermes, re.M) + assert hermes_version, "Hermes version declaration moved — update this test" + expected_hermes = version if version.count(".") >= 2 else f"{version}.0" + assert hermes_version.group(1) == expected_hermes + + ledger = (ROOT / "engraphis" / "dashboard_assets" / "ledger.js").read_text( + encoding="utf-8" + ) + assert re.findall(r"release_version=([0-9]+(?:\.[0-9]+)*)", ledger) == [version] + + static = (ROOT / "engraphis" / "static" / "dashboard.js").read_bytes() + classic = (ROOT / "engraphis" / "classic_assets" / "dashboard.js").read_bytes() + assert static == classic + static_text = static.decode("utf-8") + assert re.findall(r"p\.set\('release_version','([^']+)'\)", static_text) == [version] def test_release_version_has_a_dated_changelog_section(): diff --git a/tests/test_postgres_schema.py b/tests/test_postgres_schema.py index c89df648..7b6aab4e 100644 --- a/tests/test_postgres_schema.py +++ b/tests/test_postgres_schema.py @@ -6,7 +6,8 @@ from engraphis.backends import postgres_schema from engraphis.core.interfaces import SchemaSnapshot, SearchFilter -from engraphis.service import MemoryService +import engraphis.service as service_module +from engraphis.service import MAX_CONTENT_CHARS, MemoryService class _Cursor: @@ -360,6 +361,32 @@ def inspect(self, supplied, *, schemas=None): assert "secret" not in serialized +def test_empty_postgres_chunk_result_returns_without_indexing(monkeypatch): + snapshot = SchemaSnapshot( + title="PostgreSQL schema: empty", + text="x" * (MAX_CONTENT_CHARS + 1), + metadata={"database": "empty", "source_digest": "digest"}, + ) + + class _Introspector: + def inspect(self, supplied, *, schemas=None): + return snapshot + + class _EmptyExtractor: + def extract(self, _text): + return [] + + monkeypatch.setattr( + postgres_schema, "get_postgres_introspector", lambda: _Introspector() + ) + monkeypatch.setattr(service_module, "ChunkingExtractor", _EmptyExtractor) + service = MemoryService.create(":memory:") + + assert service.import_postgres_schema( + "postgresql://local/empty", workspace="acme" + ) == {"workspace": "acme", "stored": 0, "entities": 0, "relations": 0} + + def test_large_postgres_snapshot_keeps_every_chunk_distinct(monkeypatch): snapshot = SchemaSnapshot( title="PostgreSQL schema: large", diff --git a/tests/test_public_research_boundary.py b/tests/test_public_research_boundary.py index 76690d3c..60113ff3 100644 --- a/tests/test_public_research_boundary.py +++ b/tests/test_public_research_boundary.py @@ -116,6 +116,16 @@ def test_public_narrative_has_no_named_competitor_positioning(): *(ROOT / "eval").rglob("*.md"), }) named_competitors = ("obsidian", "mem0", "zep", "letta") + # A product-specific source adapter must name the format it parses. Keep that + # exception constrained to the importer guide and its four discovery/changelog + # surfaces; comparison/positioning language remains forbidden everywhere. + obsidian_integration_docs = { + ROOT / "AGENTS.md", + ROOT / "README.md", + ROOT / "CHANGELOG.md", + ROOT / "docs" / "DOCUMENT_IMPORT.md", + ROOT / "docs" / "OBSIDIAN_IMPORT.md", + } private_research_phrases = ( "commercial audit", "competitive analysis", @@ -127,5 +137,9 @@ def test_public_narrative_has_no_named_competitor_positioning(): ) for path in public_markdown: content = path.read_text(encoding="utf-8").casefold() - assert all(name not in content for name in named_competitors), path + forbidden_names = ( + named_competitors[1:] + if path in obsidian_integration_docs else named_competitors + ) + assert all(name not in content for name in forbidden_names), path assert all(phrase not in content for phrase in private_research_phrases), path diff --git a/tests/test_read_only_api.py b/tests/test_read_only_api.py index dede8249..c70725b2 100644 --- a/tests/test_read_only_api.py +++ b/tests/test_read_only_api.py @@ -5,7 +5,7 @@ from fastapi.testclient import TestClient from engraphis.config import settings -from engraphis.read_only_api import create_read_only_app +from engraphis.read_only_api import MAX_READ_ONLY_BODY_BYTES, create_read_only_app from engraphis.service import MemoryService from engraphis.backends.graph_extractor import RegexGraphExtractor @@ -323,3 +323,30 @@ def recall(self, *args, **kwargs): assert secret not in response.text assert secret not in caplog.text assert "RuntimeError" in caplog.text + + +def test_read_only_api_rejects_declared_oversized_body_with_fixed_detail(): + app = create_read_only_app(object()) + response = TestClient(app).post( + "/intent/recall", + content=b"{}", + headers={"content-length": str(MAX_READ_ONLY_BODY_BYTES + 1)}, + ) + + assert response.status_code == 413 + assert response.json() == {"detail": "request body too large"} + + +def test_read_only_api_rejects_streamed_oversized_body_with_413(): + # Chunked/streamed requests carry no Content-Length, so the middleware must + # translate the over-limit receive itself instead of relying on the declared + # length check or a ValueError that FastAPI's parser swallows as a 400. + app = create_read_only_app(object()) + response = TestClient(app).post( + "/intent/recall", + content=b"x" * (MAX_READ_ONLY_BODY_BYTES + 1), + headers={"transfer-encoding": "chunked"}, + ) + + assert response.status_code == 413 + assert response.json() == {"detail": "request body too large"} diff --git a/tests/test_recall.py b/tests/test_recall.py index 0217ccbb..f1b584a4 100644 --- a/tests/test_recall.py +++ b/tests/test_recall.py @@ -664,6 +664,40 @@ def test_recall_edge_filter_rejects_untrusted_source_less_edges(): } +def test_prompt_edge_support_must_match_active_scope_and_validity(): + from engraphis.core.interfaces import Edge + + store, emb, eng = _engine() + allowed = store.get_or_create_workspace("allowed") + foreign = store.get_or_create_workspace("foreign") + foreign_memory = _add(store, emb, foreign, None, "Foreign approved support.") + expired_memory = _add( + store, emb, allowed, None, "Expired approved support.", + valid_from=0.0, valid_to=10.0, + ) + edges = [ + Edge( + id="foreign-support", src="a", dst="b", relation="supports", + workspace_id=allowed, + provenance={ + "trusted": True, "review_state": "approved", + "memory_id": foreign_memory, + }, + ), + Edge( + id="expired-support", src="a", dst="c", relation="supports", + workspace_id=allowed, + provenance={ + "trusted": True, "review_state": "approved", + "memory_id": expired_memory, + }, + ), + ] + + flt = SearchFilter(workspace_id=allowed, valid_at=20.0) + assert eng._prompt_eligible_edges(edges, flt) == [] + + def test_graph_arm_backfills_workspace_mentions_for_a_later_repo_entity(): from engraphis.core.interfaces import Edge, Node diff --git a/tests/test_receipts.py b/tests/test_receipts.py index 403ac3ef..05333754 100644 --- a/tests/test_receipts.py +++ b/tests/test_receipts.py @@ -46,6 +46,89 @@ def test_receipts_are_content_free_and_tamper_evident(): } +def test_obsidian_import_receipt_keeps_only_bounded_counts(): + store = Store(":memory:") + try: + wid = store.get_or_create_workspace("private-vault-workspace") + receipt = store.record_receipt( + "obsidian_import", workspace_id=wid, target_count=3, status="partial", + metadata={ + "files_scanned": 5, "files_imported": 3, "files_updated": 1, + "files_renamed": 1, "files_skipped": 1, "files_rejected": 0, + "files_missing": 1, "files_errored": 0, "conflicts": 2, + "warnings": 2, "attachments": 4, "wikilinks": 6, + "aliases": 2, "tags": 9, "vault_path": "C:/private/vault", + "title": "private title", + }, + ) + assert receipt["operation"] == "obsidian_import" + assert receipt["metadata"] == { + "files_scanned": 5, "files_imported": 3, "files_updated": 1, + "files_renamed": 1, "files_skipped": 1, "files_rejected": 0, + "files_missing": 1, "files_errored": 0, "conflicts": 2, + "warnings": 2, "attachments": 4, "wikilinks": 6, + "aliases": 2, "tags": 9, + } + assert "private" not in json.dumps(receipt) + assert store.verify_receipts(workspace_id=wid)["valid"] is True + + adversarial = store.record_receipt( + "obsidian_import", workspace_id=wid, + metadata={ + "files_scanned": 10**30, + "files_imported": -10, + "files_updated": True, + "files_skipped": 1.5, + "warnings": "private warning text", + }, + ) + assert adversarial["metadata"] == { + "files_imported": 0, + "files_scanned": 1_000_000_000, + } + assert "private warning" not in json.dumps(adversarial) + assert store.verify_receipts(workspace_id=wid)["valid"] is True + finally: + store.close() + + +def test_completed_obsidian_import_receipt_has_public_terminal_status(): + store = Store(":memory:") + try: + wid = store.get_or_create_workspace("obsidian-complete") + receipt = store.record_receipt( + "obsidian_import", workspace_id=wid, status="completed", + metadata={"files_imported": 1}, + ) + assert receipt["status"] == "completed" + finally: + store.close() + + +def test_document_import_receipt_is_public_and_content_free(): + store = Store(":memory:") + try: + workspace_id = store.get_or_create_workspace("documents-receipt") + payload = store.record_receipt( + "document_import", workspace_id=workspace_id, status="completed", + target_count=4, + metadata={ + "files_imported": 3, "files_skipped": 1, + "warnings": 2, "source_path": "C:/private/customer/files", + "document_title": "Confidential roadmap", + }, + ) + assert payload["operation"] == "document_import" + assert payload["status"] == "completed" + assert payload["metadata"] == { + "files_imported": 3, "files_skipped": 1, "warnings": 2, + } + assert "private" not in str(payload).casefold() + assert "confidential" not in str(payload).casefold() + finally: + store.close() + + def test_short_user_controlled_receipt_labels_are_never_stored_verbatim(): store = Store(":memory:") wid = store.get_or_create_workspace("w") diff --git a/tests/test_round17_fixes.py b/tests/test_round17_fixes.py index 68a95627..d6787bea 100644 --- a/tests/test_round17_fixes.py +++ b/tests/test_round17_fixes.py @@ -22,6 +22,32 @@ def test_get_or_create_workspace_enforces_allowlist(tmp_path): s.close() +def test_bound_store_enforces_allowlist_for_scoped_reads_and_writes(tmp_path): + from engraphis.core.interfaces import MemoryRecord + from engraphis.core.store import Store + + path = tmp_path / "bound.db" + seed = Store(str(path)) + allowed_id = seed.get_or_create_workspace("allowed") + secret_id = seed.get_or_create_workspace("secret") + secret_memory = seed.add_memory(MemoryRecord( + id="", content="private", workspace_id=secret_id, + )) + seed.close() + + bound = Store(str(path), allowed_workspaces={"allowed"}) + with pytest.raises(ValueError): + bound.create_repo(secret_id, "repo") + with pytest.raises(ValueError): + bound.start_session(secret_id) + with pytest.raises(ValueError): + bound.add_memory(MemoryRecord(id="", content="blocked", workspace_id=secret_id)) + assert bound.get_memory(secret_memory) is None + assert bound.list_memories() == [] + assert bound.get_or_create_workspace("allowed") == allowed_id + bound.close() + + def test_sync_apply_preserves_future_world_validity(): from engraphis.core.sync import dict_to_record future = time.time() + 5 * 365 * 86400 # ~5 years out (a fact valid until then) diff --git a/tests/test_savings.py b/tests/test_savings.py index 8c8f9463..4a519e14 100644 --- a/tests/test_savings.py +++ b/tests/test_savings.py @@ -255,6 +255,21 @@ def test_grouped_context_savings_uses_the_same_time_and_release_filters(): assert summary["csv"].splitlines()[0].startswith("group_key,token_counter,") +def test_ungrouped_context_savings_honors_csv_format(): + service = MemoryService.create(":memory:", graph_extractor="none") + wid = service.store.get_or_create_workspace("ungrouped-csv") + service.store.record_receipt( + "recall", + workspace_id=wid, + metadata={"token_usage": _usage(100, 40, counter="engraphis.regex.v1")}, + ) + + summary = service.context_savings(workspace="ungrouped-csv", format="csv") + + assert summary["csv"].splitlines()[0].startswith("token_counter,receipt_count,") + assert "engraphis.regex.v1" in summary["csv"] + + def test_grouped_context_savings_separates_counters_and_rejects_invalid_saved(): store = Store(":memory:") wid = store.get_or_create_workspace("grouped-counters") diff --git a/tests/test_secret_hygiene.py b/tests/test_secret_hygiene.py index e9ba98e7..bf7681d1 100644 --- a/tests/test_secret_hygiene.py +++ b/tests/test_secret_hygiene.py @@ -179,6 +179,33 @@ def test_secure_erase_rebuilds_shared_edge_provenance_from_remaining_support(): ] +def test_secure_erase_removes_sync_conflict_successors(): + engine = MemoryEngine.create(":memory:") + workspace = engine.store.get_or_create_workspace("acme") + original_id = engine.remember("Original secret source.", workspace_id=workspace) + from engraphis.core import ids + successor_id = ids.new_id("memory") + engine.store.add_memory(MemoryRecord( + id=successor_id, + content="Losing secret copy.", + workspace_id=workspace, + scope=Scope.WORKSPACE, + metadata={ + "sync_conflict": {"memory_id": original_id}, + }, + provenance={"conflict_of": original_id, "trusted": False}, + )) + + engine.secure_erase(original_id) + + assert engine.store.get_memory(original_id) is None + assert engine.store.get_memory(successor_id) is None + assert { + row["id"] + for row in engine.store.list_memory_tombstones() + } >= {original_id, successor_id} + + def test_secure_erase_preserves_shared_edge_history_from_retired_support(): engine = MemoryEngine.create(":memory:") workspace = engine.store.get_or_create_workspace("acme") diff --git a/tests/test_service.py b/tests/test_service.py index afc40192..17bc822d 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -130,6 +130,26 @@ def capture(*args, **kwargs): record = service.store.get_memory(result["results"][0]["id"]) assert record.metadata["retention_supervision"]["label"] == "critical" +def test_remember_batch_forwards_metadata_retention_and_conflict_policy(): + service = MemoryService.create(":memory:", graph_extractor="none") + + result = service.remember_batch( + [{ + "content": "A critical batch fact.", + "metadata": {"origin": "batch"}, + "retention_class": "critical", + "retention_reason": "release policy", + "resolve_conflicts": False, + }], + workspace="acme", + ) + + record = service.store.get_memory(result["results"][0]["id"]) + assert record is not None + assert record.metadata["origin"] == "batch" + assert record.metadata["retention_supervision"]["label"] == "critical" + assert record.metadata["retention_supervision"]["reason"] == "release policy" + def test_memory_health_binds_time_parameters_and_scopes_conflicts_to_workspace(): service = MemoryService.create(":memory:", graph_extractor="none") @@ -546,6 +566,7 @@ def test_update_memory_preserves_metadata_changes_on_a_correction_replacement(): replacement = s.correct( original["id"], "The revised deployment runbook.", workspace="acme", ) + before_hlc = s.store.get_memory(replacement["id"]).modified_hlc out = s.update_memory( replacement["id"], workspace="acme", title="Deployment runbook", @@ -557,6 +578,7 @@ def test_update_memory_preserves_metadata_changes_on_a_correction_replacement(): assert (saved.title, saved.mtype.value, saved.importance) == ( "Deployment runbook", "procedural", 0.9, ) + assert saved.modified_hlc != before_hlc def test_update_memory_advances_descriptive_hlc(): @@ -659,7 +681,7 @@ def forbidden_embed(_texts): -def test_update_memory_rolls_back_title_when_index_update_fails(): +def test_update_memory_commits_canonical_title_when_external_index_update_fails(caplog): service = MemoryService.create(":memory:") created = service.remember("A durable release note.", workspace="acme", title="Old") mid = created["id"] @@ -667,30 +689,106 @@ def test_update_memory_rolls_back_title_when_index_update_fails(): "SELECT vector FROM mem_vectors WHERE id=?", (mid,) ).fetchone()["vector"] original_index = service.engine.index + commits = [] class BrokenIndex: dim = original_index.dim def upsert(self, _ids, _vectors, meta=None, *, commit=True): - raise RuntimeError("index unavailable") + commits.append(commit) + raise RuntimeError("sensitive-index-detail") def delete(self, _ids, *, commit=True): return None service.engine.index = BrokenIndex() - with pytest.raises(RuntimeError, match="index unavailable"): - service.update_memory(mid, workspace="acme", title="New") + with caplog.at_level("WARNING", logger="engraphis.service"): + result = service.update_memory(mid, workspace="acme", title="New") + + assert result == {"id": mid, "updated": ["title"]} + assert commits == [True] saved = service.store.get_memory(mid) - assert saved.title == "Old" + assert saved.title == "New" assert service.store.conn.execute( "SELECT vector FROM mem_vectors WHERE id=?", (mid,) - ).fetchone()["vector"] == before_vector + ).fetchone()["vector"] != before_vector assert mid in { - memory_id for memory_id, _score in service.store.fts_search("Old", 5) - } - assert mid not in { memory_id for memory_id, _score in service.store.fts_search("New", 5) } + audit = service.store.conn.execute( + "SELECT detail FROM audit WHERE action='index_upsert_failed' AND target=?", + (mid,), + ).fetchone() + assert audit is not None and audit["detail"] == "failure_type=RuntimeError" + assert "sensitive-index-detail" not in caplog.text + + +def test_update_memory_late_store_failure_does_not_publish_external_vector(monkeypatch): + service = MemoryService.create(":memory:") + created = service.remember("A durable release note.", workspace="acme", title="Old") + mid = created["id"] + before_vector = service.store.conn.execute( + "SELECT vector FROM mem_vectors WHERE id=?", (mid,) + ).fetchone()["vector"] + publications = [] + + class RecordingExternalIndex: + def upsert(self, ids, _vectors, meta=None, *, commit=True): + publications.append(("upsert", tuple(ids), commit)) + + def delete(self, ids, *, commit=True): + publications.append(("delete", tuple(ids), commit)) + + service.engine.index = RecordingExternalIndex() + original_audit = service.store.audit + + def fail_late(actor, action, target, detail="", *, commit=True): + if action == "memory_update": + raise RuntimeError("late store failure") + return original_audit(actor, action, target, detail, commit=commit) + + monkeypatch.setattr(service.store, "audit", fail_late) + with pytest.raises(RuntimeError, match="late store failure"): + service.update_memory(mid, workspace="acme", title="New") + + assert service.store.get_memory(mid).title == "Old" + assert service.store.conn.execute( + "SELECT vector FROM mem_vectors WHERE id=?", (mid,) + ).fetchone()["vector"] == before_vector + assert publications == [] + + +def test_update_memory_commit_failure_does_not_publish_external_vector(monkeypatch): + service = MemoryService.create(":memory:") + created = service.remember("A durable release note.", workspace="acme", title="Old") + mid = created["id"] + publications = [] + + class RecordingExternalIndex: + def upsert(self, ids, _vectors, meta=None, *, commit=True): + publications.append(("upsert", tuple(ids), commit)) + + def delete(self, ids, *, commit=True): + publications.append(("delete", tuple(ids), commit)) + + service.engine.index = RecordingExternalIndex() + connection_type = type(service.store.conn) + real_commit = connection_type.commit + + def fail_outer_commit(connection): + if ( + connection is service.store.conn + and not getattr(connection._pin, "defer_commits", 0) + ): + raise RuntimeError("late commit failure") + return real_commit(connection) + + monkeypatch.setattr(connection_type, "commit", fail_outer_commit) + with pytest.raises(RuntimeError, match="late commit failure"): + service.update_memory(mid, workspace="acme", title="New") + + assert service.store.get_memory(mid).title == "Old" + assert publications == [] def test_update_memory_rebuilds_missing_fts_row_when_title_is_reapplied(): service = MemoryService.create(":memory:") @@ -1576,3 +1674,38 @@ def hold_worker(_job_id): finally: release.set() service.close() + + +def test_document_import_launcher_marks_job_failed_when_worker_start_raises(monkeypatch): + """If Thread.start() raises, the job must not be left running forever: the + launcher marks it failed and removes the thread from the owned-workers dict + (the same failure pattern the graph-index launcher uses).""" + from engraphis.document_import import DocumentImporter + + s = _svc() + s.create_workspace("acme") + + original_thread_start = threading.Thread.start + + def fail_start(self, *args, **kwargs): + raise RuntimeError("thread pool exhausted") + + monkeypatch.setattr(threading.Thread, "start", fail_start) + try: + with pytest.raises(RuntimeError, match="thread pool exhausted"): + s.import_document_upload( + files=[("notes.md", b"# Title\nstart failure fact")], + attachment_manifest=None, + workspace="acme", + source_label="start-failure-source", + confirmed=True, + ) + finally: + monkeypatch.setattr(threading.Thread, "start", original_thread_start) + + assert s._obsidian_job_threads == {} + row = s.store.conn.execute( + "SELECT id, state FROM jobs WHERE kind=? ORDER BY created_at DESC LIMIT 1", + (DocumentImporter.JOB_KIND,), + ).fetchone() + assert row is not None and row["state"] == "failed" diff --git a/tests/test_sync.py b/tests/test_sync.py index 383809f6..cae2d47d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -896,7 +896,8 @@ def delete(self, _ids, *, commit=True): assert store.conn.execute( "SELECT 1 FROM mem_vectors WHERE id='mem_existing'" ).fetchone() is None - assert commits == [False] + # A separate provider is called only after the canonical quarantine/delete commits. + assert commits == [True] audit = store.conn.execute( "SELECT actor, action, target, detail FROM audit " "WHERE action='index_delete_failed'" @@ -2619,7 +2620,9 @@ def upsert(self, _ids, _vecs, meta=None, *, commit=True): report = syncer.apply_bundle(_bundle(3)) assert report["added"] == 3 - assert commits == [False, False, False] + # The separately-backed provider is published only after each canonical Store + # batch commits, so it owns its own durability boundary. + assert commits == [True, True, True] assert engine.store.conn.execute( "SELECT COUNT(*) FROM mem_vectors" ).fetchone()[0] == 3 @@ -2638,6 +2641,140 @@ def upsert(self, _ids, _vecs, meta=None, *, commit=True): assert "sensitive-index-detail" not in caplog.text +def test_sync_late_store_failure_does_not_publish_external_vector(monkeypatch): + engine = MemoryEngine.create(":memory:", vector_backend="numpy") + publications = [] + + class RecordingExternalIndex: + def upsert(self, ids, _vecs, meta=None, *, commit=True): + publications.append(("upsert", tuple(ids), commit)) + + def delete(self, ids, *, commit=True): + publications.append(("delete", tuple(ids), commit)) + + syncer = SyncEngine( + engine.store, + embedder=engine.embedder, + vector_index=RecordingExternalIndex(), + ) + original_audit = engine.store.audit + + def fail_late(actor, action, target, detail="", *, commit=True): + if action == "sync_add": + raise RuntimeError("late sync store failure") + return original_audit(actor, action, target, detail, commit=commit) + + monkeypatch.setattr(engine.store, "audit", fail_late) + with pytest.raises(RuntimeError, match="late sync store failure"): + syncer.apply_bundle(_bundle(1)) + + assert engine.store.get_memory("mem_0") is None + assert publications == [] + + +def test_hlc_conflict_variant_publishes_external_vector(): + engine = MemoryEngine.create(":memory:", vector_backend="numpy") + publications = [] + + class RecordingExternalIndex: + def upsert(self, ids, _vecs, meta=None, *, commit=True): + publications.extend(ids) + + def delete(self, ids, *, commit=True): + del ids, commit + + lower_node = f"dev_{'0' * 26}" + higher_node = f"dev_{'1' * 26}" + + def bundle(content, node): + return { + "format": SYNC_FORMAT, + "version": 2, + "device_id": node, + "workspace_name": "w", + "repos": {}, + "memories": [{ + "id": "same-hlc-id", + "content": content, + "ingested_at": 42.0, + "valid_from": 42.0, + "modified_hlc": format_modified_hlc(42, 1, node), + }], + "mem_links": [], + } + + syncer = SyncEngine( + engine.store, + embedder=engine.embedder, + vector_index=RecordingExternalIndex(), + ) + syncer.apply_bundle(bundle("lower-node edit", lower_node), into_workspace="w") + publications.clear() + syncer.apply_bundle(bundle("higher-node edit", higher_node), into_workspace="w") + + conflict_id = engine.store.conn.execute( + "SELECT id FROM memories WHERE id <> 'same-hlc-id'" + ).fetchone()["id"] + assert conflict_id in publications + + +def test_hlc_conflict_successor_external_vector_matches_store_vector(): + engine = MemoryEngine.create(":memory:", vector_backend="numpy") + publications = [] + + class RecordingExternalIndex: + def upsert(self, ids, vecs, meta=None, *, commit=True): + publications.append((tuple(ids), vecs.copy(), meta)) + + def delete(self, ids, *, commit=True): + publications.append(("delete", tuple(ids), None)) + + lower_node = f"dev_{'0' * 26}" + higher_node = f"dev_{'1' * 26}" + + def bundle(content, node): + return { + "format": SYNC_FORMAT, + "version": 2, + "device_id": node, + "workspace_name": "w", + "repos": {}, + "memories": [{ + "id": "same-hlc-id", + "content": content, + "ingested_at": 42.0, + "valid_from": 42.0, + "modified_hlc": format_modified_hlc(42, 1, node), + }], + "mem_links": [], + } + + syncer = SyncEngine( + engine.store, + embedder=engine.embedder, + vector_index=RecordingExternalIndex(), + ) + syncer.apply_bundle(bundle("lower-node edit", lower_node), into_workspace="w") + publications.clear() + syncer.apply_bundle(bundle("higher-node edit", higher_node), into_workspace="w") + + conflict_id = engine.store.conn.execute( + "SELECT id FROM memories WHERE id <> 'same-hlc-id'" + ).fetchone()["id"] + # The preserved successor must be published to the separately-backed index with + # exactly the canonical vector the Store committed for its id. + published = [entry for entry in publications + if entry[0] != "delete" and conflict_id in entry[0]] + assert published, "conflict successor was not published to the external index" + _, external_vector, external_meta = published[-1] + external_vector = np.asarray(external_vector, dtype=np.float32).reshape(-1) + + stored = engine.store.get_vectors([conflict_id])[conflict_id] + assert external_vector.shape == stored.shape + np.testing.assert_allclose(external_vector, stored, rtol=0, atol=0) + assert external_meta == [{"model": syncer.embedding_space}] + + def test_sync_configured_embedder_failure_aborts_before_memory_write(caplog): engine = MemoryEngine.create(":memory:", vector_backend="numpy") diff --git a/tests/test_sync_cli.py b/tests/test_sync_cli.py index 40ec9bbf..332df95e 100644 --- a/tests/test_sync_cli.py +++ b/tests/test_sync_cli.py @@ -19,6 +19,7 @@ EncryptedRelayTransport, RelayError, RelayTransport, + _saved_sync_token, decode_sync_e2ee_key, ) from engraphis.core.engine import MemoryEngine @@ -45,6 +46,26 @@ def test_decode_sync_e2ee_key_rejects_short_and_malformed_values(): decode_sync_e2ee_key("A" * 43 + "==") # two pads is not a 32-byte key +def test_saved_sync_token_rejects_configured_token_bound_to_another_relay(monkeypatch): + monkeypatch.setenv("ENGRAPHIS_SYNC_TOKEN", "engr_ut_" + "x" * 32) + monkeypatch.setenv("ENGRAPHIS_SYNC_TOKEN_ORIGIN", "https://trusted.test") + + with pytest.raises(RelayError, match="belongs to another relay") as caught: + _saved_sync_token("https://other.test") + + assert caught.value.status == 409 + + +def test_saved_sync_token_rejects_configured_token_without_valid_origin(monkeypatch): + monkeypatch.setenv("ENGRAPHIS_SYNC_TOKEN", "engr_ut_" + "x" * 32) + monkeypatch.setenv("ENGRAPHIS_SYNC_TOKEN_ORIGIN", "") + + with pytest.raises(RelayError, match="no valid relay binding") as caught: + _saved_sync_token("https://other.test") + + assert caught.value.status == 409 + + def test_get_transport_relay_builds_relay_transport(monkeypatch): pytest.importorskip("cryptography") monkeypatch.setattr(