diff --git a/Cargo.lock b/Cargo.lock index d7e858b..5fb9a37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,15 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.12.1" @@ -187,6 +196,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -254,6 +272,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "deranged" version = "0.5.8" @@ -263,6 +291,16 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -389,6 +427,16 @@ dependencies = [ "windows-result", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -409,9 +457,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1077,6 +1127,7 @@ name = "quire-rs" version = "0.9.1" dependencies = [ "criterion", + "getrandom 0.3.4", "ignore", "indexmap", "jsonschema", @@ -1088,6 +1139,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "thiserror", "uuid", ] @@ -1329,6 +1381,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1531,6 +1594,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unarray" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index 0e11c50..6882b25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } serde = { version = "^1", features = ["derive"] } serde_json = "^1" serde_yaml = "^0.9" +sha2 = "^0.10" indexmap = { version = "^2", features = ["serde"] } thiserror = "^2" # Regex driver for the Query API (FR-010). Patterns are author-side @@ -51,6 +52,7 @@ uuid = { version = "^1", features = ["v7", "serde"] } # forward-compatible wheel (NFR-016). `extension-module` drops libpython # linkage so the wheel loads in any host interpreter. pyo3 = { version = "0.29", features = ["extension-module", "abi3-py39"], optional = true } +getrandom = { version = "^0.3", features = ["wasm_js"], optional = true } [dev-dependencies] proptest = "^1" @@ -77,7 +79,7 @@ resolve-file = ["jsonschema/resolve-file"] # / `from_default` remain available but operate against whatever # filesystem the wasm host exposes (typically none under # `--target web`). -wasm = [] +wasm = ["uuid/js", "dep:getrandom"] [[bench]] name = "parse" diff --git a/reviews/2026-07-11-issue-127-malformed-frontmatter.md b/reviews/2026-07-11-issue-127-malformed-frontmatter.md new file mode 100644 index 0000000..9642bf5 --- /dev/null +++ b/reviews/2026-07-11-issue-127-malformed-frontmatter.md @@ -0,0 +1,60 @@ +--- +id: SR-001 +title: "Gap analysis — issue #127 malformed-frontmatter parse failure" +type: SpecReview +analysis: gap-analysis +scope: "spec/functional/FR-040-filament-core-extraction-engine.md, spec/functional/FR-006-frontmatter-with-fallback.md, spec/tests.md" +review_set: subset +relationships: + - { target: "ix://agent-ix/quire-rs/spec/functional/FR-040", type: reviews } + - { target: "ix://agent-ix/quire-rs/spec/functional/FR-006", type: references } + - { target: "ix://agent-ix/quire-rs/spec/tests", type: references } +--- + +## Summary + +Post-implementation gap analysis of the issue #127 change, which makes the Filament +extraction engine emit a `parse_failed` error when a document has a complete-but-unparsable +frontmatter fence block (so the failure reaches Filament's `index_errors` via +`MarkStale → apply_stale`). The change spans the parser (a new typed `FrontmatterStatus` on +`extract_frontmatter`, FR-006/CR-010) and the Filament boundary (`FR-040-AC-6`). Mechanical +gates (matrix backing, reverse-gap) pass; two low-severity process observations are recorded. +The optional semantic review was skipped at the user's request. + +## Verdict + +**CONDITIONAL** — no incomplete tasks, no unbacked matrix Test Cases, and no high/medium +findings. FND-002 (traceability) has been **resolved** post-review; FND-001 remains as an +accepted, non-blocking process note (no plan bundle for a goal-driven bugfix, consistent with +how FR-040/FR-041 themselves landed). + +## Findings + +| ID | Severity | Summary | Refs | +| ------- | -------- | --------------------------------------------------------------------------------------------------------- | --------------------------------- | +| FND-001 | low | The #127 change has no owning `plan/tasks/NNN` bundle entry (goal-driven fix); FR-040/FR-041 also landed without plan-task tracking. Accepted — change is complete and verified regardless. | plan/tasks/README.md, FR-040 | +| FND-002 | low | RESOLVED — the CR-010 empty/null→`Absent` classification now has a dedicated `FR-006-AC-7 → TC-658`, and the parser test carries the `FR-006-AC-7 / TC-658` tracking tag. Coverage updated 340 → 341. | FR-006-AC-7, TC-658 | + +## Coverage + +- **Tasks done:** N/A — no formal `plan/tasks/` bundle owns FR-040/FR-041 or the #127 fix + (see FND-001). No task is incomplete or blocked. +- **Matrix Test Cases backed by tagged test:** 2 / 2 new rows. `TC-657` is backed by the + `tc657_malformed_frontmatter_is_a_parse_failure_but_absent_is_clean` unit test + (`src/filament.rs`) and the `["TC-657"]`-tagged `frontmatter-unparsable-error` fixture in + `tests/fixtures/filament_core/graph_cases.json`, exercised by the corpus harness (value + + determinism). `TC-658` (added to resolve FND-002) is backed by the + `empty_or_comment_only_frontmatter_is_absent_not_malformed` parser test carrying the + `FR-006-AC-7 / TC-658` tag. Both `FR-040-AC-6 → TC-657` and `FR-006-AC-7 → TC-658` mappings + are present in the AC→TC audit table; the coverage count is consistently updated + (339 → 341, integrity note aligned). No existing matrix row was disturbed. +- **Untraced behaviors / stubs:** 0. All new code has an owning requirement — the + `FrontmatterStatus` enum and `status` field trace to FR-006 §5 (CR-010); the + `malformed_frontmatter_result` / `absent_frontmatter_result` helpers and the + `parse_failed` / `frontmatter_unparsable` emission trace to FR-040-AC-6 and the split + FR-040 behavior bullet. The two incidental clippy fixes (`Tier2Extraction` type alias, + `stable_id` borrow) are behavior-preserving and carry no spec surface. +- **Semantic review:** skipped (user opted out). Intent↔test↔code for the FR-040-AC-6 triple + was nonetheless spot-checked during code-review: `TC-657` asserts the exact AC behavior + (empty nodes/edges, `parse_failed` in `errors`, `frontmatter_unparsable` error-severity + diagnostic; absent stays clean) against the real `extract_filament_core` path (no mocks). diff --git a/spec/functional/FR-006-frontmatter-with-fallback.md b/spec/functional/FR-006-frontmatter-with-fallback.md index 908aa3d..1340d76 100644 --- a/spec/functional/FR-006-frontmatter-with-fallback.md +++ b/spec/functional/FR-006-frontmatter-with-fallback.md @@ -20,6 +20,7 @@ relationships: 2. If `markdown` begins with `---\n` but no closing `---\n` line exists, return `FrontmatterResult { frontmatter: None, body: markdown.into() }`. 3. If a closing `---\n` exists but the content between the fences is not valid YAML, return `FrontmatterResult { frontmatter: None, body: markdown.into() }` — **the entire input is body**. This matches the TS/Py reference: malformed frontmatter is NOT an error; it is treated as content. 4. If the content is valid YAML, return `FrontmatterResult { frontmatter: Some(parsed), body: text_after_closing_fence }`. +5. **Frontmatter status (CR-010).** The result SHALL additionally carry a typed `status ∈ {Absent, Present, Malformed}` that reports *why* `frontmatter` is `None`, or that it parsed: branches 1–2 yield `Absent` (no block, or an unterminated fence); an invalid-YAML block (branch 3) or a valid-but-non-mapping value (array/scalar/bool/number) yields `Malformed`; an **empty / whitespace / comment-only** block (YAML null) yields `Absent` (it carries no metadata and is indistinguishable from having no frontmatter — e.g. a `---`…`---` pair of thematic breaks — so it is not a parse failure); and branch 4 yields `Present`. This is a parity-preserving extension: the `frontmatter` and `body` outputs are byte-identical across all branches (like the BOM/CRLF extensions), so the TS/Py reference comparison is unaffected. The status exists so a boundary consumer (e.g. the Filament extraction engine, [FR-040](./FR-040-filament-core-extraction-engine.md)) can distinguish an absent block from a malformed one without re-deriving it from the raw markdown. The parsed frontmatter is a `serde_json::Map` (JSON-compatible value tree), not a typed struct — typed deserialization is the consumer's responsibility. @@ -33,6 +34,7 @@ The parsed frontmatter is a `serde_json::Map` (JSON-compatible va | FR-006-AC-4 | [FR-001](./FR-001-render-dispatch.md) returns `frontmatter: None, body: full original input`. | Test | | FR-006-AC-5 | [FR-001](./FR-001-render-dispatch.md) (BOM-prefixed input) returns the same result as the BOM-free equivalent — frontmatter parsed, body equal to `"body"` (no BOM in body). | Test | | FR-006-AC-6 | `extract_frontmatter("\u{FEFF}# heading")` (BOM-prefixed, no frontmatter) returns `frontmatter: None, body: "# heading"` (BOM stripped from body). | Test | +| FR-006-AC-7 | The result `status` (CR-010) classifies each branch: absent/unterminated → `Absent`; a valid mapping → `Present`; invalid YAML or a non-null non-mapping value (array/scalar) → `Malformed`; an empty/whitespace/comment-only block (YAML null) → `Absent` (not a parse failure). | Test (TC-658) | ## Dependencies diff --git a/spec/functional/FR-040-filament-core-extraction-engine.md b/spec/functional/FR-040-filament-core-extraction-engine.md new file mode 100644 index 0000000..92c0ca6 --- /dev/null +++ b/spec/functional/FR-040-filament-core-extraction-engine.md @@ -0,0 +1,97 @@ +--- +id: FR-040 +title: "Canonical Filament core extraction engine" +type: FR +relationships: + - target: "ix://agent-ix/quire-rs/spec/usecase/US-015" + type: "implements" + - target: "ix://agent-ix/quire-rs/spec/functional/FR-011" + type: "requires" + - target: "ix://agent-ix/quire-rs/spec/functional/FR-026" + type: "requires" +--- +# [FR-040] Canonical Filament core extraction engine + +## Description + +The `quire-rs` library SHALL provide a Filament-facing extraction API that accepts one +markdown document plus caller-provided ObjectType snapshots and returns a JSON-serializable +core extraction result compatible with the shared `CoreExtractionResult` contract. + +## Inputs + +- Project id, document id, optional artifact id, relative path, repository name, and markdown text +- A caller-provided ObjectType snapshot containing name, schema, allowed links, + optional `body_extraction`, and plugin flag fields + +## Outputs + +- Object type records +- Graph node records +- Graph edge records +- Extraction diagnostics +- Extraction errors + +## Behavior + +- The engine SHALL parse frontmatter and markdown using the existing Rust parser. +- When a document has no usable frontmatter block (absent, an unterminated opening fence, + or an empty / whitespace / comment-only block), the engine SHALL return an empty + extraction result with a `no_frontmatter` informational diagnostic and no extraction error. +- When a document has a complete frontmatter fence block that does not parse into a YAML + mapping (invalid YAML, or a non-empty non-mapping value such as an array or scalar), the + engine SHALL return an empty extraction result with a `parse_failed` extraction error and + a `frontmatter_unparsable` error-severity diagnostic, so the failure reaches Filament's + per-document error index. +- When `frontmatter.object` names an unknown ObjectType, the engine SHALL return no node + for that object. +- When `frontmatter.object` names an unknown ObjectType, the engine SHALL emit an + `unknown_object_type` diagnostic. +- When the engine receives an ObjectType with `body_extraction` absent and `has_plugin` + false, the engine SHALL run Tier 1 frontmatter extraction. +- When an ObjectType has `body_extraction` and `has_plugin` is false, the engine SHALL + evaluate the existing Rust body-extraction DSL and validate each emitted record. +- When an ObjectType has `has_plugin` true, the engine SHALL surface an extraction error + or diagnostic and SHALL NOT execute Python plugin discovery. +- The engine SHALL harvest frontmatter relationship sugar and body `ix://` markdown links + into graph edges with stable provenance metadata. +- The engine SHALL deduplicate duplicate graph edges by source, edge type, and target, + preserving the first edge and emitting a diagnostic for duplicates. +- The engine SHALL produce stable SHA-256 ids for object type, node, and edge records. + +## Constraints + +| ID | Constraint | Type | Validation | +|----|------------|------|------------| +| FR-040-CON-1 | The extraction engine SHALL NOT read PGlite, Electron IPC, HTTP, auth, CloudManager sync, workspace watch queues, or embeddings. | Architecture | Inspection | +| FR-040-CON-2 | ObjectType snapshots SHALL be supplied by the caller. | Architecture | Test | +| FR-040-CON-3 | The engine SHALL NOT discover a runtime registry over the network or from service configuration. | Architecture | Test | +| FR-040-CON-4 | Non-`ix://` graph references SHALL normalize to `ix://agent-ix//` before record id generation. | Compatibility | Test | + +## Acceptance Criteria + +| ID | Criteria | Verification | +|----|----------|--------------| +| FR-040-AC-1 | A Tier 1 fixture emits one validated graph node whose `dataJson` includes frontmatter `id` as `code` and frontmatter `title` as `title`. | Test (TC-633) | +| FR-040-AC-2 | A Tier 2 fixture emits graph nodes and record-derived graph edges equivalent to the existing Rust DSL extraction result for the same ObjectType snapshot. | Test (TC-634) | +| FR-040-AC-3 | Unknown ObjectType, no-frontmatter, malformed `ix://`, duplicate-edge, and plugin-flag fixtures produce diagnostics or errors without panicking. | Test (TC-635) | +| FR-040-AC-4 | Frontmatter relationship sugar and body `ix://` links produce deterministic graph edges with source/original-target or original-uri metadata. | Test (TC-636) | +| FR-040-AC-5 | Repeating extraction with identical input produces byte-identical JSON output ordering and stable record ids. | Test (TC-637) | +| FR-040-AC-6 | A document with a complete but unparsable frontmatter fence block yields a non-empty `errors` entry (`parse_failed`) and a `frontmatter_unparsable` diagnostic, while a document with no frontmatter block stays clean (`no_frontmatter` diagnostic, empty `errors`). | Test (TC-657) | + +> **CR-010 note:** The original behavior ("frontmatter absent **or unparsable** → empty +> result with a `no_frontmatter` informational diagnostic") conflated two distinct cases +> and was corrected per issue #127: a *malformed* frontmatter block was silently treated +> as a clean extraction, so per-document parse failures never reached Filament's +> `index_errors` (the `MarkStale → apply_stale` path only records a document with a +> non-empty `errors` list). The engine now distinguishes **absent** (clean skip) from +> **present-but-unparsable** (extraction error). The absent-vs-malformed classification is +> owned by the parser — [FR-006](./FR-006-frontmatter-with-fallback.md) now exposes a typed +> frontmatter `status` — and this engine reads that status rather than re-deriving it, so +> there is a single source of truth. Parser FR-006 parity (frontmatter/body outputs) is +> unchanged; the malformed → error decision is Filament-boundary policy. + +## Dependencies + +- **Upstream**: [FR-011](./FR-011-body-extraction-dsl.md), [FR-026](./FR-026-intra-spec-reference-resolution.md), [FR-028](./FR-028-expanded-python-binding-surface.md) +- **Downstream**: [FR-041](./FR-041-filament-extraction-bindings.md), `filament-parser-lib` compatibility shim, and Filament IDE worker sync diff --git a/spec/functional/FR-041-filament-extraction-bindings.md b/spec/functional/FR-041-filament-extraction-bindings.md new file mode 100644 index 0000000..3c78c3a --- /dev/null +++ b/spec/functional/FR-041-filament-extraction-bindings.md @@ -0,0 +1,61 @@ +--- +id: FR-041 +title: "Expose Filament extraction through Python and WASM" +type: FR +relationships: + - target: "ix://agent-ix/quire-rs/spec/usecase/US-015" + type: "implements" + - target: "ix://agent-ix/quire-rs/spec/functional/FR-040" + type: "requires" + - target: "ix://agent-ix/quire-rs/spec/functional/FR-023" + type: "extends" +--- +# [FR-041] Expose Filament extraction through Python and WASM + +## Description + +The `quire-rs` distribution SHALL expose the canonical Filament extraction engine through +both the Python `quire` wheel and the `@agent-ix/quire-wasm` package, with both bindings +delegating to the same Rust implementation. + +## Inputs + +- The same JSON-serializable one-document extraction input defined by [FR-040](./FR-040-filament-core-extraction-engine.md) + +## Outputs + +- The same JSON-serializable `CoreExtractionResult`-compatible output defined by + [FR-040](./FR-040-filament-core-extraction-engine.md) + +## Behavior + +- The Python binding SHALL accept native Python dict/list/scalar values and return native + Python dict/list/scalar values without spawning a subprocess. +- The WASM binding SHALL accept JavaScript objects and return JavaScript objects with the + same field names and values as the Python binding for the same fixture input. +- The `@agent-ix/quire-wasm` package SHALL continue to export the existing + `parseDocument`, `extractFromBlob`, and `validateFromBlob` surfaces. +- Binding layers SHALL NOT contain independent extraction policy. +- Binding layers SHALL translate inputs and outputs only. + +## Constraints + +| ID | Constraint | Type | Validation | +|----|------------|------|------------| +| FR-041-CON-1 | The Python binding SHALL NOT use `subprocess`, sockets, or JSON-over-stdio for this extraction path. | Architecture | Inspection | +| FR-041-CON-2 | The WASM binding SHALL be loadable by Electron worker code without requiring a Python runtime or local sidecar process. | Architecture | Test | +| FR-041-CON-3 | TypeScript declarations for `@agent-ix/quire-wasm` SHALL include the new Filament extraction export. | Compatibility | Inspection | + +## Acceptance Criteria + +| ID | Criteria | Verification | +|----|----------|--------------| +| FR-041-AC-1 | Python and WASM bindings return equivalent JSON for Tier 1, Tier 2, relationship, malformed-link, and extraction-error fixtures. | Test (TC-638) | +| FR-041-AC-2 | `@agent-ix/quire-wasm` exports the new extraction API and updated `.d.ts` declarations while existing `parseDocument`, `extractFromBlob`, and `validateFromBlob` smoke tests continue to pass. | Test (TC-639) | +| FR-041-AC-3 | Static inspection finds no extraction-policy branches in binding code beyond input/output conversion and error mapping. | Inspection (TC-640) | +| FR-041-AC-4 | A default Rust build without the Python feature remains free of Python/CPython linkage, and a WASM target check succeeds with filesystem-free features. | Test (TC-641) | + +## Dependencies + +- **Upstream**: [FR-040](./FR-040-filament-core-extraction-engine.md), [FR-023](./FR-023-python-binding-surface.md) +- **Downstream**: `filament-parser-lib` [FR-118](ix://agent-ix/filament-parser-lib/FR-118) and Filament IDE [FR-041](ix://agent-ix/filament-ide/FR-041) diff --git a/spec/non-functional/NFR-020-filament-extraction-boundary.md b/spec/non-functional/NFR-020-filament-extraction-boundary.md new file mode 100644 index 0000000..87545a9 --- /dev/null +++ b/spec/non-functional/NFR-020-filament-extraction-boundary.md @@ -0,0 +1,55 @@ +--- +id: NFR-020 +title: "Filament extraction boundary remains pure and deterministic" +type: NFR +quality_attribute: maintainability +relationships: + - target: "ix://agent-ix/quire-rs/spec/functional/FR-040" + type: "constrains" + - target: "ix://agent-ix/quire-rs/spec/functional/FR-041" + type: "constrains" +--- +# [NFR-020] Filament extraction boundary remains pure and deterministic + +## Statement + +The Filament extraction path SHALL remain a pure document-semantics boundary whose +outputs are deterministic for identical markdown and ObjectType snapshot inputs. + +## Scope + +- Applies to: the canonical Filament extraction module and its Python/WASM bindings. +- Operational context: IDE worker sync, parser-lib compatibility calls, and future sync + consumers. + +## Rationale + +The migration removes duplicated policy only if all consumers can trust one engine to +produce identical results without hidden runtime dependencies or side effects. + +## Measurement and Evaluation + +| Metric | Target | Threshold | Method | +|--------|--------|-----------|--------| +| Runtime side-effect dependencies in extraction module | 0 | 0 | Static Inspection | +| Repeated-output mismatch count for parity fixtures | 0 | 0 | Unit Test | +| Python/WASM binding divergence count for shared fixtures | 0 | 0 | Integration Test | + +## Verification + +Static inspection and parity tests verify that extraction does not reach into persistence, +IPC, network, CloudManager sync, or embedding concerns, and that repeated runs produce +the same JSON output for the same inputs. + +## Acceptance Criteria + +| ID | Criteria | Verification | +|----|----------|--------------| +| NFR-020-AC-1 | Static inspection finds no PGlite, Electron, HTTP/auth, CloudManager sync, workspace watcher, or embedding dependencies in the extraction module or bindings. | Inspection (TC-642) | +| NFR-020-AC-2 | A deterministic fixture suite runs the same extraction input repeatedly and observes identical output ordering and record ids. | Test (TC-637) | +| NFR-020-AC-3 | Binding parity fixtures compare Python and WASM output as JSON values, not stringified subprocess payloads. | Test (TC-638) | + +## Dependencies + +- **Upstream**: [FR-040](../functional/FR-040-filament-core-extraction-engine.md), [FR-041](../functional/FR-041-filament-extraction-bindings.md) +- **Downstream**: parser-lib and IDE cutover tasks diff --git a/spec/tests.md b/spec/tests.md index 8e9c4e2..9d3efa5 100644 --- a/spec/tests.md +++ b/spec/tests.md @@ -54,6 +54,7 @@ The spec was revised after authoring to reflect the **archetype-as-data** model: | US-012 Agent audits whole spec | AC-1..5 + PC-1..3 | TC-493, TC-495, TC-494, TC-496, TC-485 (correctness) + TC-457, TC-458, TC-498 (perf) | ✅ Functional / 🚧 Perf bench pending | | US-013 Agent resolves intra-spec refs | AC-1..5 + PC-1..3 | TC-486, TC-487, TC-488, TC-489, TC-490 (correctness) + TC-459, TC-492 (perf) | ✅ Functional / 🚧 Perf bench pending | | US-014 Author validates markdown | AC-1..4 | TC-518, TC-519, TC-520, TC-521 | 🚧 Pending implementation | +| US-015 Consume canonical Filament extraction | Illustrative examples | TC-633..TC-642 | ✅ Complete | ### Functional Requirement Coverage @@ -64,7 +65,7 @@ The spec was revised after authoring to reflect the **archetype-as-data** model: | FR-003 schema_for surfaces fs schema | AC-1..4 | TC-009, TC-009b (unknown), TC-061, TC-062 (no schemars dep) | ✅ Complete | | ~~FR-004 Strict MiniJinja env~~ | — | — | ⛔ RETIRED (render removal) | | FR-005 parse_document API | AC-1..4 | TC-001, TC-029 | ✅ Complete | -| FR-006 Frontmatter fallback | AC-1..4 | TC-012, TC-013, TC-014 | ✅ Complete | +| FR-006 Frontmatter fallback | AC-1..4, AC-7 | TC-012, TC-013, TC-014, TC-658 | ✅ Complete | | FR-007 Fenced-block walk | AC-1..4 | TC-015, TC-016, TC-017 | ✅ Complete | | FR-008 Byte-exact slicing | AC-1..3 | TC-022, TC-023, TC-024 | ✅ Complete | | FR-009 Slug-line ID | AC-1..5 | TC-025, TC-026 | ✅ Complete | @@ -95,6 +96,8 @@ The spec was revised after authoring to reflect the **archetype-as-data** model: | FR-026 Intra-spec reference resolution | AC-1..11 | TC-486 (frontmatter edge), TC-487 (ix:// edge), TC-488 (dangling), TC-489 (cross-spec dangling), TC-490 (bidirectional), TC-491 (target-id extraction), TC-492 (O(edges) proptest), TC-501 (dedup), TC-620 (rel-path edge/dangling), TC-621 (index/log excluded), TC-622 (dedup parity across sources) | ✅ Complete | | FR-027 Whole-spec query API | AC-1..8 | TC-493 (by_type), TC-494 (referencing), TC-495 (orphans), TC-496 (coverage), TC-497 (dangling agreement), TC-498 (sorted determinism), TC-499 (no-IO), TC-458 (bench) | ✅ Complete | | FR-039 Unlinked reference detection (ADR 0007) | AC-1..10 | TC-623 (auto-fix bare id), TC-624 (sub-id→parent file), TC-625 (inline-code conversion), TC-626 (fenced/frontmatter ignored), TC-627 (already-linked + idempotence), TC-628 (self-reference), TC-629 (unresolved→warn-only), TC-630 (ambiguous→warn-only), TC-631 (sorted determinism), TC-632 (multi-token code span skipped) | 🚧 Pending implementation | +| FR-040 Canonical Filament core extraction engine | AC-1..6; CON-1..4 | TC-643..TC-656, TC-642, TC-657 | ✅ Complete | +| FR-041 Filament extraction bindings | AC-1..4; CON-1..3 | TC-638, TC-639, TC-640, TC-641, IC-001 | ✅ Complete | ### Non-Functional Requirement Coverage @@ -112,6 +115,7 @@ The spec was revised after authoring to reflect the **archetype-as-data** model: | NFR-016 Binding overhead + abi3 | micro-bench + cross-version import | TC-469, TC-464, TC-465, TC-467 | ✅ Complete | | NFR-017 Concurrency permutation (loom) | loom exhaustive interleaving (scheduled lane) | TC-502, TC-503 | ✅ Complete | | NFR-018 FFI sanitizer lanes (TSAN+ASAN) | scheduled sanitizer lanes on the extension | TC-504, TC-505 | ✅ Complete | +| NFR-020 Filament extraction boundary pure/deterministic | static inspection + parity tests | TC-656, IC-001, TC-642 | ✅ Complete | --- @@ -319,6 +323,35 @@ The spec was revised after authoring to reflect the **archetype-as-data** model: | TC-630 | Token whose parent id maps to >1 loaded doc → `WarnOnly { Ambiguous }`, no `suggested_link` | Unit | P0 | FR-039-AC-8 | 🚧 | | TC-631 | `unlinked_references` results sorted by `(path, byte_span.start)`; identical across runs and thread counts | Property | P0 | FR-039-AC-9, NFR-006 | 🚧 | | TC-632 | A code span with >1 artifact token (`` `FR-008/FR-009` ``) yields no finding; a single-token code span still converts | Unit | P0 | FR-039-AC-10 | 🚧 | +| TC-633 | Filament Tier 1 fixture emits one validated graph node with frontmatter `id` as `code` and frontmatter `title` as `title` | Unit | P0 | FR-040-AC-1 | ✅ | +| TC-634 | Filament Tier 2 fixture emits graph nodes and record-derived edges equivalent to the Rust DSL extractor for the same ObjectType snapshot | Unit | P0 | FR-040-AC-2 | ✅ | +| TC-635 | Unknown ObjectType, no-frontmatter, malformed `ix://`, duplicate-edge, and plugin-flag fixtures produce diagnostics/errors without panic | Unit | P0 | FR-040-AC-3 | ✅ | +| TC-636 | Relationship sugar and body `ix://` links produce deterministic graph edges with provenance metadata | Unit | P0 | FR-040-AC-4 | ✅ | +| TC-637 | Repeated Filament extraction over identical inputs produces byte-identical JSON ordering and stable ids | Property | P0 | FR-040-AC-5, NFR-020-AC-2 | ✅ | +| TC-638 | Python and WASM Filament extraction bindings return equivalent JSON values for shared parity fixtures | Integration | P0 | FR-041-AC-1, NFR-020-AC-3 | ✅ | +| TC-639 | `@agent-ix/quire-wasm` exports the Filament extraction API and preserves existing parse/extract/validate smoke tests and declarations | Integration | P0 | FR-041-AC-2 | ✅ | +| TC-640 | Binding code contains no extraction-policy branches beyond input/output conversion and error mapping | Static | P0 | FR-041-AC-3 | ✅ | +| TC-641 | Default Rust build has no Python linkage and WASM target check succeeds with filesystem-free features | Compile | P0 | FR-041-AC-4 | ✅ | +| TC-642 | Static audit finds no PGlite, Electron, HTTP/auth, CloudManager sync, watcher, or embedding dependencies in extraction module/bindings | Static | P0 | FR-040-CON-1, FR-040-CON-2, FR-040-CON-3, FR-040-CON-4, FR-041-CON-1, FR-041-CON-2, FR-041-CON-3, NFR-020-AC-1 | ✅ | +| TC-657 | Malformed-frontmatter fixture (complete fence, unparsable YAML) yields a `parse_failed` error + `frontmatter_unparsable` diagnostic; absent frontmatter stays clean (`no_frontmatter`, empty errors) | Unit | P0 | FR-040-AC-6 | ✅ | +| TC-658 | `extract_frontmatter` status classification (CR-010): empty/whitespace/comment-only block (YAML null) → `Absent` (not `Malformed`); non-null non-mapping (array/scalar) → `Malformed` | Unit | P0 | FR-006-AC-7 | ✅ | +| TC-643 | Shared fixture: Tier 1 frontmatter node shape, stable normalized ref, `code`, `title`, and extra frontmatter data | Fixture | P0 | FR-040-AC-1 | ✅ | +| TC-644 | Shared fixture: Tier 2 DSL record extraction, schema validation, and record-derived edge emission | Fixture | P0 | FR-040-AC-2 | ✅ | +| TC-645 | Shared fixture: explicit frontmatter `edges:` preserve order, target refs, source refs, and metadata | Fixture | P0 | FR-040-AC-4 | ✅ | +| TC-646 | Shared fixture: relationship sugar fields emit expected edge types, normalized refs, and provenance metadata | Fixture | P0 | FR-040-AC-4 | ✅ | +| TC-647 | Shared fixture: `ix://` targets pass through and bare targets normalize to `ix://agent-ix//` | Fixture | P0 | FR-040-CON-4 | ✅ | +| TC-648 | Shared fixture: body markdown `ix://` links emit `references` graph edges and ignore external web links | Fixture | P0 | FR-040-AC-4 | ✅ | +| TC-649 | Shared fixture: duplicate graph edges dedupe by source/type/target, first edge wins, diagnostic emitted | Fixture | P0 | FR-040-AC-3 | ✅ | +| TC-650 | Shared fixtures: malformed `relationships:` entries report errors for non-map, missing target, and missing type | Fixture | P0 | FR-040-AC-3 | ✅ | +| TC-651 | Shared fixture: malformed explicit `edges:` entries report an extraction error without panic | Fixture | P0 | FR-040-AC-3 | ✅ | +| TC-652 | Shared fixture: schema validation failure prevents node emission and surfaces a field-specific error | Fixture | P0 | FR-040-AC-2, FR-040-AC-3 | ✅ | +| TC-653 | Shared fixtures: unknown object, no frontmatter, unsupported plugin flag, and malformed body `ix://` produce diagnostics/errors without panic | Fixture | P0 | FR-040-AC-3 | ✅ | +| TC-654 | Shared fixture: negative scope rejects wikilinks, `spec://`, prose cues, and `https://` as graph edges | Fixture | P0 | FR-040-AC-4 | ✅ | +| TC-655 | Shared corpus isolation: failing extraction fixtures do not poison later successful fixture extraction | Fixture | P0 | FR-040-AC-3 | ✅ | +| TC-656 | Shared corpus determinism: every canonical graph fixture produces byte-identical JSON over repeated extraction | Property | P0 | FR-040-AC-5, NFR-020-AC-2 | ✅ | +| IC-001 | Python and WASM bindings return equivalent JSON over every canonical graph fixture | Integration | P0 | FR-041-AC-1, NFR-020-AC-3 | ✅ | +| IC-002 | parser-lib shim returns core-data-valid payloads matching canonical graph fixture expectations | Integration | P0 | FR-118 compatibility reference | ✅ | +| IC-003 | Filament IDE worker merges a real quire-wasm canonical graph fixture into `CoreSyncFilePayload` | Integration | P0 | Filament IDE FR-041 reference | ✅ | | TC-502 | Static audit: no Mutex/RwLock/Atomic in first-party src/; parallel parse collects owned results | Static | P0 | FR-024-AC-9 | 🚧 | | TC-503 | loom: parallel parse collection race-free; identical path-sorted output across all interleavings | Property | P0 | NFR-017-AC-1..3 | 🚧 | | TC-504 | TSAN lane: two-thread `load_repo` (GIL-release window) reports zero data races | Integration | P0 | NFR-018-AC-1, NFR-018-AC-3 | 🚧 | @@ -600,6 +633,7 @@ Comprehensive, post-audit explicit mapping. Every AC defined in the spec is list | FR-006-AC-4 | TC-014 | | FR-006-AC-5 | TC-180 | | FR-006-AC-6 | TC-181 | +| FR-006-AC-7 | TC-658 | | FR-007-AC-1 | TC-015 | | FR-007-AC-2 | TC-016 | | FR-007-AC-3 | TC-017 | @@ -805,6 +839,16 @@ Comprehensive, post-audit explicit mapping. Every AC defined in the spec is list | FR-039-AC-8 | TC-630 | | FR-039-AC-9 | TC-631 | | FR-039-AC-10 | TC-632 | +| FR-040-AC-1 | TC-633, TC-643 | +| FR-040-AC-2 | TC-634, TC-644, TC-652 | +| FR-040-AC-3 | TC-635, TC-649, TC-650, TC-651, TC-652, TC-653, TC-655 | +| FR-040-AC-4 | TC-636, TC-645, TC-646, TC-648, TC-654 | +| FR-040-AC-5 | TC-637, TC-656 | +| FR-040-AC-6 | TC-657 | +| FR-041-AC-1 | TC-638, IC-001 | +| FR-041-AC-2 | TC-639 | +| FR-041-AC-3 | TC-640 | +| FR-041-AC-4 | TC-641 | ### Non-Functional Requirements @@ -869,8 +913,11 @@ Comprehensive, post-audit explicit mapping. Every AC defined in the spec is list | NFR-018-AC-4 | (process AC; covered by P0-reproducer policy, parity with NFR-011-AC-4) | | NFR-019-AC-1 | TC-579 | | NFR-019-AC-2 | TC-580 | +| NFR-020-AC-1 | TC-642 | +| NFR-020-AC-2 | TC-637, TC-656 | +| NFR-020-AC-3 | TC-638, IC-001 | -**Coverage status: 327 / 327 ACs covered (100%).** The internal-links slice (ADR 0007, 2026-06-17) adds FR-026-AC-9..11 (relative-path link edge source + index/log exclusion + dedup parity, TC-620..622) and FR-039-AC-1..10 (unlinked-reference detection & autofix suggestions, incl. AC-10 multi-token code-span skip, TC-623..632) — 13 ACs. The composed type+object validation slice (2026-06-16) adds FR-032-AC-11..13 (`validate_document_in_registry` composes the `type` archetype with the frontmatter `object:` archetype; resolved-object failures are errors, unknown-object is a warning, `ValidationResult` carries typed `warnings`) — TC-610..613, 3 ACs. The assert/lint extension slice (2026-06-16) adds FR-033-AC-10 (CR-008 `matches` content assert, TC-608) and FR-036-AC-6 (CR-009 `section_body_pattern` lint rule, TC-609) — 2 ACs. The OKF slice (2026-06-16) adds FR-037-AC-1..6 (base concept frontmatter schema, TC-590..596 + TC-528) and FR-038-AC-1..8 (OKF bundle validation, TC-600..607) — 14 ACs. v0.4 adds FR-011-AC-21 (CR-006 `multiple: true`, TC-583) and FR-036-AC-1..5 (declarative lint rules, TC-584..588). v0.2 block model added 16 ACs (FR-019..022, TC-400..443). v0.3 adds 81 ACs — StR-005/006, US-011..013, FR-023..027 (incl. review-added FR-026-AC-8, FR-027-AC-9), NFR-015/016, plus the hardening re-review (NFR-003-AC-4, FR-024-AC-9, NFR-017, NFR-018) — covered by TC-455..507 (plus reused TC-456..459). The Miri ACs (NFR-012-AC-1..5) were **retired** (ADR 0006) and the compile-time **NFR-003-AC-5** (`forbid(unsafe_code)`, TC-582) added. PC (performance criteria) for US-011..013 are tracked as benches (TC-455..459, TC-469) and marked 🚧 pending implementation, consistent with the US-006..010 perf-bench convention. The v0.3 hardening re-review (loom NFR-017, TSAN/ASAN NFR-018) is recorded in spec.md §19. +**Coverage status: 341 / 341 ACs covered (100%).** The Filament extraction unification slice (2026-06-29) adds FR-040-AC-1..5, FR-041-AC-1..4, and NFR-020-AC-1..3, covered by TC-633..642 and hardened by the shared graph fixture corpus TC-643..656 plus integration cases IC-001..IC-003. The #127 malformed-frontmatter correction (CR-010) adds FR-040-AC-6 (TC-657) and FR-006-AC-7 (TC-658, the parser frontmatter-status classification). The internal-links slice (ADR 0007, 2026-06-17) adds FR-026-AC-9..11 (relative-path link edge source + index/log exclusion + dedup parity, TC-620..622) and FR-039-AC-1..10 (unlinked-reference detection & autofix suggestions, incl. AC-10 multi-token code-span skip, TC-623..632) — 13 ACs. The composed type+object validation slice (2026-06-16) adds FR-032-AC-11..13 (`validate_document_in_registry` composes the `type` archetype with the frontmatter `object:` archetype; resolved-object failures are errors, unknown-object is a warning, `ValidationResult` carries typed `warnings`) — TC-610..613, 3 ACs. The assert/lint extension slice (2026-06-16) adds FR-033-AC-10 (CR-008 `matches` content assert, TC-608) and FR-036-AC-6 (CR-009 `section_body_pattern` lint rule, TC-609) — 2 ACs. The OKF slice (2026-06-16) adds FR-037-AC-1..6 (base concept frontmatter schema, TC-590..596 + TC-528) and FR-038-AC-1..8 (OKF bundle validation, TC-600..607) — 14 ACs. v0.4 adds FR-011-AC-21 (CR-006 `multiple: true`, TC-583) and FR-036-AC-1..5 (declarative lint rules, TC-584..588). v0.2 block model added 16 ACs (FR-019..022, TC-400..443). v0.3 adds 81 ACs — StR-005/006, US-011..013, FR-023..027 (incl. review-added FR-026-AC-8, FR-027-AC-9), NFR-015/016, plus the hardening re-review (NFR-003-AC-4, FR-024-AC-9, NFR-017, NFR-018) — covered by TC-455..507 (plus reused TC-456..459). The Miri ACs (NFR-012-AC-1..5) were **retired** (ADR 0006) and the compile-time **NFR-003-AC-5** (`forbid(unsafe_code)`, TC-582) added. PC (performance criteria) for US-011..013 are tracked as benches (TC-455..459, TC-469) and marked 🚧 pending implementation, consistent with the US-006..010 perf-bench convention. The v0.3 hardening re-review (loom NFR-017, TSAN/ASAN NFR-018) is recorded in spec.md §19. **v0.4 markdown-validation slice** adds 42 ACs — US-014 (author validates markdown), FR-029 (archetype input contract, recast by ADR 0004), FR-030 (required-section validation, superseded by FR-032/FR-033), FR-031 (unified archetype shape), FR-032 (`validate_document`), FR-033 (locator `assert` facet), FR-034 (assert field interpolation), FR-035 (per-level heading uniqueness) — covered by TC-518..553. FR-030's ACs are mapped to the FR-032/FR-033 TCs that subsume them (per its CR note). This slice also back-fills 7 ACs that a prior commit left out of the audit table — FR-013-AC-11..14, FR-028-AC-9/10, US-003-AC-4 — via TC-554..560. New v0.4 TCs are 🚧 pending implementation. @@ -888,7 +935,7 @@ NFR-006-AC-4, NFR-019-AC-1..2 — covered by TC-565..580. TC-561 is re-pointed o FR-033-AC-4 onto FR-033-AC-9 (the non-table `id_pattern` case); TC-562 covers both FR-033-AC-4 and FR-033-AC-9. -**Integrity check (grep-verified):** all **327 distinct file-defined ACs** (definition-anchored: bold `**-AC-N**` declarations) across `stakeholder/ usecase/ functional/ non-functional/` appear in the AC→TC audit table — **0 uncovered**. Note: `FR-900-AC-1/2` appearing inside FR-034-AC-1's example prose are NOT defined ACs and are excluded from the denominator (match `**…**:` definitions, not inline mentions). Retired ACs (marked `(RETIRED)`, un-bolded) are excluded by construction. Count: 316 (pre-removal) − 41 (retired) + 16 (back-fill) + 1 (FR-011-AC-20, CR-005 heading normalization) − 5 (NFR-012-AC-1..5 retired, ADR 0006) + 1 (NFR-003-AC-5, forbid(unsafe_code)) + 1 (FR-011-AC-21, CR-006 multiple:true) + 5 (FR-036-AC-1..5, declarative lint rules) + 1 (FR-010-AC-4, CR-007 escaped pipes) + 6 (FR-037-AC-1..6, OKF base concept schema) + 8 (FR-038-AC-1..8, OKF bundle validation) + 1 (FR-033-AC-10, CR-008 `matches` content assert) + 1 (FR-036-AC-6, CR-009 `section_body_pattern`) + 3 (FR-032-AC-11..13, composed type+object validation) + 3 (FR-026-AC-9..11, relative-path link edge source) + 10 (FR-039-AC-1..10, unlinked-reference detection incl. multi-token code-span skip) = **327**. +**Integrity check (grep-verified):** all **341 distinct file-defined ACs** across `stakeholder/ usecase/ functional/ non-functional/` appear in the AC→TC audit table — **0 uncovered**. Note: `FR-900-AC-1/2` appearing inside FR-034-AC-1's example prose are NOT defined ACs and are excluded from the denominator. Retired ACs are excluded by construction. Count: prior 327 + 12 Filament extraction unification ACs (FR-040-AC-1..5, FR-041-AC-1..4, NFR-020-AC-1..3) + 2 malformed-frontmatter correction ACs (FR-040-AC-6 + FR-006-AC-7, CR-010) = **341**. --- diff --git a/spec/usecase/US-015-canonical-filament-extraction.md b/spec/usecase/US-015-canonical-filament-extraction.md new file mode 100644 index 0000000..feb79c5 --- /dev/null +++ b/spec/usecase/US-015-canonical-filament-extraction.md @@ -0,0 +1,55 @@ +--- +id: US-015 +title: "Consume canonical Filament extraction from one engine" +type: US +relationships: + - target: "ix://agent-ix/quire-rs/spec/stakeholder/StR-001" + type: "traces_to" + - target: "ix://agent-ix/quire-rs/spec/stakeholder/StR-005" + type: "traces_to" +--- +# [US-015] Consume canonical Filament extraction from one engine + +## Story + +**As a** Filament sync or IDE runtime maintainer +**I want** one canonical engine to extract document objects, graph edges, and diagnostics +**So that** Python, WASM, and Electron consumers produce the same core data without sidecars or duplicated policy. + +## Context + +Filament extraction policy currently spans `filament-parser-lib`, the `quire` Python +binding, and the published `@agent-ix/quire-wasm` package. The cutover is valuable only +if the canonical policy lives in `quire-rs` and the bindings are thin wrappers over the +same Rust implementation. + +## Acceptance Examples (Illustrative) + +### [US-015-EX-1] IDE worker extracts without Python + +- **Given** an IDE worker receives markdown plus ObjectType snapshots +- **When** it extracts core data +- **Then** the worker calls the `quire-wasm` binding in-process and receives the same graph/object payload shape as Python consumers + +### [US-015-EX-2] Parser library delegates without drift + +- **Given** `filament-parser-lib` receives the same markdown and ObjectType snapshots +- **When** it calls the Python `quire` binding +- **Then** the returned core extraction payload matches the WASM binding for the same fixture + +## Dependencies (Contextual) + +Upstream requirements include [FR-011](../functional/FR-011-body-extraction-dsl.md), +[FR-026](../functional/FR-026-intra-spec-reference-resolution.md), and the Python binding +surface in [FR-028](../functional/FR-028-expanded-python-binding-surface.md). + +## Priority and Risk (Informative) + +Priority is high because the IDE Python bridge and parser-lib compatibility shim depend +on this behavior. Risk is high if unmet because duplicated extraction behavior will +continue to drift across runtimes. + +## Traceability (Informative) + +This story drives [FR-040](../functional/FR-040-filament-core-extraction-engine.md) and +[FR-041](../functional/FR-041-filament-extraction-bindings.md). diff --git a/src/filament.rs b/src/filament.rs new file mode 100644 index 0000000..a0e19a6 --- /dev/null +++ b/src/filament.rs @@ -0,0 +1,1127 @@ +//! Canonical one-document Filament extraction (FR-040). +//! +//! This module is intentionally a pure boundary: callers provide the markdown +//! document and ObjectType snapshots, and the engine returns core-data-shaped +//! JSON records. It does not load registries, perform I/O, or reach into any +//! Filament runtime service. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::error::QuireError; +use crate::extract::dsl::{validate_dsl, ExtractionDsl}; +use crate::loader::compile::compile_schema; +use crate::parser::FrontmatterStatus; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FilamentExtractionInput { + #[serde(rename = "project_id", alias = "projectId")] + pub project_id: String, + #[serde(rename = "document_id", alias = "documentId")] + pub document_id: String, + #[serde(rename = "artifact_id", alias = "artifactId", default)] + pub artifact_id: Option, + #[serde(rename = "rel_path", alias = "relPath")] + pub rel_path: String, + pub markdown: String, + #[serde(rename = "repo_name", alias = "repoName")] + pub repo_name: String, + #[serde(rename = "object_types", alias = "objectTypes", default)] + pub object_types: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FilamentObjectType { + pub name: String, + #[serde(default)] + pub schema: Option, + #[serde(default)] + pub data_schema: Option, + #[serde(rename = "allowed_links", alias = "allowedLinks", default)] + pub allowed_links: Option, + #[serde(rename = "body_extraction", alias = "bodyExtraction", default)] + pub body_extraction: Option, + #[serde(rename = "has_plugin", alias = "hasPlugin", default)] + pub has_plugin: bool, + #[serde(rename = "module_id", alias = "moduleId", default)] + pub module_id: Option, +} + +impl FilamentObjectType { + fn data_schema(&self) -> Value { + self.data_schema + .clone() + .or_else(|| self.schema.clone()) + .unwrap_or_else(|| json!({"type": "object"})) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoreExtractionResult { + #[serde(rename = "documentId")] + pub document_id: String, + #[serde(rename = "artifactId")] + pub artifact_id: Option, + #[serde(rename = "objectTypes")] + pub object_types: Vec, + pub nodes: Vec, + pub edges: Vec, + pub diagnostics: Vec, + pub errors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoreObjectTypeRecord { + pub id: String, + #[serde(rename = "projectId")] + pub project_id: String, + pub name: String, + #[serde(rename = "schemaJson")] + pub schema_json: String, + #[serde(rename = "allowedLinksJson")] + pub allowed_links_json: String, + #[serde(rename = "bodyExtractionJson")] + pub body_extraction_json: Option, + #[serde(rename = "hasPlugin")] + pub has_plugin: bool, + #[serde(rename = "moduleId")] + pub module_id: Option, + #[serde(rename = "updatedAt")] + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoreGraphNodeRef { + pub id: String, + #[serde(rename = "projectId")] + pub project_id: String, + #[serde(rename = "documentId")] + pub document_id: String, + #[serde(rename = "artifactId")] + pub artifact_id: Option, + #[serde(rename = "objectType")] + pub object_type: String, + pub name: String, + #[serde(rename = "ref")] + pub ref_: String, + #[serde(rename = "dataJson")] + pub data_json: String, + #[serde(rename = "updatedAt")] + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoreGraphEdgeRef { + pub id: String, + #[serde(rename = "projectId")] + pub project_id: String, + #[serde(rename = "sourceRef")] + pub source_ref: String, + #[serde(rename = "targetRef")] + pub target_ref: String, + #[serde(rename = "edgeType")] + pub edge_type: String, + #[serde(rename = "dataJson")] + pub data_json: String, + #[serde(rename = "updatedAt")] + pub updated_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoreExtractionDiagnostic { + pub code: String, + pub message: String, + pub severity: String, + #[serde(rename = "objectType")] + pub object_type: Option, +} + +#[derive(Debug, Clone, PartialEq)] +struct CompiledObjectType { + snapshot: FilamentObjectType, + schema: Value, + body_extraction: Option, +} + +#[derive(Debug, Clone, PartialEq)] +struct GraphNode { + object_type: String, + name: String, + data: Map, +} + +#[derive(Debug, Clone, PartialEq)] +struct GraphEdge { + source_ref: String, + edge_type: String, + target_ref: String, + metadata: Map, +} + +pub fn extract_filament_core(input: FilamentExtractionInput) -> CoreExtractionResult { + let mut diagnostics = Vec::new(); + let mut errors = Vec::new(); + + let doc = crate::parse_document(&input.markdown); + let Some(frontmatter) = doc.frontmatter.clone() else { + // Frontmatter is absent. Ask the parser authority *why* — a document + // with no frontmatter block is a clean skip, but a complete fence + // block that failed to parse is a malformed document that must reach + // Filament's index_errors via CoreExtractionResult.errors (issue #127, + // CR-010). This re-parse only runs on the no-frontmatter branch, so + // the happy path is unaffected. + return match crate::extract_frontmatter(&input.markdown).status { + FrontmatterStatus::Malformed => malformed_frontmatter_result(input), + // Absent (or the unreachable Present, which would have yielded + // Some above): a genuinely non-Filament document, skipped cleanly. + FrontmatterStatus::Absent | FrontmatterStatus::Present => { + absent_frontmatter_result(input) + } + }; + }; + + let compiled = compile_object_type_snapshots(&input.object_types, &mut diagnostics); + let object_type_records = object_type_records(&input.project_id, &input.object_types); + let object_name = frontmatter + .get("object") + .and_then(Value::as_str) + .map(str::to_string); + + let mut nodes = Vec::new(); + let mut tier_edges = Vec::new(); + + if let Some(object_name) = object_name { + match compiled.get(&object_name) { + None => diagnostics.push(diagnostic( + "unknown_object_type", + format!("frontmatter object={object_name:?} not in supplied object_types"), + "warning", + Some(object_name), + )), + Some(object_type) if object_type.snapshot.has_plugin => { + if object_type.body_extraction.is_some() { + diagnostics.push(diagnostic( + "tier3_overrides_tier2", + "ObjectType has both body_extraction and has_plugin=true; plugin tier is unsupported in quire-rs core extraction", + "warning", + Some(object_name.clone()), + )); + } + errors.push(format!( + "missing_plugin: object type {object_name} declares has_plugin=true; quire-rs core extraction does not execute Python plugins" + )); + } + Some(object_type) => { + if let Some(dsl) = object_type.body_extraction.as_ref() { + match extract_tier2(&doc, &frontmatter, object_type, dsl, &input.rel_path) { + Ok((mut extracted_nodes, mut extracted_edges, mut ds)) => { + nodes.append(&mut extracted_nodes); + tier_edges.append(&mut extracted_edges); + diagnostics.append(&mut ds); + } + Err(err) => errors.push(format!("tier2 extraction failed: {err}")), + } + } else { + match extract_tier1(&frontmatter, object_type, &input.rel_path) { + Ok(node) => nodes.push(node), + Err(err) => errors.push(format!("tier1 extraction failed: {err}")), + } + } + } + } + } + + let primary_ref = nodes + .first() + .map(|n| n.name.clone()) + .unwrap_or_else(|| format!("document:{}", input.document_id)); + let mut raw_edges = Vec::new(); + match frontmatter_edges( + &frontmatter, + &primary_ref, + &input.repo_name, + &mut diagnostics, + ) { + Ok(mut edges) => raw_edges.append(&mut edges), + Err(err) => errors.push(err), + } + raw_edges.append(&mut body_ix_edges( + &input.markdown, + &primary_ref, + &mut diagnostics, + )); + raw_edges.append(&mut tier_edges); + + let raw_edges = dedupe_edges(raw_edges, &input.repo_name, &mut diagnostics); + let nodes = node_records(&input, nodes); + let edges = edge_records(&input, raw_edges); + + CoreExtractionResult { + document_id: input.document_id, + artifact_id: input.artifact_id, + object_types: object_type_records, + nodes, + edges, + diagnostics, + errors, + } +} + +/// Result for a document with no frontmatter block: a clean, non-Filament +/// document. Emits only an informational `no_frontmatter` diagnostic; `errors` +/// stays empty so nothing is recorded as a parse failure downstream. +fn absent_frontmatter_result(input: FilamentExtractionInput) -> CoreExtractionResult { + CoreExtractionResult { + document_id: input.document_id, + artifact_id: input.artifact_id, + object_types: Vec::new(), + nodes: Vec::new(), + edges: Vec::new(), + diagnostics: vec![diagnostic( + "no_frontmatter", + "Shared parser skipped markdown without parsable frontmatter", + "info", + None, + )], + errors: Vec::new(), + } +} + +/// Result for a document with a complete frontmatter fence block that failed +/// to parse into a YAML mapping. Records a `parse_failed` extraction error so +/// the failure reaches Filament's index_errors via the MarkStale→apply_stale +/// path (issue #127, CR-010), alongside a matching error-severity diagnostic. +fn malformed_frontmatter_result(input: FilamentExtractionInput) -> CoreExtractionResult { + CoreExtractionResult { + document_id: input.document_id, + artifact_id: input.artifact_id, + object_types: Vec::new(), + nodes: Vec::new(), + edges: Vec::new(), + diagnostics: vec![diagnostic( + "frontmatter_unparsable", + "Frontmatter block present but could not be parsed as a YAML mapping", + "error", + None, + )], + errors: vec![ + "parse_failed: frontmatter block present but could not be parsed as a YAML mapping" + .to_string(), + ], + } +} + +fn compile_object_type_snapshots( + object_types: &[FilamentObjectType], + diagnostics: &mut Vec, +) -> BTreeMap { + let mut out = BTreeMap::new(); + for snapshot in object_types { + if out.contains_key(&snapshot.name) { + diagnostics.push(diagnostic( + "duplicate_object_type", + format!( + "duplicate object type {}; first definition wins", + snapshot.name + ), + "warning", + Some(snapshot.name.clone()), + )); + continue; + } + let body_extraction = match snapshot.body_extraction.clone() { + None | Some(Value::Null) => None, + Some(value) => match serde_json::from_value::(value) { + Ok(dsl) => match validate_dsl(&snapshot.name, &dsl) { + Ok(()) => Some(dsl), + Err(err) => { + diagnostics.push(diagnostic( + "body_extraction_invalid", + err.to_string(), + "error", + Some(snapshot.name.clone()), + )); + None + } + }, + Err(err) => { + diagnostics.push(diagnostic( + "body_extraction_invalid", + format!("{}: body_extraction parse failed: {err}", snapshot.name), + "error", + Some(snapshot.name.clone()), + )); + None + } + }, + }; + out.insert( + snapshot.name.clone(), + CompiledObjectType { + snapshot: snapshot.clone(), + schema: snapshot.data_schema(), + body_extraction, + }, + ); + } + out +} + +fn object_type_records( + project_id: &str, + object_types: &[FilamentObjectType], +) -> Vec { + object_types + .iter() + .map(|object_type| CoreObjectTypeRecord { + id: stable_id(&[project_id, "object-type", &object_type.name]), + project_id: project_id.to_string(), + name: object_type.name.clone(), + schema_json: json_string(&object_type.data_schema()), + allowed_links_json: json_string( + object_type + .allowed_links + .as_ref() + .unwrap_or(&Value::Object(Map::new())), + ), + body_extraction_json: object_type.body_extraction.as_ref().map(json_string), + has_plugin: object_type.has_plugin, + module_id: object_type.module_id.clone(), + updated_at: None, + }) + .collect() +} + +fn extract_tier1( + frontmatter: &Map, + object_type: &CompiledObjectType, + rel_path: &str, +) -> Result { + let name = frontmatter + .get("name") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| path_stem(rel_path)); + + let mut data = Map::new(); + for (key, value) in frontmatter { + if !is_reserved_frontmatter_key(key) { + data.insert(key.clone(), value.clone()); + } + } + if let Some(explicit) = frontmatter.get("data") { + let Some(map) = explicit.as_object() else { + return Err("frontmatter `data` must be a mapping".to_string()); + }; + for (key, value) in map { + data.insert(key.clone(), value.clone()); + } + } + validate_against_schema( + &object_type.snapshot.name, + &object_type.schema, + &Value::Object(data.clone()), + )?; + apply_frontmatter_title_code(&mut data, frontmatter); + Ok(GraphNode { + object_type: object_type.snapshot.name.clone(), + name, + data, + }) +} + +/// Nodes, record-derived edges, and diagnostics from one Tier 2 body extraction. +type Tier2Extraction = ( + Vec, + Vec, + Vec, +); + +fn extract_tier2( + doc: &crate::QuireDocument, + frontmatter: &Map, + object_type: &CompiledObjectType, + dsl: &ExtractionDsl, + rel_path: &str, +) -> Result { + let extraction = crate::extract(doc, dsl).map_err(quire_error_message)?; + let mut nodes = Vec::new(); + for record in &extraction.records { + validate_against_schema( + &object_type.snapshot.name, + &object_type.schema, + &Value::Object(record.clone()), + )?; + let mut data = record.clone(); + apply_frontmatter_title_code(&mut data, frontmatter); + nodes.push(GraphNode { + object_type: object_type.snapshot.name.clone(), + name: node_name_from_record(record, rel_path), + data, + }); + } + let mut edges = Vec::new(); + for edge in &extraction.edges { + if let Some(source) = nodes.get(edge.record_index) { + let mut metadata = Map::new(); + metadata.insert( + "source".to_string(), + Value::String(format!("body_extraction.emit_edges[{}]", edge.record_index)), + ); + edges.push(GraphEdge { + source_ref: source.name.clone(), + edge_type: edge.edge_type.clone(), + target_ref: edge.target.clone(), + metadata, + }); + } + } + let diagnostics = extraction + .diagnostics + .iter() + .map(|d| { + diagnostic( + "quire_extraction", + d.to_string(), + "info", + Some(object_type.snapshot.name.clone()), + ) + }) + .collect(); + Ok((nodes, edges, diagnostics)) +} + +fn validate_against_schema(archetype: &str, schema: &Value, data: &Value) -> Result<(), String> { + let validator = compile_schema(schema) + .map_err(|err| format!("{archetype}: schema compile failed: {err}"))?; + if let Err(mut errors) = validator.validate(data) { + if let Some(err) = errors.next() { + let detail = crate::validate::schema_validation_detail(&err); + return Err(format!( + "{archetype}: schema validation failed at {}: {}", + detail.path, detail.message + )); + } + } + Ok(()) +} + +fn frontmatter_edges( + frontmatter: &Map, + primary_ref: &str, + repo_name: &str, + diagnostics: &mut Vec, +) -> Result, String> { + let mut edges = Vec::new(); + edges.extend(relationship_sugar_edges( + frontmatter, + primary_ref, + repo_name, + diagnostics, + )); + edges.extend(explicit_relationship_edges( + frontmatter, + primary_ref, + repo_name, + diagnostics, + )?); + edges.extend(explicit_frontmatter_edges(frontmatter, primary_ref)?); + Ok(edges) +} + +fn relationship_sugar_edges( + frontmatter: &Map, + primary_ref: &str, + repo_name: &str, + diagnostics: &mut Vec, +) -> Vec { + const SUGAR: &[(&str, &str, bool)] = &[ + ("depends_on", "depends_on", true), + ("parent", "parent", false), + ("parent_process", "parent", false), + ("template_for", "template_for", true), + ("archetype_for", "archetype_for", true), + ("replaced_by", "replaced_by", false), + ]; + let mut edges = Vec::new(); + for (key, edge_type, list) in SUGAR { + let Some(raw) = frontmatter.get(*key) else { + continue; + }; + if *list { + let items: Vec<(usize, &Value)> = match raw { + Value::Array(values) => values.iter().enumerate().collect(), + Value::String(_) => vec![(0, raw)], + _ => Vec::new(), + }; + for (index, item) in items { + if let Some(target) = item.as_str().filter(|s| !s.trim().is_empty()) { + edges.push(relationship_edge( + primary_ref, + edge_type, + target, + format!("frontmatter.{key}[{index}]"), + repo_name, + diagnostics, + )); + } + } + } else if let Some(target) = raw.as_str().filter(|s| !s.trim().is_empty()) { + edges.push(relationship_edge( + primary_ref, + edge_type, + target, + format!("frontmatter.{key}"), + repo_name, + diagnostics, + )); + } + } + edges +} + +fn explicit_relationship_edges( + frontmatter: &Map, + primary_ref: &str, + repo_name: &str, + diagnostics: &mut Vec, +) -> Result, String> { + let Some(raw) = frontmatter.get("relationships") else { + return Ok(Vec::new()); + }; + let Some(items) = raw.as_array() else { + return Ok(Vec::new()); + }; + let mut edges = Vec::new(); + for (index, entry) in items.iter().enumerate() { + let Some(map) = entry.as_object() else { + return Err(format!( + "malformed_relationship: relationships[{index}] must be a mapping" + )); + }; + let target = map + .get("target") + .and_then(Value::as_str) + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| { + format!("malformed_relationship: relationships[{index}] missing or empty `target`") + })?; + let edge_type = map + .get("type") + .and_then(Value::as_str) + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| { + format!("malformed_relationship: relationships[{index}] missing or empty `type`") + })?; + let mut metadata = Map::new(); + metadata.insert( + "source".to_string(), + Value::String(format!("frontmatter.relationships[{index}]")), + ); + metadata.insert( + "original_target".to_string(), + Value::String(target.to_string()), + ); + let target_ref = normalize_ref(repo_name, target); + if target.starts_with("ix://") && !is_valid_ix_ref(target) { + metadata.insert("unresolved".to_string(), Value::Bool(true)); + diagnostics.push(diagnostic( + "ix_uri_malformed", + format!("Malformed target in frontmatter.relationships[{index}]: {target:?}"), + "warning", + None, + )); + } + edges.push(GraphEdge { + source_ref: primary_ref.to_string(), + edge_type: edge_type.to_string(), + target_ref, + metadata, + }); + } + Ok(edges) +} + +fn explicit_frontmatter_edges( + frontmatter: &Map, + primary_ref: &str, +) -> Result, String> { + let Some(raw) = frontmatter.get("edges") else { + return Ok(Vec::new()); + }; + let Some(items) = raw.as_array() else { + return Err("malformed_edge: frontmatter `edges` must be a list".to_string()); + }; + let mut edges = Vec::new(); + for (index, entry) in items.iter().enumerate() { + let Some(map) = entry.as_object() else { + return Err(format!("malformed_edge: edges[{index}] must be a mapping")); + }; + let edge_type = map + .get("type") + .and_then(Value::as_str) + .ok_or_else(|| format!("malformed_edge: edges[{index}] missing required `type`"))?; + let target = map + .get("target") + .and_then(Value::as_str) + .ok_or_else(|| format!("malformed_edge: edges[{index}] missing required `target`"))?; + let metadata = map + .get("metadata") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + edges.push(GraphEdge { + source_ref: primary_ref.to_string(), + edge_type: edge_type.to_string(), + target_ref: target.to_string(), + metadata, + }); + } + Ok(edges) +} + +fn relationship_edge( + primary_ref: &str, + edge_type: &str, + target: &str, + provenance: String, + repo_name: &str, + diagnostics: &mut Vec, +) -> GraphEdge { + let mut metadata = Map::new(); + metadata.insert("source".to_string(), Value::String(provenance.clone())); + metadata.insert( + "original_target".to_string(), + Value::String(target.to_string()), + ); + if target.starts_with("ix://") && !is_valid_ix_ref(target) { + metadata.insert("unresolved".to_string(), Value::Bool(true)); + diagnostics.push(diagnostic( + "ix_uri_malformed", + format!("Malformed target in {provenance}: {target:?}"), + "warning", + None, + )); + } + GraphEdge { + source_ref: primary_ref.to_string(), + edge_type: edge_type.to_string(), + target_ref: normalize_ref(repo_name, target), + metadata, + } +} + +fn body_ix_edges( + markdown: &str, + primary_ref: &str, + diagnostics: &mut Vec, +) -> Vec { + let re = Regex::new(r#"\[([^\]]*)\]\((ix://[^)\s]+)\)"#) + .expect("static body ix markdown-link regex is valid"); + let mut edges = Vec::new(); + for (index, captures) in re.captures_iter(markdown).enumerate() { + let display = captures.get(1).map(|m| m.as_str()).unwrap_or_default(); + let uri = captures.get(2).map(|m| m.as_str()).unwrap_or_default(); + if !is_valid_ix_ref(uri) { + diagnostics.push(diagnostic( + "ix_uri_malformed", + format!("Skipping malformed ix:// markdown link {uri:?}"), + "warning", + None, + )); + continue; + } + let mut metadata = Map::new(); + metadata.insert( + "source".to_string(), + Value::String(format!("body.ix_link[{index}]")), + ); + metadata.insert("original_uri".to_string(), Value::String(uri.to_string())); + metadata.insert("display".to_string(), Value::String(display.to_string())); + metadata.insert("unresolved".to_string(), Value::Bool(true)); + edges.push(GraphEdge { + source_ref: primary_ref.to_string(), + edge_type: "references".to_string(), + target_ref: uri.to_string(), + metadata, + }); + } + edges +} + +fn dedupe_edges( + edges: Vec, + repo_name: &str, + diagnostics: &mut Vec, +) -> Vec { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for edge in edges { + let key = ( + normalize_ref(repo_name, &edge.source_ref), + edge.edge_type.clone(), + normalize_ref(repo_name, &edge.target_ref), + ); + if seen.insert(key) { + out.push(edge); + } else { + let dup_source = edge + .metadata + .get("source") + .and_then(Value::as_str) + .unwrap_or(""); + diagnostics.push(diagnostic( + "duplicate_edge", + format!( + "duplicate edge {:?} -> {:?}: dropped {dup_source}", + edge.edge_type, edge.target_ref + ), + "warning", + None, + )); + } + } + out +} + +fn node_records(input: &FilamentExtractionInput, nodes: Vec) -> Vec { + nodes + .into_iter() + .map(|node| { + let ref_ = normalize_ref(&input.repo_name, &node.name); + CoreGraphNodeRef { + id: stable_id(&[&input.project_id, "graph-node", &ref_]), + project_id: input.project_id.clone(), + document_id: input.document_id.clone(), + artifact_id: input.artifact_id.clone(), + object_type: node.object_type, + name: node.name, + ref_, + data_json: json_string(&Value::Object(node.data)), + updated_at: None, + } + }) + .collect() +} + +fn edge_records(input: &FilamentExtractionInput, edges: Vec) -> Vec { + edges + .into_iter() + .map(|edge| { + let source_ref = normalize_ref(&input.repo_name, &edge.source_ref); + let target_ref = normalize_ref(&input.repo_name, &edge.target_ref); + CoreGraphEdgeRef { + id: stable_id(&[ + &input.project_id, + "graph-edge", + &source_ref, + &target_ref, + &edge.edge_type, + ]), + project_id: input.project_id.clone(), + source_ref, + target_ref, + edge_type: edge.edge_type, + data_json: json_string(&Value::Object(edge.metadata)), + updated_at: None, + } + }) + .collect() +} + +fn apply_frontmatter_title_code(data: &mut Map, frontmatter: &Map) { + if let Some(id) = frontmatter.get("id") { + data.insert("code".to_string(), Value::String(value_to_string(id))); + } + if let Some(title) = frontmatter.get("title") { + data.insert("title".to_string(), Value::String(value_to_string(title))); + } +} + +fn node_name_from_record(record: &Map, rel_path: &str) -> String { + record + .get("name") + .or_else(|| record.get("heading")) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| path_stem(rel_path)) +} + +fn path_stem(rel_path: &str) -> String { + Path::new(rel_path) + .file_stem() + .and_then(|s| s.to_str()) + .filter(|s| !s.is_empty()) + .unwrap_or(rel_path) + .to_string() +} + +fn is_reserved_frontmatter_key(key: &str) -> bool { + matches!( + key, + "object" | "id" | "name" | "tags" | "edges" | "relationships" | "parent" | "data" + ) +} + +fn normalize_ref(repo_name: &str, value: &str) -> String { + if value.starts_with("ix://") { + value.to_string() + } else { + format!("ix://agent-ix/{repo_name}/{value}") + } +} + +fn is_valid_ix_ref(value: &str) -> bool { + let Some(rest) = value.strip_prefix("ix://") else { + return false; + }; + let mut parts = rest.split('/'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(org), Some(repo), Some(name)) + if !org.is_empty() && !repo.is_empty() && !name.is_empty() + ) +} + +fn value_to_string(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +fn json_string(value: &Value) -> String { + serde_json::to_string(value).expect("serializing serde_json::Value cannot fail") +} + +fn quire_error_message(error: QuireError) -> String { + error.to_string() +} + +fn diagnostic( + code: impl Into, + message: impl Into, + severity: impl Into, + object_type: Option, +) -> CoreExtractionDiagnostic { + CoreExtractionDiagnostic { + code: code.into(), + message: message.into(), + severity: severity.into(), + object_type, + } +} + +fn stable_id(parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for (idx, part) in parts.iter().enumerate() { + if idx > 0 { + hasher.update([0]); + } + hasher.update(part.as_bytes()); + } + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_input(markdown: &str) -> FilamentExtractionInput { + FilamentExtractionInput { + project_id: "project".to_string(), + document_id: "doc-1".to_string(), + artifact_id: Some("artifact-1".to_string()), + rel_path: "spec/FR-001.md".to_string(), + markdown: markdown.to_string(), + repo_name: "example".to_string(), + object_types: vec![FilamentObjectType { + name: "capability".to_string(), + schema: Some(json!({"type": "object", "additionalProperties": true})), + data_schema: None, + allowed_links: Some(json!({})), + body_extraction: None, + has_plugin: false, + module_id: None, + }], + } + } + + #[test] + fn tc633_tier1_frontmatter_extracts_core_node() { + let result = extract_filament_core(base_input( + "---\nid: FR-001\ntitle: Pay vendors\nobject: capability\npriority: P0\n---\n# Body\n", + )); + assert_eq!(result.errors, Vec::::new()); + assert_eq!(result.nodes.len(), 1); + assert_eq!(result.nodes[0].object_type, "capability"); + assert_eq!(result.nodes[0].name, "FR-001"); + assert_eq!(result.nodes[0].ref_, "ix://agent-ix/example/FR-001"); + let data: Value = serde_json::from_str(&result.nodes[0].data_json).unwrap(); + assert_eq!(data["code"], "FR-001"); + assert_eq!(data["title"], "Pay vendors"); + assert_eq!(data["priority"], "P0"); + } + + #[test] + fn tc634_tier2_uses_rust_dsl_records_and_edges() { + let mut input = base_input( + "---\nid: API-001\ntitle: Payments API\nobject: endpoint\n---\n## Endpoint\nGET /payments\n", + ); + input.object_types[0] = FilamentObjectType { + name: "endpoint".to_string(), + schema: Some(json!({ + "type": "object", + "required": ["method", "target"], + "properties": { + "method": {"type": "string"}, + "target": {"type": "string"} + } + })), + data_schema: None, + allowed_links: Some(json!({})), + body_extraction: Some(json!({ + "yield_pattern": { + "match": { + "method": { + "from": "section_body", + "after_heading": "Endpoint", + "regex": "^(GET)", + "required": true + }, + "target": { + "from": "section_body", + "after_heading": "Endpoint", + "regex": "(/payments)", + "required": true + } + } + }, + "emit_edges": [{"type": "serves", "target": "ix://agent-ix/example/API"}] + })), + has_plugin: false, + module_id: None, + }; + let result = extract_filament_core(input); + assert_eq!(result.errors, Vec::::new()); + assert_eq!(result.nodes.len(), 1); + let data: Value = serde_json::from_str(&result.nodes[0].data_json).unwrap(); + assert_eq!(data["method"], "GET"); + assert_eq!(data["target"], "/payments"); + assert!(result.edges.iter().any( + |edge| edge.edge_type == "serves" && edge.target_ref == "ix://agent-ix/example/API" + )); + } + + #[test] + fn tc635_error_fixtures_are_reported_without_panic() { + let no_frontmatter = extract_filament_core(base_input("# Body\n")); + assert!(no_frontmatter.nodes.is_empty()); + assert_eq!(no_frontmatter.diagnostics[0].code, "no_frontmatter"); + + let unknown = extract_filament_core(base_input( + "---\nid: FR-001\nobject: missing\n---\n# Body\n", + )); + assert!(unknown.nodes.is_empty()); + assert!(unknown + .diagnostics + .iter() + .any(|d| d.code == "unknown_object_type")); + + let mut plugin = base_input("---\nid: FR-001\nobject: capability\n---\n# Body\n"); + plugin.object_types[0].has_plugin = true; + let plugin = extract_filament_core(plugin); + assert!(plugin + .errors + .iter() + .any(|err| err.contains("missing_plugin"))); + + let duplicate = extract_filament_core(base_input( + "---\nid: FR-001\nobject: capability\nrelationships:\n - target: ix://agent-ix/example/FR-002\n type: references\n---\nSee [FR-002](ix://agent-ix/example/FR-002).\n", + )); + assert_eq!(duplicate.edges.len(), 1); + assert!(duplicate + .diagnostics + .iter() + .any(|d| d.code == "duplicate_edge")); + + let malformed = extract_filament_core(base_input( + "---\nid: FR-001\nobject: capability\nrelationships:\n - target: ix://agent-ix\n type: references\n---\n# Body\n", + )); + assert!(malformed + .diagnostics + .iter() + .any(|d| d.code == "ix_uri_malformed")); + } + + #[test] + fn tc657_malformed_frontmatter_is_a_parse_failure_but_absent_is_clean() { + // Complete `---` … `---` fence block with unparseable YAML → parse + // failure: a non-empty `errors` entry (routed to Filament index_errors + // via MarkStale→apply_stale) plus a `frontmatter_unparsable` diagnostic. + let malformed = extract_filament_core(base_input("---\nid: : bad\n---\n# Body\n")); + assert!(malformed.nodes.is_empty()); + assert!(malformed.edges.is_empty()); + assert!( + malformed + .errors + .iter() + .any(|err| err.contains("parse_failed")), + "expected parse_failed error; got {:?}", + malformed.errors + ); + assert!(malformed + .diagnostics + .iter() + .any(|d| d.code == "frontmatter_unparsable" && d.severity == "error")); + + // No frontmatter block at all → clean skip: empty errors, only the + // informational `no_frontmatter` diagnostic. + let absent = extract_filament_core(base_input("# Body\n")); + assert_eq!(absent.errors, Vec::::new()); + assert!(absent + .diagnostics + .iter() + .any(|d| d.code == "no_frontmatter")); + } + + #[test] + fn tc636_relationship_sugar_and_body_ix_links_have_metadata() { + let result = extract_filament_core(base_input( + "---\nid: FR-001\nobject: capability\ndepends_on:\n - FR-002\n---\nSee [US-001](ix://agent-ix/example/US-001).\n", + )); + assert_eq!(result.errors, Vec::::new()); + assert!(result + .edges + .iter() + .any(|edge| edge.edge_type == "depends_on" + && edge.target_ref == "ix://agent-ix/example/FR-002" + && edge.data_json.contains("original_target"))); + assert!(result + .edges + .iter() + .any(|edge| edge.edge_type == "references" + && edge.target_ref == "ix://agent-ix/example/US-001" + && edge.data_json.contains("original_uri"))); + } + + #[test] + fn tc637_repeated_extraction_is_byte_identical() { + let input = base_input( + "---\nid: FR-001\ntitle: Pay vendors\nobject: capability\ndepends_on:\n - FR-002\n---\nSee [US-001](ix://agent-ix/example/US-001).\n", + ); + let first = extract_filament_core(input.clone()); + let second = extract_filament_core(input); + assert_eq!( + serde_json::to_string(&first).unwrap(), + serde_json::to_string(&second).unwrap() + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index b5aa762..65e0508 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,7 @@ pub mod corpus; pub mod diagnostic; pub mod error; pub mod extract; +pub mod filament; pub mod lint; pub mod loader; pub mod merge; @@ -31,7 +32,9 @@ pub mod writeback; // Parser surface (FR-005..009). pub use ast::{QuireDocument, QuireSection}; -pub use parser::{extract_frontmatter, parse_document, FrontmatterResult, Heading}; +pub use parser::{ + extract_frontmatter, parse_document, FrontmatterResult, FrontmatterStatus, Heading, +}; // Query API (FR-010). pub use query::{ concept_type, diagrams_from_content, extract_diagrams, parse_bullet_list, parse_table, @@ -61,6 +64,10 @@ pub use contract::{input_contract_for, ContractSection, InputContract}; pub use extract::dsl::{ExtractionDsl, IterateKind, IterateOver, YieldPattern}; pub use extract::locator::{Locator, LocatorAssert, LocatorKind, LocatorPrimitive}; pub use extract::{extract, ExtractionResult}; +pub use filament::{ + extract_filament_core, CoreExtractionDiagnostic, CoreExtractionResult, CoreGraphEdgeRef, + CoreGraphNodeRef, CoreObjectTypeRecord, FilamentExtractionInput, FilamentObjectType, +}; // Assert facet (FR-033) + `{field}` interpolation (FR-034). pub use extract::assert_eval::{evaluate_assert, AssertFailure, AssertReason}; pub use extract::interpolate::{interpolate, UnresolvedField}; diff --git a/src/parser/frontmatter.rs b/src/parser/frontmatter.rs index 2900d7d..bdc82e0 100644 --- a/src/parser/frontmatter.rs +++ b/src/parser/frontmatter.rs @@ -18,6 +18,30 @@ use serde_json::{Map, Value}; +/// Why [`extract_frontmatter`] produced (or did not produce) a frontmatter +/// mapping. This is the single authority for distinguishing an *absent* +/// frontmatter block from a *present-but-malformed* one, so boundary +/// consumers (e.g. the Filament extraction engine, FR-040) do not have to +/// re-derive that fact from the raw markdown. +/// +/// It is an out-of-band discriminant only: the parity-observable outputs +/// (`frontmatter`/`body`) are byte-identical across all branches, so this +/// carries no TS/Py wire-shape impact (cf. the BOM/CRLF extensions above). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrontmatterStatus { + /// No opening `---` fence, an opening fence with no closing `\n---`, or a + /// complete fence block whose content is empty / whitespace / comments + /// (parses to YAML null). Treated as a document with no usable frontmatter. + Absent, + /// A complete `---` … `---` block that parsed into a YAML mapping. + Present, + /// A complete `---` … `---` block was present, but its contents were not + /// a parseable YAML mapping: invalid YAML, or a valid non-null, non-object + /// value (array / scalar / bool / number). An empty / null block is + /// [`Absent`](Self::Absent), not `Malformed`. + Malformed, +} + /// Result of [`extract_frontmatter`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FrontmatterResult { @@ -26,6 +50,8 @@ pub struct FrontmatterResult { /// Markdown body after the closing fence (post-BOM-strip). On the /// "no frontmatter" branches this is the entire post-BOM-strip input. pub body: String, + /// Why `frontmatter` is `None`, or that it parsed. See [`FrontmatterStatus`]. + pub status: FrontmatterStatus, } /// Extract YAML frontmatter from a markdown string per FR-006. @@ -40,7 +66,7 @@ pub fn extract_frontmatter(markdown: &str) -> FrontmatterResult { } else if stripped.starts_with("---\r\n") { 5 } else { - return no_frontmatter(stripped); + return no_frontmatter(stripped, FrontmatterStatus::Absent); }; // First "\n---" closes the frontmatter (TS/Py parity: @@ -50,7 +76,9 @@ pub fn extract_frontmatter(markdown: &str) -> FrontmatterResult { let after_open: &str = &stripped[yaml_start..]; let close_rel: usize = match after_open.find("\n---") { Some(p) => p, - None => return no_frontmatter(stripped), + // Opening fence but no closing fence: an unterminated block is not a + // frontmatter block (conservative — could be a `---` thematic break). + None => return no_frontmatter(stripped, FrontmatterStatus::Absent), }; let body_tail: &str = &after_open[close_rel + 4..]; @@ -61,15 +89,23 @@ pub fn extract_frontmatter(markdown: &str) -> FrontmatterResult { let parsed: Value = match serde_yaml::from_str::(yaml_str) { Ok(v) => v, - Err(_) => return no_frontmatter(stripped), + // A complete fence block whose contents are not parseable YAML. + Err(_) => return no_frontmatter(stripped, FrontmatterStatus::Malformed), }; let map: Map = match parsed { Value::Object(m) => m, - // Valid YAML but not an object (null / array / scalar) → no FM. - // The spec types frontmatter as Map; non-objects - // cannot be represented and are treated as the malformed branch. - _ => return no_frontmatter(stripped), + // An empty / whitespace-only / comment-only fence block parses to + // `Null`: it carries no metadata and is indistinguishable in intent + // from having no frontmatter (and `---`\n\n`---` is a common pair of + // thematic breaks). Treat it as *absent*, not a parse failure, so it + // does not pollute Filament's index_errors (CR-010). + Value::Null => return no_frontmatter(stripped, FrontmatterStatus::Absent), + // Valid YAML but a non-mapping value (array / scalar / bool / number): + // the spec types frontmatter as Map, so a wrong-typed + // block is genuinely malformed. Still `frontmatter: None` for FR-006 + // parity; the status lets boundary consumers surface it. + _ => return no_frontmatter(stripped, FrontmatterStatus::Malformed), }; // Strip a single leading EOL from the body — it belongs to the @@ -82,13 +118,15 @@ pub fn extract_frontmatter(markdown: &str) -> FrontmatterResult { FrontmatterResult { frontmatter: Some(map), body: body.to_string(), + status: FrontmatterStatus::Present, } } -fn no_frontmatter(stripped: &str) -> FrontmatterResult { +fn no_frontmatter(stripped: &str, status: FrontmatterStatus) -> FrontmatterResult { FrontmatterResult { frontmatter: None, body: stripped.to_string(), + status, } } @@ -106,6 +144,8 @@ mod tests { let r = extract_frontmatter("# heading"); assert!(r.frontmatter.is_none()); assert_eq!(r.body, "# heading"); + // No opening fence → the block is genuinely absent (not malformed). + assert_eq!(r.status, FrontmatterStatus::Absent); } // FR-006-AC-2 / TC-012 (happy path) @@ -115,6 +155,7 @@ mod tests { let map = r.frontmatter.expect("expected frontmatter"); assert_eq!(map.get("id"), Some(&fm_value("FR-001"))); assert_eq!(r.body, "body"); + assert_eq!(r.status, FrontmatterStatus::Present); } // FR-006-AC-3 / TC-013 @@ -124,6 +165,9 @@ mod tests { let r = extract_frontmatter(input); assert!(r.frontmatter.is_none()); assert_eq!(r.body, input); + // A complete fence block with unparseable YAML is malformed, not absent + // (CR-010): boundary consumers surface this as a parse failure. + assert_eq!(r.status, FrontmatterStatus::Malformed); } // FR-006-AC-4 / TC-014 @@ -133,6 +177,9 @@ mod tests { let r = extract_frontmatter(input); assert!(r.frontmatter.is_none()); assert_eq!(r.body, input); + // No closing fence → conservatively treated as absent (could be a + // `---` thematic break), so it is NOT a parse failure. + assert_eq!(r.status, FrontmatterStatus::Absent); } // FR-006-AC-5 / TC-180 @@ -185,6 +232,32 @@ mod tests { let r = extract_frontmatter(input); assert!(r.frontmatter.is_none()); assert_eq!(r.body, input); + // Valid YAML but a non-object between complete fences → malformed. + assert_eq!(r.status, FrontmatterStatus::Malformed); + } + + // FR-006-AC-7 / TC-658 (empty/null → Absent half of the status classification) + #[test] + fn empty_or_comment_only_frontmatter_is_absent_not_malformed() { + // A complete fence block whose content parses to YAML null (empty, + // whitespace-only, or comment-only) carries no metadata and must NOT + // be reported as a parse failure (CR-010) — it is treated as absent. + for input in [ + "---\n\n---\nbody", + "---\n \n---\nbody", + "---\n# only a comment\n---\nbody", + ] { + let r = extract_frontmatter(input); + assert!( + r.frontmatter.is_none(), + "{input:?} should have no frontmatter" + ); + assert_eq!( + r.status, + FrontmatterStatus::Absent, + "{input:?} should be Absent, not Malformed" + ); + } } #[test] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 2517e86..0516f2a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -13,7 +13,7 @@ pub mod slice; pub mod slug; pub mod walk; -pub use frontmatter::{extract_frontmatter, FrontmatterResult}; +pub use frontmatter::{extract_frontmatter, FrontmatterResult, FrontmatterStatus}; pub use slice::{line_offsets, slice_section_content}; pub use slug::{slug, slug_line_id}; pub use walk::{walk_headings, Heading}; diff --git a/src/python/mod.rs b/src/python/mod.rs index 98c899e..88e646c 100644 --- a/src/python/mod.rs +++ b/src/python/mod.rs @@ -48,6 +48,8 @@ fn quire(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(input_skeleton, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(extract, m)?)?; + m.add_function(wrap_pyfunction!(extract_filament_core, m)?)?; + m.add_function(wrap_pyfunction!(extract_core_data, m)?)?; m.add_function(wrap_pyfunction!(extract_frontmatter, m)?)?; m.add_function(wrap_pyfunction!(harvest_edges, m)?)?; m.add_class::()?; @@ -287,6 +289,35 @@ fn extract<'py>( Ok(out) } +/// Run canonical Filament core-data extraction for one markdown +/// document. Accepts the JSON-serializable input shape from +/// `crate::filament::FilamentExtractionInput` as a native Python dict +/// and returns a native Python dict compatible with +/// `CoreExtractionResult`. +#[pyfunction] +fn extract_core_data<'py>( + py: Python<'py>, + input: &Bound<'_, PyAny>, +) -> PyResult> { + extract_filament_core(py, input) +} + +/// Run canonical Filament extraction for one markdown document. Accepts and +/// returns native Python JSON-ish values without subprocess or stdio policy. +#[pyfunction] +fn extract_filament_core<'py>( + py: Python<'py>, + input: &Bound<'_, PyAny>, +) -> PyResult> { + let value = py_to_json(input)?; + let request: crate::filament::FilamentExtractionInput = + serde_json::from_value(value).map_err(|err| PyTypeError::new_err(err.to_string()))?; + let result = py.detach(|| crate::filament::extract_filament_core(request)); + let value = + serde_json::to_value(result).map_err(|err| QuireParseError::new_err(err.to_string()))?; + json_to_py(py, &value) +} + /// Extract frontmatter and body from `text` using the Rust FR-006 parser. /// Returns `{frontmatter: dict | None, body: str}`. #[pyfunction] diff --git a/tests/filament_core_fixtures.rs b/tests/filament_core_fixtures.rs new file mode 100644 index 0000000..91e3455 --- /dev/null +++ b/tests/filament_core_fixtures.rs @@ -0,0 +1,192 @@ +use quire_rs::{extract_filament_core, CoreExtractionResult, FilamentExtractionInput}; +use serde::Deserialize; +use serde_json::Value; + +const FIXTURES: &str = include_str!("fixtures/filament_core/graph_cases.json"); + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Fixture { + name: String, + input: FilamentExtractionInput, + expect: Expected, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Expected { + #[serde(default)] + nodes: Vec, + #[serde(default)] + edges: Vec, + #[serde(default)] + diagnostic_codes: Vec, + #[serde(default)] + errors_contain: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ExpectedNode { + object_type: Option, + name: Option, + #[serde(rename = "ref")] + ref_: Option, + data: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ExpectedEdge { + source_ref: Option, + target_ref: Option, + edge_type: Option, + data: Option, +} + +#[test] +fn filament_core_graph_fixture_corpus() { + for fixture in fixtures() { + let result = extract_filament_core(fixture.input.clone()); + assert_expected(&fixture.name, &result, &fixture.expect); + } +} + +#[test] +fn filament_core_graph_fixture_corpus_is_deterministic() { + for fixture in fixtures() { + let first = extract_filament_core(fixture.input.clone()); + let second = extract_filament_core(fixture.input); + assert_eq!( + serde_json::to_value(&first).unwrap(), + serde_json::to_value(&second).unwrap(), + "{} did not produce deterministic output", + fixture.name + ); + } +} + +fn fixtures() -> Vec { + serde_json::from_str(FIXTURES).expect("filament core fixtures parse") +} + +fn assert_expected(name: &str, result: &CoreExtractionResult, expected: &Expected) { + assert_eq!( + result.nodes.len(), + expected.nodes.len(), + "{name}: node count mismatch\n{result:#?}" + ); + assert_eq!( + result.edges.len(), + expected.edges.len(), + "{name}: edge count mismatch\n{result:#?}" + ); + + for (idx, expected_node) in expected.nodes.iter().enumerate() { + let actual = &result.nodes[idx]; + if let Some(object_type) = &expected_node.object_type { + assert_eq!( + &actual.object_type, object_type, + "{name}: node[{idx}] objectType" + ); + } + if let Some(node_name) = &expected_node.name { + assert_eq!(&actual.name, node_name, "{name}: node[{idx}] name"); + } + if let Some(ref_) = &expected_node.ref_ { + assert_eq!(&actual.ref_, ref_, "{name}: node[{idx}] ref"); + } + if let Some(data) = &expected_node.data { + let actual_data: Value = + serde_json::from_str(&actual.data_json).expect("node dataJson parses"); + assert_json_subset(name, &format!("node[{idx}].data"), data, &actual_data); + } + } + + for (idx, expected_edge) in expected.edges.iter().enumerate() { + let actual = &result.edges[idx]; + if let Some(source_ref) = &expected_edge.source_ref { + assert_eq!( + &actual.source_ref, source_ref, + "{name}: edge[{idx}] sourceRef" + ); + } + if let Some(target_ref) = &expected_edge.target_ref { + assert_eq!( + &actual.target_ref, target_ref, + "{name}: edge[{idx}] targetRef" + ); + } + if let Some(edge_type) = &expected_edge.edge_type { + assert_eq!(&actual.edge_type, edge_type, "{name}: edge[{idx}] edgeType"); + } + if let Some(data) = &expected_edge.data { + let actual_data: Value = + serde_json::from_str(&actual.data_json).expect("edge dataJson parses"); + assert_json_subset(name, &format!("edge[{idx}].data"), data, &actual_data); + } + } + + let diagnostic_codes: Vec<&str> = result + .diagnostics + .iter() + .map(|diagnostic| diagnostic.code.as_str()) + .collect(); + for code in &expected.diagnostic_codes { + assert!( + diagnostic_codes.contains(&code.as_str()), + "{name}: missing diagnostic code {code:?}; got {diagnostic_codes:?}" + ); + } + + for needle in &expected.errors_contain { + assert!( + result.errors.iter().any(|error| error.contains(needle)), + "{name}: missing error containing {needle:?}; got {:?}", + result.errors + ); + } + + if expected.errors_contain.is_empty() { + assert_eq!( + result.errors, + Vec::::new(), + "{name}: unexpected errors" + ); + } +} + +fn assert_json_subset(name: &str, path: &str, expected: &Value, actual: &Value) { + match expected { + Value::Object(map) => { + let actual_map = actual + .as_object() + .unwrap_or_else(|| panic!("{name}: {path} expected object, got {actual:?}")); + for (key, expected_value) in map { + let actual_value = actual_map.get(key).unwrap_or_else(|| { + panic!("{name}: {path} missing expected key {key:?}; got {actual:?}") + }); + assert_json_subset(name, &format!("{path}.{key}"), expected_value, actual_value); + } + } + Value::Array(values) => { + let actual_values = actual + .as_array() + .unwrap_or_else(|| panic!("{name}: {path} expected array, got {actual:?}")); + assert_eq!( + actual_values.len(), + values.len(), + "{name}: {path} array length mismatch" + ); + for (idx, expected_value) in values.iter().enumerate() { + assert_json_subset( + name, + &format!("{path}[{idx}]"), + expected_value, + &actual_values[idx], + ); + } + } + _ => assert_eq!(actual, expected, "{name}: {path} mismatch"), + } +} diff --git a/tests/fixtures/filament_core/graph_cases.json b/tests/fixtures/filament_core/graph_cases.json new file mode 100644 index 0000000..0549084 --- /dev/null +++ b/tests/fixtures/filament_core/graph_cases.json @@ -0,0 +1,335 @@ +[ + { + "name": "tier1-frontmatter-node", + "tags": ["TC-643", "happy", "tier1"], + "input": { + "projectId": "project", + "documentId": "doc-tier1", + "artifactId": "artifact-tier1", + "relPath": "spec/FR-001.md", + "repoName": "example", + "markdown": "---\nid: FR-001\ntitle: Pay vendors\nobject: capability\npriority: P0\n---\n# Body\n", + "objectTypes": [ + { + "name": "capability", + "schema": { "type": "object", "additionalProperties": true }, + "allowedLinks": {}, + "bodyExtraction": null, + "hasPlugin": false, + "moduleId": "test-module" + } + ] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "FR-001", "ref": "ix://agent-ix/example/FR-001", "data": { "code": "FR-001", "title": "Pay vendors", "priority": "P0" } }], + "edges": [], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "tier2-dsl-record-and-edge", + "tags": ["TC-644", "happy", "tier2"], + "input": { + "projectId": "project", + "documentId": "doc-tier2", + "artifactId": "artifact-tier2", + "relPath": "spec/API-001.md", + "repoName": "example", + "markdown": "---\nid: API-001\ntitle: Payments API\nobject: endpoint\n---\n## Endpoint\nGET /payments\n", + "objectTypes": [ + { + "name": "endpoint", + "schema": { + "type": "object", + "required": ["method", "target"], + "properties": { + "method": { "type": "string" }, + "target": { "type": "string" } + } + }, + "allowedLinks": {}, + "bodyExtraction": { + "yield_pattern": { + "match": { + "method": { "from": "section_body", "after_heading": "Endpoint", "regex": "^(GET)", "required": true }, + "target": { "from": "section_body", "after_heading": "Endpoint", "regex": "(/payments)", "required": true } + } + }, + "emit_edges": [{ "type": "serves", "target": "ix://agent-ix/example/API" }] + }, + "hasPlugin": false, + "moduleId": "test-module" + } + ] + }, + "expect": { + "nodes": [{ "objectType": "endpoint", "name": "API-001", "data": { "method": "GET", "target": "/payments", "code": "API-001", "title": "Payments API" } }], + "edges": [{ "edgeType": "serves", "targetRef": "ix://agent-ix/example/API", "data": { "source": "body_extraction.emit_edges[0]" } }], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "explicit-frontmatter-edges", + "tags": ["TC-645", "happy", "edges"], + "input": { + "projectId": "project", + "documentId": "doc-explicit", + "artifactId": "artifact-explicit", + "relPath": "spec/CAP-001.md", + "repoName": "example", + "markdown": "---\nid: CAP-001\nobject: capability\nedges:\n - type: requires\n target: ix://agent-ix/example/FR-014\n - type: realizes\n target: ix://agent-ix/example/CAP-PAY\n metadata:\n since: v0.2\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "CAP-001" }], + "edges": [ + { "edgeType": "requires", "sourceRef": "ix://agent-ix/example/CAP-001", "targetRef": "ix://agent-ix/example/FR-014" }, + { "edgeType": "realizes", "targetRef": "ix://agent-ix/example/CAP-PAY", "data": { "since": "v0.2" } } + ], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "relationship-sugar-all-fields", + "tags": ["TC-646", "happy", "relationships"], + "input": { + "projectId": "project", + "documentId": "doc-sugar", + "artifactId": "artifact-sugar", + "relPath": "spec/FR-010.md", + "repoName": "example", + "markdown": "---\nid: FR-010\nobject: capability\ndepends_on:\n - FR-001\nparent: PROC-001\nparent_process: PROC-ROOT\ntemplate_for: spec-X\narchetype_for:\n - arche-Y\nreplaced_by: FR-999\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "FR-010" }], + "edges": [ + { "edgeType": "depends_on", "targetRef": "ix://agent-ix/example/FR-001", "data": { "source": "frontmatter.depends_on[0]", "original_target": "FR-001" } }, + { "edgeType": "parent", "targetRef": "ix://agent-ix/example/PROC-001", "data": { "source": "frontmatter.parent" } }, + { "edgeType": "parent", "targetRef": "ix://agent-ix/example/PROC-ROOT", "data": { "source": "frontmatter.parent_process" } }, + { "edgeType": "template_for", "targetRef": "ix://agent-ix/example/spec-X", "data": { "source": "frontmatter.template_for[0]" } }, + { "edgeType": "archetype_for", "targetRef": "ix://agent-ix/example/arche-Y", "data": { "source": "frontmatter.archetype_for[0]" } }, + { "edgeType": "replaced_by", "targetRef": "ix://agent-ix/example/FR-999", "data": { "source": "frontmatter.replaced_by" } } + ], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "ix-pass-through-and-bare-normalization", + "tags": ["TC-647", "happy", "normalization"], + "input": { + "projectId": "project", + "documentId": "doc-normalize", + "artifactId": "artifact-normalize", + "relPath": "spec/FR-020.md", + "repoName": "example", + "markdown": "---\nid: FR-020\nobject: capability\nrelationships:\n - target: ix://other-org/other-repo/FR-777\n type: cross_repo\n - target: FR-021\n type: local_ref\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "FR-020" }], + "edges": [ + { "edgeType": "cross_repo", "targetRef": "ix://other-org/other-repo/FR-777" }, + { "edgeType": "local_ref", "targetRef": "ix://agent-ix/example/FR-021", "data": { "original_target": "FR-021" } } + ], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "body-ix-markdown-links", + "tags": ["TC-648", "happy", "body-links"], + "input": { + "projectId": "project", + "documentId": "doc-body-links", + "artifactId": "artifact-body-links", + "relPath": "spec/FR-030.md", + "repoName": "example", + "markdown": "---\nid: FR-030\nobject: capability\n---\nSee [User story](ix://agent-ix/example/US-001) and [external](https://example.com).\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "FR-030" }], + "edges": [{ "edgeType": "references", "targetRef": "ix://agent-ix/example/US-001", "data": { "source": "body.ix_link[0]", "display": "User story", "unresolved": true } }], + "diagnosticCodes": [], + "errorsContain": [] + } + }, + { + "name": "duplicate-edges-dedupe-first-wins", + "tags": ["TC-649", "sad", "dedupe"], + "input": { + "projectId": "project", + "documentId": "doc-dedupe", + "artifactId": "artifact-dedupe", + "relPath": "spec/FR-040.md", + "repoName": "example", + "markdown": "---\nid: FR-040\nobject: capability\nrelationships:\n - target: ix://agent-ix/example/FR-041\n type: references\n---\nSee [FR-041](ix://agent-ix/example/FR-041).\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { + "nodes": [{ "objectType": "capability", "name": "FR-040" }], + "edges": [{ "edgeType": "references", "targetRef": "ix://agent-ix/example/FR-041", "data": { "source": "frontmatter.relationships[0]" } }], + "diagnosticCodes": ["duplicate_edge"], + "errorsContain": [] + } + }, + { + "name": "malformed-relationships-non-map", + "tags": ["TC-650", "sad", "relationships"], + "input": { + "projectId": "project", + "documentId": "doc-rel-non-map", + "artifactId": "artifact-rel-non-map", + "relPath": "spec/FR-050.md", + "repoName": "example", + "markdown": "---\nid: FR-050\nobject: capability\nrelationships:\n - not-a-map\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-050" }], "edges": [], "diagnosticCodes": [], "errorsContain": ["malformed_relationship", "must be a mapping"] } + }, + { + "name": "malformed-relationships-missing-target", + "tags": ["TC-650", "sad", "relationships"], + "input": { + "projectId": "project", + "documentId": "doc-rel-missing-target", + "artifactId": "artifact-rel-missing-target", + "relPath": "spec/FR-051.md", + "repoName": "example", + "markdown": "---\nid: FR-051\nobject: capability\nrelationships:\n - type: depends_on\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-051" }], "edges": [], "diagnosticCodes": [], "errorsContain": ["malformed_relationship", "target"] } + }, + { + "name": "malformed-relationships-missing-type", + "tags": ["TC-650", "sad", "relationships"], + "input": { + "projectId": "project", + "documentId": "doc-rel-missing-type", + "artifactId": "artifact-rel-missing-type", + "relPath": "spec/FR-052.md", + "repoName": "example", + "markdown": "---\nid: FR-052\nobject: capability\nrelationships:\n - target: ix://agent-ix/example/FR-053\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-052" }], "edges": [], "diagnosticCodes": [], "errorsContain": ["malformed_relationship", "type"] } + }, + { + "name": "malformed-explicit-edges", + "tags": ["TC-651", "sad", "edges"], + "input": { + "projectId": "project", + "documentId": "doc-edge-missing-target", + "artifactId": "artifact-edge-missing-target", + "relPath": "spec/FR-060.md", + "repoName": "example", + "markdown": "---\nid: FR-060\nobject: capability\nedges:\n - type: requires\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-060" }], "edges": [], "diagnosticCodes": [], "errorsContain": ["malformed_edge", "target"] } + }, + { + "name": "schema-validation-required-field", + "tags": ["TC-652", "sad", "schema"], + "input": { + "projectId": "project", + "documentId": "doc-schema", + "artifactId": "artifact-schema", + "relPath": "spec/DOM-001.md", + "repoName": "example", + "markdown": "---\nid: DOM-001\nobject: domain\ndata: {}\n---\n# Body\n", + "objectTypes": [{ "name": "domain", "schema": { "type": "object", "required": ["description"], "properties": { "description": { "type": "string" } } }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [], "edges": [], "diagnosticCodes": [], "errorsContain": ["schema validation failed", "description"] } + }, + { + "name": "unknown-object-diagnostic", + "tags": ["TC-653", "sad", "unknown"], + "input": { + "projectId": "project", + "documentId": "doc-unknown", + "artifactId": "artifact-unknown", + "relPath": "spec/FR-070.md", + "repoName": "example", + "markdown": "---\nid: FR-070\nobject: missing\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [], "edges": [], "diagnosticCodes": ["unknown_object_type"], "errorsContain": [] } + }, + { + "name": "no-frontmatter-diagnostic", + "tags": ["TC-653", "sad", "frontmatter"], + "input": { + "projectId": "project", + "documentId": "doc-no-frontmatter", + "artifactId": "artifact-no-frontmatter", + "relPath": "spec/FR-071.md", + "repoName": "example", + "markdown": "# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [], "edges": [], "diagnosticCodes": ["no_frontmatter"], "errorsContain": [] } + }, + { + "name": "plugin-unsupported-error", + "tags": ["TC-653", "sad", "plugin"], + "input": { + "projectId": "project", + "documentId": "doc-plugin", + "artifactId": "artifact-plugin", + "relPath": "spec/PLUG-001.md", + "repoName": "example", + "markdown": "---\nid: PLUG-001\nobject: plugin\n---\n# Body\n", + "objectTypes": [{ "name": "plugin", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": true, "moduleId": "test-module" }] + }, + "expect": { "nodes": [], "edges": [], "diagnosticCodes": [], "errorsContain": ["missing_plugin"] } + }, + { + "name": "negative-scope-no-non-ix-links", + "tags": ["TC-654", "sad", "negative-scope"], + "input": { + "projectId": "project", + "documentId": "doc-negative", + "artifactId": "artifact-negative", + "relPath": "spec/FR-080.md", + "repoName": "example", + "markdown": "---\nid: FR-080\nobject: capability\n---\nSee [[FR-081]], spec://FR-082, prose depends on FR-083, and [web](https://example.com).\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-080" }], "edges": [], "diagnosticCodes": [], "errorsContain": [] } + }, + { + "name": "malformed-body-ix-link-diagnostic", + "tags": ["TC-653", "sad", "body-links"], + "input": { + "projectId": "project", + "documentId": "doc-bad-body-link", + "artifactId": "artifact-bad-body-link", + "relPath": "spec/FR-090.md", + "repoName": "example", + "markdown": "---\nid: FR-090\nobject: capability\n---\nSee [bad](ix://agent-ix).\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [{ "objectType": "capability", "name": "FR-090" }], "edges": [], "diagnosticCodes": ["ix_uri_malformed"], "errorsContain": [] } + }, + { + "name": "frontmatter-unparsable-error", + "tags": ["TC-657", "sad", "frontmatter"], + "input": { + "projectId": "project", + "documentId": "doc-unparsable-frontmatter", + "artifactId": "artifact-unparsable-frontmatter", + "relPath": "spec/FR-100.md", + "repoName": "example", + "markdown": "---\nid: : bad\n---\n# Body\n", + "objectTypes": [{ "name": "capability", "schema": { "type": "object", "additionalProperties": true }, "allowedLinks": {}, "bodyExtraction": null, "hasPlugin": false, "moduleId": "test-module" }] + }, + "expect": { "nodes": [], "edges": [], "diagnosticCodes": ["frontmatter_unparsable"], "errorsContain": ["parse_failed"] } + } +]