diff --git a/.github/workflows/product.yml b/.github/workflows/product.yml new file mode 100644 index 00000000..8700247d --- /dev/null +++ b/.github/workflows/product.yml @@ -0,0 +1,99 @@ +name: Product + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, closed] + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + rust-quality: + if: ${{ github.event_name != 'pull_request' || (github.event.action != 'closed' && github.event.pull_request.draft == false) }} + runs-on: ubuntu-24.04 + env: + COVERAGE_TOOLCHAIN: nightly-2026-08-20 + steps: + - name: Checkout exact source revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact source revision + env: + EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + + - name: Validate Product CI contract + run: python3 scripts/check_ci_contract.py + + - name: Show pinned Rust toolchain + run: rustc --version && cargo --version + + - name: Format + run: cargo fmt --all --check + + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: Test + run: cargo test --workspace + + - name: Public documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --workspace --no-deps + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 + with: + tool: cargo-llvm-cov + + - name: Install pinned branch-coverage toolchain + run: rustup toolchain install "$COVERAGE_TOOLCHAIN" --profile minimal --component llvm-tools-preview + + - name: Exact owned coverage + run: ./scripts/check_coverage.sh + + - name: Validate public JSON contract + run: | + npx --yes ajv-cli@5.0.0 compile \ + --spec=draft2020 \ + -s contracts/semantic-candidate.schema.json + npx --yes ajv-cli@5.0.0 test \ + --spec=draft2020 \ + -s contracts/semantic-candidate.schema.json \ + -d contracts/fixtures/semantic-candidate.valid.json \ + --valid + npx --yes ajv-cli@5.0.0 test \ + --spec=draft2020 \ + -s contracts/semantic-candidate.schema.json \ + -d contracts/fixtures/semantic-candidate.invalid-whitespace.json \ + --invalid + npx --yes ajv-cli@5.0.0 test \ + --spec=draft2020 \ + -s contracts/semantic-candidate.schema.json \ + -d contracts/fixtures/semantic-candidate.invalid-published-truth.json \ + --invalid + npx --yes ajv-cli@5.0.0 test \ + --spec=draft2020 \ + -s contracts/semantic-candidate.schema.json \ + -d contracts/fixtures/semantic-candidate.invalid-state-truth-mismatch.json \ + --invalid + + - name: Lockfile freshness + run: | + cargo generate-lockfile --locked + git ls-files --error-unmatch Cargo.lock >/dev/null + test -z "$(git status --porcelain=v1 --untracked-files=all -- Cargo.lock)" + + - name: Clean working tree + run: test -z "$(git status --porcelain=v1 --untracked-files=all)" diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c11a1dfc --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +*.profraw +coverage.json +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..67347566 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,23 @@ +# AGENTS.md — ConceptWeave + +Read the organization `ContextualWisdomLab/.github` master context and product goal directive before material work. Live GitHub state and this repository's accepted ADRs override remembered chat state. + +## Product boundary + +ConceptWeave owns automatic, evidence-bound **Semantic Model Engineering**. Do not turn it into a semantic catalog, lineage engine, generic LLM gateway, browser crawler, or another product's system of record. + +## Development rules + +- Apply DDD continuously; maintain the Context Map and Ubiquitous Language. +- Rust 1.98.0 is the production baseline for core logic. Production mathematical/vector/model-scoring computation, if introduced, remains Rust-first. +- `conceptweave-domain` has no provider/network/database dependencies. +- External products and providers enter through versioned ports and Anti-Corruption Layers. +- LLM work uses `contextual-orchestrator`; model output is proposal evidence, never semantic authority. +- No direct cross-service application-table SQL. +- New database objects, when introduced, use descriptive two-or-more-word `snake_case` names and 3NF by default. +- Preserve source evidence, truth status, and publication state separately. +- Published semantic truth is immutable; correction uses supersession/new release. +- Public Rust APIs require beginner-readable documentation. +- Owned production coverage target is 100% line/function/region/branch where tooling exposes it. +- Never suppress deprecation warnings; fix causes. +- Never force-push shared branches, self-approve, fabricate checks, or weaken branch protection. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..c5dd4993 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,77 @@ +# ConceptWeave Architecture + +## Product responsibility + +ConceptWeave owns the process that turns observed enterprise evidence into governed semantic-model releases. It does not own source-system truth or downstream catalog/query experiences. + +```mermaid +flowchart LR + S[Source systems and artifacts] --> O[Source Observation] + O --> D[Semantic Discovery] + D --> V[Model Validation] + V --> G[Governance & Publication] + G --> P[Versioned semantic release] + + CO[contextual-orchestrator] -. proposal assistance .-> D + LW[LineageWeave] -. inferred/proposed lineage .-> O + CG[context-graph-contracts] -. shared graph/provenance contracts .-> P + P --> SDP[semantic-data-portal] + P --> GRC[governance-risk-compliance] + P --> EA[enterprise-architecture-core] +``` + +## DDD context map + +| Context | Type | Owns | Does not own | +| --- | --- | --- | --- | +| Source Observation | Supporting | immutable observations, parser receipts, evidence locations | source-system business truth | +| Semantic Discovery | Core | candidate generation and evidence binding | publication authority | +| Model Validation | Supporting | deterministic validation reports | human review decisions | +| Governance & Publication | Core | proposal lifecycle, review receipts, releases, supersession | catalog/search runtime | +| Interoperability | Supporting | versioned import/export and ACL adapters | foreign product internals | + +## Aggregate boundaries + +### SemanticCandidate + +Smallest consistency boundary for a single proposed semantic artifact and its evidence-bound publication state. It cannot jump directly from Draft to Published. + +### SemanticModelRelease (planned) + +Immutable publication aggregate containing approved candidate identities, release version, artifact digests, validation receipts, reviewer receipts, and supersession metadata. It will reference candidates rather than copy foreign source records. + +## Truth model + +- `observed`: exact source fact; +- `inferred`: derived candidate; +- `proposed`: submitted for governance; +- `authoritative`: explicitly reviewed and published; +- `superseded`: formerly authoritative and replaced; +- `rejected`: explicitly rejected. + +Truth status and publication workflow are distinct. A source observation can be authoritative in its source domain without making an inferred semantic interpretation authoritative. + +## Integration boundaries + +- `contextual-orchestrator`: LLM/model routing only. +- `LineageWeave`: inferred/proposed lineage evidence only. +- `semantic-data-portal`: published semantic artifact consumer/governance/catalog plane; it is not ConceptWeave's internal database. +- `context-graph-contracts`: shared cross-product identifiers, truth/provenance/event contracts where adopted. +- Keyverse: future identity/tenant authentication boundary. + +No direct cross-service application-table SQL is permitted. + +## Foundation directory structure + +```text +crates/ + conceptweave-domain/ # Core domain contract only +contracts/ # Versioned public schemas +docs/ + adr/ # Binding architecture decisions + doctoring/ # Standards/research evidence +scripts/ # Deterministic repository-quality helpers +.github/workflows/ # CI evidence +``` + +Adapters and application services are added only when their bounded responsibility exists; generic `utils`, `helpers`, or `services` dumping grounds are prohibited. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..8910d6fa --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to ConceptWeave are documented here. + +## Unreleased + +### Added + +- Initial ConceptWeave product, DDD, security, test, and operability baselines. +- Rust 1.98.0 `conceptweave-domain` foundation with evidence-bound semantic candidate contracts. +- Fail-closed Draft -> Proposed -> Validated -> Reviewed -> Published lifecycle with explicit rejection and supersession. +- Draft 2020-12 JSON Schema for the semantic-candidate public contract. +- Standards and research doctoring covering stable W3C ontology standards, 2026 RDF/SHACL work in progress, Apache Ossie, and recent LLM ontology-engineering research. + +### Security + +- Model-generated semantics remain non-authoritative until deterministic validation and authorized review. +- Unsafe Rust is forbidden in the core domain crate. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..d8db2650 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +# CLAUDE.md — ConceptWeave + +Follow `AGENTS.md`, `ARCHITECTURE.md`, accepted ADRs, and the organization master context before making changes. + +ConceptWeave's core invariant is: **inference is not authority**. Every generated concept, relation, constraint, dimension, measure, or physical mapping must retain evidence and pass the explicit governance lifecycle before publication. + +Keep domain logic in bounded domain modules, LLM/provider logic behind ports/adapters, and source/consumer systems independent. Prefer deterministic validation and explicit abstention over plausible unsupported output. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..451324f0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "conceptweave-domain" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..0eec8e8c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[workspace] +members = ["crates/conceptweave-domain"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.98" +repository = "https://github.com/ContextualWisdomLab/ConceptWeave" +license = "Apache-2.0" diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/OPERABILITY.md b/OPERABILITY.md new file mode 100644 index 00000000..b3302610 --- /dev/null +++ b/OPERABILITY.md @@ -0,0 +1,23 @@ +# Operability Baseline + +ConceptWeave has no production network service or durable database in the foundation slice. This document defines requirements before either is introduced. + +## Runtime requirements + +- explicit startup/readiness/liveness semantics; +- bounded source job queues, deadlines, cancellation, retry classification, and idempotency; +- persistent job receipts before accepting asynchronous work; +- OpenTelemetry sender/receiver ownership documented using the CWL shared observability contract; +- detailed structured error messages with safe identifiers, failure boundary, cause code, retryability, impact, and next action; +- no secrets or unnecessary raw PII in telemetry; +- backup/restore and migration rehearsal before durable persistence is production-ready; +- graceful drain of source parsing, model calls, validation, and publication jobs; +- deterministic replay from immutable source snapshot + extractor/config revisions. + +## Degraded modes + +- LLM unavailable: deterministic observation/validation remains available; discovery may return a typed `model_assistance_unavailable` result rather than fabricate candidates. +- external research unavailable: internal source modeling remains available and reports the missing evidence channel. +- downstream catalog unavailable: publication retains a durable release/outbox receipt and does not lose the governed release. + +Concrete SLO/RPO/RTO values require measured runtime evidence and are not guessed in the foundation. diff --git a/README.md b/README.md index f6c0a4d6..afb42260 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,172 @@ # ConceptWeave -Automatic ontology and semantic-layer engineering for governed enterprise meaning. +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) -> This repository was initialized with a minimal protected baseline. Substantive architecture and implementation changes are introduced through pull requests. +**Automatic, evidence-bound ontology and semantic-layer engineering for governed enterprise meaning.** + +ConceptWeave turns heterogeneous enterprise evidence—schemas, APIs, event contracts, documents, code structure, vocabularies, and lineage—into **reviewable semantic-model candidates**. Generated meaning never becomes authoritative merely because a model proposed it: candidates retain source evidence, pass deterministic validation, and move through an explicit review/publication lifecycle. + +## Why it exists + +Enterprise semantic models are valuable only when teams can explain where meaning came from, what was inferred, who reviewed it, and what is actually published. ConceptWeave makes that lifecycle explicit instead of collapsing discovery, generation, governance, and publication into one opaque step. + +| Need | What ConceptWeave provides | +| --- | --- | +| Semantic discovery | Evidence-bound candidate concepts, relations, dimensions, measures, constraints, and mappings | +| Governance | Separate truth status from publication state with explicit review before publication | +| Traceability | Exact source-evidence bindings carried with semantic candidates | +| Deterministic validation | Machine-checkable structural and lifecycle invariants before authority changes | +| Interoperability | Versioned semantic packages and explicit integration boundaries | +| Safe LLM assistance | Proposal assistance only; model output is never publication authority | + +## Product boundary + +ConceptWeave owns the semantic-model engineering lifecycle: + +```text +observe → discover → propose → validate → review → publish +``` + +Adjacent responsibilities remain separate: + +- [`semantic-data-portal`](https://github.com/ContextualWisdomLab/semantic-data-portal) owns published semantic catalog, governance, and consumption surfaces. +- [`LineageWeave`](https://github.com/ContextualWisdomLab/LineageWeave) provides inferred/proposed lineage evidence. +- `context-graph-contracts` owns cross-product provider-neutral graph/event interoperability contracts. +- [`contextual-orchestrator`](https://github.com/ContextualWisdomLab/contextual-orchestrator) owns LLM/provider discovery and routing. +- Source systems remain authoritative for their own business data. + +External source-analysis tools can sit behind adapters, but no external fork or model output becomes ConceptWeave product authority. + +## First vertical + +The first product vertical is **relational schema → governed semantic-model proposal**: + +1. ingest an immutable schema snapshot; +2. derive observed physical entities and relationships; +3. propose concepts, semantic/taxonomy relations, dimensions, measures, constraints, and physical mappings; +4. bind every proposal to exact evidence; +5. validate structure and consistency; +6. require authorized review; and +7. publish a versioned semantic package only after the lifecycle permits it. + +Future publication adapters may target standards such as OWL/RDFS/SKOS, SHACL, and JSON-LD. Emerging formats such as Apache Ossie are tracked as evolving interoperability targets rather than represented as finalized standards. + +## Current implementation + +The current foundation establishes the reusable domain and governance core rather than claiming the entire product is complete. + +Implemented in this branch: + +- Rust workspace and `conceptweave-domain` core; +- evidence-bound `SemanticCandidate` contract; +- independent truth-status and publication-state semantics; +- fail-closed candidate lifecycle with rejection and supersession paths; +- Draft 2020-12 JSON Schema for the public candidate contract; +- DDD Context Map and Ubiquitous Language; +- architecture, PRD/TRD, ADR, security, test, operability, and research baselines; +- pinned product CI for formatting, Clippy, tests, rustdoc, coverage, schema validation, lock freshness, and clean-tree checks. + +Source adapters, LLM-assisted induction, persistence, reasoning, review UI, and publication adapters remain explicit product gaps until they land with evidence. + +## Quick start + +The repository is pinned to Rust 1.98.0. The current domain core has no third-party runtime dependencies. + +```bash +cargo test --workspace +``` + +Run the full local quality set used by the foundation contract: + +```bash +cargo fmt --all --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +cargo doc --workspace --no-deps +``` + +The repository CI also validates the JSON Schema, lock/toolchain freshness, documentation contracts, and coverage expectations defined by the current source. + +## Core contract + +A semantic candidate is not the same thing as published semantic truth. + +```text +Observed evidence + │ + ▼ +Semantic candidate + │ + ├─ Draft + ├─ Proposed + ├─ Validated + ├─ Reviewed + └─ Published +``` + +Publication is an authority boundary. A candidate must preserve the evidence and lifecycle invariants required by the current domain contract; callers must not bypass those invariants by mutating public state or treating a validated proposal as published truth. + +The machine-readable public shape is in [`contracts/semantic-candidate.schema.json`](contracts/semantic-candidate.schema.json). + +## Architecture at a glance + +```text +Enterprise evidence + schemas · APIs · events · docs · code · vocabularies · lineage + │ + ▼ +┌──────────────────────────────────┐ +│ ConceptWeave │ +│ semantic-model engineering │ +├──────────────────────────────────┤ +│ observe / evidence normalization │ +│ candidate discovery & proposal │ +│ deterministic validation │ +│ review / publication lifecycle │ +└───────────────┬──────────────────┘ + │ versioned published semantics + ▼ + catalog / analytics / ontology consumers +``` + +ConceptWeave is the owner of semantic candidate engineering and publication lifecycle rules—not the catalog UI, source-system truth, LLM provider layer, lineage inference engine, or enterprise-wide application data. + +## Standards and research posture + +Stable standards and recommendations are distinguished from drafts and emerging specifications. LLM-assisted ontology engineering is treated as proposal assistance and must pass deterministic validation plus authorized review before publication. + +The standards/research register and design implications live in [`docs/doctoring/`](docs/doctoring/) and are linked through the repository traceability documents. + +## Documentation map + +| Goal | Start here | +| --- | --- | +| Product requirements | [`docs/PRD.md`](docs/PRD.md) | +| Technical requirements | [`docs/TRD.md`](docs/TRD.md) | +| Architecture | [`ARCHITECTURE.md`](ARCHITECTURE.md) | +| Bounded contexts | [`docs/CONTEXT_MAP.md`](docs/CONTEXT_MAP.md) | +| Domain language | [`docs/UBIQUITOUS_LANGUAGE.md`](docs/UBIQUITOUS_LANGUAGE.md) | +| Lifecycle / sequence views | [`docs/UML.md`](docs/UML.md) | +| Architecture decisions | [`docs/adr/README.md`](docs/adr/README.md) | +| Security | [`SECURITY.md`](SECURITY.md) | +| Test strategy | [`TEST_STRATEGY.md`](TEST_STRATEGY.md) | +| Operations | [`OPERABILITY.md`](OPERABILITY.md) | +| Current product/technical gaps | [`docs/product-technical-gap-baseline.md`](docs/product-technical-gap-baseline.md) | +| Documentation home | [`docs/index.md`](docs/index.md) | + +## Product principles + +1. **Evidence before authority.** Semantic meaning remains traceable to source evidence. +2. **Proposal is not publication.** Discovery and LLM assistance cannot self-authorize semantic truth. +3. **Deterministic gates matter.** Lifecycle and structural invariants are executable contracts. +4. **Product boundaries stay explicit.** Integrations use contracts rather than copying adjacent product responsibilities. +5. **Standards claims stay precise.** Drafts and emerging specifications are never presented as stable standards. +6. **Current source is the truth boundary.** Planned adapters and open-PR behavior are not described as already shipped. + +## Contributing + +Before changing the domain contract or lifecycle, read [`AGENTS.md`](AGENTS.md), the PRD/TRD, architecture, applicable ADRs, and the current product-gap baseline. Behavioral changes should preserve the repository's test-first and evidence-bound publication discipline and update the matching public contracts/documentation in the same change. + +## License + +ConceptWeave is licensed under the [Apache License 2.0](LICENSE). Third-party tools and future adapters retain their own licenses and must satisfy the repository's commercial-use and attribution policy before incorporation. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..5d930ad0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Baseline + +## Trust boundaries + +All source artifacts, generated candidate payloads, external ontology files, model outputs, and future web-retrieved content are untrusted input. + +## Required controls + +- source size, type, nesting, archive/decompression, and parser-time bounds; +- immutable source digests and parser/extractor provenance; +- no credentials, secrets, tokens, DSNs, or raw authorization material in semantic evidence; +- prompt-injection text is source data, never tool or policy instruction; +- LLM calls only through `contextual-orchestrator` with minimum necessary context; +- outbound retrieval, when introduced, uses a reviewed SSRF/DNS-rebinding-safe CWL egress boundary; +- no source-system writes from discovery or validation; +- reviewed authorization required before publication; +- future tenant isolation applies to source snapshots, candidates, review receipts, releases, exports, and object storage; +- published semantic truth is immutable: a published artifact must never be overwritten in place, including when an audit trail exists; corrections are issued as a new release that explicitly supersedes the prior release while retaining both releases and their provenance. + +## Threats tracked from foundation + +1. semantic poisoning by malicious source text; +2. hallucinated concepts/relations treated as facts; +3. ontology import cycles or reasoning/resource exhaustion; +4. unsafe generated query/expression execution; +5. cross-tenant evidence exposure; +6. provenance stripping during export; +7. malicious or oversized schema/API artifacts; +8. external-source SSRF or credential leakage; +9. model/provider compromise or unexpected retention; +10. governance bypass from Proposed/Validated directly to Published; +11. in-place mutation or overwrite of previously published semantic truth. + +Security findings become tests before the related runtime capability can be marked release-ready. diff --git a/TEST_STRATEGY.md b/TEST_STRATEGY.md new file mode 100644 index 00000000..6d137795 --- /dev/null +++ b/TEST_STRATEGY.md @@ -0,0 +1,40 @@ +# Test Strategy + +## Foundation gates + +- Rust formatting and Clippy with warnings denied; +- unit tests for every domain lifecycle branch; +- owned production line/function/region and LLVM branch coverage target of 100%; +- JSON Schema syntax validation; +- lockfile freshness and clean-tree verification; +- public Rust documentation with `missing_docs` denied. + +## Future product test families + +### Source observation + +Realistic PostgreSQL schema snapshots, OpenAPI/AsyncAPI fixtures, malformed contracts, deep nesting, invalid encoding, duplicate identifiers, archive bombs, parser cancellation, and exact digest/location provenance. + +### Ontology and semantic discovery + +Golden concept/type/taxonomy/relation sets; mapping precision/recall; multilingual labels; synonyms/homonyms; false friends; unrelated sources; cross-domain collisions; explicit no-answer cases. + +### Semantic measures + +Exact deterministic calculations, grain correctness, join/cardinality safety, units, null semantics, time windows, currency/unit conversions through approved deterministic layers, and no LLM arithmetic authority. + +### Validation/reasoning + +OWL consistency where supported, SHACL conformance, cycle constraints, unsatisfiable classes, contradictory ranges/domains, duplicate measures, and bounded reasoner resources. + +### Governance + +No bypass of Reviewed before Published, immutable published releases, rejection, supersession, stale-review protection, maker-checker requirements where configured, and exact audit receipts. + +### Security + +Prompt injection, malicious ontology/source content, SSRF, cross-tenant leakage, secret leakage, expression injection, resource exhaustion, replay, malformed source provenance, and hostile export values. + +### Evaluation + +Model-backed evaluation must include deterministic fixtures and human-reviewed expert cases. Report extraction recall, semantic precision, structural validity, mapping accuracy, citation/provenance completeness, and abstention quality separately rather than collapsing them into one opaque score. diff --git a/contracts/fixtures/semantic-candidate.invalid-published-truth.json b/contracts/fixtures/semantic-candidate.invalid-published-truth.json new file mode 100644 index 00000000..7bcebe21 --- /dev/null +++ b/contracts/fixtures/semantic-candidate.invalid-published-truth.json @@ -0,0 +1,13 @@ +{ + "candidate_id": "candidate-1", + "kind": "concept", + "truth_status": "observed", + "publication_state": "published", + "evidence": [ + { + "source_id": "source-1", + "source_digest": "sha256:abc", + "location": "public.orders" + } + ] +} diff --git a/contracts/fixtures/semantic-candidate.invalid-state-truth-mismatch.json b/contracts/fixtures/semantic-candidate.invalid-state-truth-mismatch.json new file mode 100644 index 00000000..759a0518 --- /dev/null +++ b/contracts/fixtures/semantic-candidate.invalid-state-truth-mismatch.json @@ -0,0 +1,13 @@ +{ + "candidate_id": "candidate-draft-authoritative", + "kind": "concept", + "truth_status": "authoritative", + "publication_state": "draft", + "evidence": [ + { + "source_id": "source-1", + "source_digest": "sha256:abc", + "location": "public.orders" + } + ] +} diff --git a/contracts/fixtures/semantic-candidate.invalid-whitespace.json b/contracts/fixtures/semantic-candidate.invalid-whitespace.json new file mode 100644 index 00000000..b3097f73 --- /dev/null +++ b/contracts/fixtures/semantic-candidate.invalid-whitespace.json @@ -0,0 +1,13 @@ +{ + "candidate_id": " ", + "kind": "concept", + "truth_status": "authoritative", + "publication_state": "published", + "evidence": [ + { + "source_id": "source-1", + "source_digest": "sha256:abc", + "location": "public.orders" + } + ] +} diff --git a/contracts/fixtures/semantic-candidate.valid.json b/contracts/fixtures/semantic-candidate.valid.json new file mode 100644 index 00000000..69797936 --- /dev/null +++ b/contracts/fixtures/semantic-candidate.valid.json @@ -0,0 +1,13 @@ +{ + "candidate_id": "candidate-1", + "kind": "concept", + "truth_status": "authoritative", + "publication_state": "published", + "evidence": [ + { + "source_id": "source-1", + "source_digest": "sha256:abc", + "location": "public.orders" + } + ] +} diff --git a/contracts/semantic-candidate.schema.json b/contracts/semantic-candidate.schema.json new file mode 100644 index 00000000..3b0b9c0b --- /dev/null +++ b/contracts/semantic-candidate.schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.contextualwisdomlab.org/conceptweave/semantic-candidate/0.1.0", + "title": "ConceptWeave Semantic Candidate", + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "kind", + "truth_status", + "publication_state", + "evidence" + ], + "properties": { + "candidate_id": { + "type": "string", + "minLength": 1, + "pattern": ".*\\S.*" + }, + "kind": { + "enum": [ + "concept", + "taxonomy_relation", + "semantic_relation", + "constraint", + "dimension", + "measure", + "physical_mapping" + ] + }, + "truth_status": { + "enum": [ + "observed", + "inferred", + "proposed", + "authoritative", + "superseded", + "rejected" + ] + }, + "publication_state": { + "enum": [ + "draft", + "proposed", + "validated", + "reviewed", + "published", + "superseded", + "rejected" + ] + }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["source_id", "source_digest", "location"], + "properties": { + "source_id": { + "type": "string", + "minLength": 1, + "pattern": ".*\\S.*" + }, + "source_digest": { + "type": "string", + "minLength": 1, + "pattern": ".*\\S.*" + }, + "location": { + "type": "string", + "minLength": 1, + "pattern": ".*\\S.*" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "publication_state": {"const": "draft"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "inferred"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "proposed"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "proposed"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "validated"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "inferred"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "reviewed"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "inferred"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "published"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "authoritative"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "superseded"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "superseded"} + } + } + }, + { + "if": { + "properties": { + "publication_state": {"const": "rejected"} + }, + "required": ["publication_state"] + }, + "then": { + "properties": { + "truth_status": {"const": "rejected"} + } + } + } + ] +} diff --git a/crates/conceptweave-domain/Cargo.toml b/crates/conceptweave-domain/Cargo.toml new file mode 100644 index 00000000..8f9ce7b2 --- /dev/null +++ b/crates/conceptweave-domain/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "conceptweave-domain" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true +description = "Core domain contracts for governed ontology and semantic-layer engineering" + +[lib] +path = "src/lib.rs" diff --git a/crates/conceptweave-domain/src/lib.rs b/crates/conceptweave-domain/src/lib.rs new file mode 100644 index 00000000..b4a8aa6e --- /dev/null +++ b/crates/conceptweave-domain/src/lib.rs @@ -0,0 +1,525 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Core domain contracts for ConceptWeave. +//! +//! ConceptWeave separates observed source evidence from inferred semantic +//! candidates and from reviewed, published semantic-model truth. This crate +//! contains only that domain contract; adapters, persistence, LLM orchestration, +//! and publication formats belong to other bounded contexts. + +use core::fmt; + +/// The kind of source material observed by ConceptWeave. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceKind { + /// A relational schema or database-introspection snapshot. + RelationalSchema, + /// An OpenAPI contract. + OpenApi, + /// An AsyncAPI or event contract. + AsyncApi, + /// Human-authored documentation or a business glossary. + Document, + /// Source-code structure observed through a bounded adapter. + SourceCode, + /// An existing ontology or controlled vocabulary. + ExistingOntology, + /// Provenance or lineage evidence produced by another system. + Lineage, +} + +/// The semantic artifact that a candidate proposes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CandidateKind { + /// A domain concept or class. + Concept, + /// A broader/narrower or parent/child taxonomy relation. + TaxonomyRelation, + /// A non-taxonomic semantic relation or object property. + SemanticRelation, + /// A data-quality, cardinality, or semantic constraint. + Constraint, + /// An analytical dimension. + Dimension, + /// A governed analytical measure or metric definition. + Measure, + /// A mapping between a physical source element and a semantic concept. + PhysicalMapping, +} + +/// The epistemic status of a fact or relation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TruthStatus { + /// Directly observed from a source without semantic inference. + Observed, + /// Derived by deterministic or model-assisted inference. + Inferred, + /// Explicitly proposed for governance review. + Proposed, + /// Approved and published by the owning governance process. + Authoritative, + /// Previously authoritative but replaced by a newer fact or release. + Superseded, + /// Explicitly rejected by validation or governance review. + Rejected, +} + +/// The governance state of a semantic candidate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublicationState { + /// Newly discovered and not yet submitted for validation. + Draft, + /// Submitted as a candidate for validation. + Proposed, + /// Passed deterministic validation and consistency checks. + Validated, + /// Reviewed by an authorized semantic steward or equivalent workflow. + Reviewed, + /// Published as governed semantic truth. + Published, + /// Replaced by a later published release. + Superseded, + /// Rejected and no longer eligible for publication. + Rejected, +} + +/// A stable reference to the evidence supporting a semantic candidate. +/// +/// Evidence identity is immutable outside this crate. Callers must construct a +/// reference through [`EvidenceReference::new`], which rejects blank identity +/// fields, and can inspect values only through read-only accessors. +/// +/// ```compile_fail +/// use conceptweave_domain::EvidenceReference; +/// +/// let reference = EvidenceReference { +/// source_id: "source-1".into(), +/// source_digest: "sha256:abc".into(), +/// location: "public.orders".into(), +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EvidenceReference { + source_id: String, + source_digest: String, + location: String, +} + +impl EvidenceReference { + /// Creates an evidence reference, rejecting blank identity fields. + pub fn new( + source_id: impl Into, + source_digest: impl Into, + location: impl Into, + ) -> Result { + let reference = Self { + source_id: source_id.into(), + source_digest: source_digest.into(), + location: location.into(), + }; + reference.validate()?; + Ok(reference) + } + + /// Returns the stable identifier of the observed source snapshot or artifact. + pub fn source_id(&self) -> &str { + &self.source_id + } + + /// Returns the content digest of the exact source revision used as evidence. + pub fn source_digest(&self) -> &str { + &self.source_digest + } + + /// Returns the human- and machine-readable location within the source artifact. + pub fn location(&self) -> &str { + &self.location + } + + fn validate(&self) -> Result<(), ContractError> { + if self.source_id.trim().is_empty() { + return Err(ContractError::EmptyField("source_id")); + } + if self.source_digest.trim().is_empty() { + return Err(ContractError::EmptyField("source_digest")); + } + if self.location.trim().is_empty() { + return Err(ContractError::EmptyField("location")); + } + Ok(()) + } +} + +/// A governed candidate for an ontology or semantic-layer artifact. +/// +/// External consumers cannot mutate evidence or governance state without a +/// validated domain operation. The public transition API deliberately stops at +/// the semantic-steward boundary: entering `Reviewed`, entering `Published`, or +/// superseding/rejecting an already reviewed artifact requires the separate +/// Governance & Publication context to establish authority first. +/// +/// ```compile_fail +/// use conceptweave_domain::{CandidateKind, EvidenceReference, SemanticCandidate}; +/// +/// let evidence = EvidenceReference::new("source-1", "sha256:abc", "public.orders").unwrap(); +/// let mut candidate = SemanticCandidate::new("candidate-1", CandidateKind::Concept, vec![evidence]).unwrap(); +/// candidate.evidence.clear(); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SemanticCandidate { + candidate_id: String, + kind: CandidateKind, + truth_status: TruthStatus, + publication_state: PublicationState, + evidence: Vec, +} + +impl SemanticCandidate { + /// Creates an inferred draft candidate with at least one valid evidence reference. + pub fn new( + candidate_id: impl Into, + kind: CandidateKind, + evidence: Vec, + ) -> Result { + let candidate_id = candidate_id.into(); + if candidate_id.trim().is_empty() { + return Err(ContractError::EmptyField("candidate_id")); + } + validate_evidence(&evidence)?; + let publication_state = PublicationState::Draft; + Ok(Self { + candidate_id, + kind, + truth_status: truth_for_state(publication_state), + publication_state, + evidence, + }) + } + + /// Returns the stable candidate identifier. + pub fn candidate_id(&self) -> &str { + &self.candidate_id + } + + /// Returns the semantic artifact kind proposed by this candidate. + pub fn kind(&self) -> CandidateKind { + self.kind + } + + /// Returns the candidate's current epistemic status. + pub fn truth_status(&self) -> TruthStatus { + self.truth_status + } + + /// Returns the candidate's current governance/publication state. + pub fn publication_state(&self) -> PublicationState { + self.publication_state + } + + /// Returns the immutable evidence references supporting this candidate. + pub fn evidence(&self) -> &[EvidenceReference] { + &self.evidence + } + + /// Moves a candidate through transitions that do not require steward authority. + /// + /// Deterministic discovery and validation code may propose, validate, or + /// reject a candidate. Crossing into a steward-reviewed or published state, + /// or changing an already reviewed/published artifact, fails closed until + /// the Governance & Publication bounded context supplies an authorized path. + pub fn transition(&mut self, target: PublicationState) -> Result<(), ContractError> { + let from = self.publication_state; + if !ALLOWED_TRANSITIONS.contains(&(from, target)) { + return Err(ContractError::InvalidTransition { from, to: target }); + } + if requires_governance_authorization(from, target) { + return Err(ContractError::GovernanceAuthorizationRequired { target }); + } + self.publication_state = target; + self.truth_status = truth_for_state(target); + Ok(()) + } + + /// Returns whether the current reviewed candidate has valid publication evidence. + /// + /// This is an eligibility check only. It does not grant steward authority or + /// publish the candidate. + pub fn is_publishable(&self) -> bool { + self.publication_state == PublicationState::Reviewed + && validate_evidence(&self.evidence).is_ok() + } +} + +fn validate_evidence(evidence: &[EvidenceReference]) -> Result<(), ContractError> { + if evidence.is_empty() { + return Err(ContractError::MissingEvidence); + } + for reference in evidence { + reference.validate()?; + } + Ok(()) +} + +fn requires_governance_authorization(from: PublicationState, target: PublicationState) -> bool { + matches!( + (from, target), + (PublicationState::Validated, PublicationState::Reviewed) + | (PublicationState::Reviewed, PublicationState::Published) + | (PublicationState::Reviewed, PublicationState::Rejected) + | (PublicationState::Published, PublicationState::Superseded) + ) +} + +const ALLOWED_TRANSITIONS: &[(PublicationState, PublicationState)] = &[ + (PublicationState::Draft, PublicationState::Proposed), + (PublicationState::Draft, PublicationState::Rejected), + (PublicationState::Proposed, PublicationState::Validated), + (PublicationState::Proposed, PublicationState::Rejected), + (PublicationState::Validated, PublicationState::Reviewed), + (PublicationState::Validated, PublicationState::Rejected), + (PublicationState::Reviewed, PublicationState::Published), + (PublicationState::Reviewed, PublicationState::Rejected), + (PublicationState::Published, PublicationState::Superseded), +]; + +fn truth_for_state(state: PublicationState) -> TruthStatus { + match state { + PublicationState::Draft | PublicationState::Validated | PublicationState::Reviewed => { + TruthStatus::Inferred + } + PublicationState::Proposed => TruthStatus::Proposed, + PublicationState::Published => TruthStatus::Authoritative, + PublicationState::Superseded => TruthStatus::Superseded, + PublicationState::Rejected => TruthStatus::Rejected, + } +} + +/// A domain-contract validation failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ContractError { + /// A required identity or evidence field was blank. + EmptyField(&'static str), + /// A candidate was created without any supporting evidence. + MissingEvidence, + /// A caller attempted to cross a steward-governed lifecycle boundary. + GovernanceAuthorizationRequired { + /// Requested state that requires the Governance & Publication context. + target: PublicationState, + }, + /// A governance state transition attempted to skip or reverse required review. + InvalidTransition { + /// State before the rejected transition. + from: PublicationState, + /// Requested target state. + to: PublicationState, + }, +} + +impl fmt::Display for ContractError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyField(field) => write!(formatter, "required field `{field}` is blank"), + Self::MissingEvidence => { + write!(formatter, "semantic candidates require source evidence") + } + Self::GovernanceAuthorizationRequired { target } => write!( + formatter, + "publication state {target:?} requires authorized governance" + ), + Self::InvalidTransition { from, to } => write!( + formatter, + "publication transition from {from:?} to {to:?} is not permitted" + ), + } + } +} + +impl std::error::Error for ContractError {} + +#[cfg(test)] +mod tests { + use super::*; + + fn evidence() -> EvidenceReference { + EvidenceReference::new("source-1", "sha256:abc", "schema.orders.total").unwrap() + } + + fn candidate() -> SemanticCandidate { + SemanticCandidate::new("candidate-1", CandidateKind::Concept, vec![evidence()]).unwrap() + } + + fn reviewed_candidate_for_governance_test() -> SemanticCandidate { + let mut candidate = candidate(); + candidate.publication_state = PublicationState::Reviewed; + candidate.truth_status = truth_for_state(PublicationState::Reviewed); + candidate + } + + #[test] + fn evidence_reference_accepts_valid_values() { + let reference = evidence(); + assert_eq!(reference.source_id(), "source-1"); + assert_eq!(reference.source_digest(), "sha256:abc"); + assert_eq!(reference.location(), "schema.orders.total"); + } + + #[test] + fn evidence_reference_rejects_each_blank_field() { + assert_eq!( + EvidenceReference::new(" ", "digest", "location"), + Err(ContractError::EmptyField("source_id")) + ); + assert_eq!( + EvidenceReference::new("source", " ", "location"), + Err(ContractError::EmptyField("source_digest")) + ); + assert_eq!( + EvidenceReference::new("source", "digest", " "), + Err(ContractError::EmptyField("location")) + ); + } + + #[test] + fn candidate_requires_identity_and_evidence() { + assert_eq!( + SemanticCandidate::new(" ", CandidateKind::Concept, vec![evidence()]), + Err(ContractError::EmptyField("candidate_id")) + ); + assert_eq!( + SemanticCandidate::new("candidate", CandidateKind::Concept, vec![]), + Err(ContractError::MissingEvidence) + ); + } + + #[test] + fn candidate_accessors_expose_read_only_domain_state() { + let candidate = candidate(); + assert_eq!(candidate.candidate_id(), "candidate-1"); + assert_eq!(candidate.kind(), CandidateKind::Concept); + assert_eq!(candidate.truth_status(), TruthStatus::Inferred); + assert_eq!(candidate.publication_state(), PublicationState::Draft); + assert_eq!(candidate.evidence().len(), 1); + assert_eq!(candidate.evidence()[0].source_id(), "source-1"); + } + + #[test] + fn deterministic_lifecycle_stops_at_governance_boundary() { + let mut candidate = candidate(); + candidate.transition(PublicationState::Proposed).unwrap(); + candidate.transition(PublicationState::Validated).unwrap(); + + assert_eq!( + candidate.transition(PublicationState::Reviewed), + Err(ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Reviewed, + }) + ); + assert_eq!(candidate.publication_state(), PublicationState::Validated); + assert_eq!(candidate.truth_status(), TruthStatus::Inferred); + } + + #[test] + fn lifecycle_rejects_skipped_and_post_rejection_transitions() { + let mut candidate = candidate(); + assert_eq!( + candidate.transition(PublicationState::Published), + Err(ContractError::InvalidTransition { + from: PublicationState::Draft, + to: PublicationState::Published, + }) + ); + candidate.transition(PublicationState::Rejected).unwrap(); + assert_eq!(candidate.truth_status(), TruthStatus::Rejected); + assert_eq!( + candidate.transition(PublicationState::Proposed), + Err(ContractError::InvalidTransition { + from: PublicationState::Rejected, + to: PublicationState::Proposed, + }) + ); + } + + #[test] + fn reviewed_and_published_state_changes_require_governance() { + let mut candidate = reviewed_candidate_for_governance_test(); + assert!(candidate.is_publishable()); + assert_eq!( + candidate.transition(PublicationState::Published), + Err(ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Published, + }) + ); + assert_eq!( + candidate.transition(PublicationState::Rejected), + Err(ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Rejected, + }) + ); + + candidate.publication_state = PublicationState::Published; + candidate.truth_status = truth_for_state(PublicationState::Published); + assert_eq!(candidate.truth_status(), TruthStatus::Authoritative); + assert_eq!( + candidate.transition(PublicationState::Superseded), + Err(ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Superseded, + }) + ); + } + + #[test] + fn publication_eligibility_rechecks_evidence() { + let mut candidate = reviewed_candidate_for_governance_test(); + assert!(candidate.is_publishable()); + + candidate.evidence.clear(); + assert!(!candidate.is_publishable()); + } + + #[test] + fn publication_eligibility_revalidates_each_evidence_reference() { + let mut candidate = reviewed_candidate_for_governance_test(); + candidate.evidence[0].source_id = " ".into(); + + assert!(!candidate.is_publishable()); + } + + #[test] + fn truth_mapping_preserves_published_and_superseded_semantics() { + assert_eq!( + truth_for_state(PublicationState::Published), + TruthStatus::Authoritative + ); + assert_eq!( + truth_for_state(PublicationState::Superseded), + TruthStatus::Superseded + ); + } + + #[test] + fn contract_errors_explain_the_failure() { + assert_eq!( + ContractError::EmptyField("field").to_string(), + "required field `field` is blank" + ); + assert_eq!( + ContractError::MissingEvidence.to_string(), + "semantic candidates require source evidence" + ); + assert_eq!( + ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Reviewed, + } + .to_string(), + "publication state Reviewed requires authorized governance" + ); + assert_eq!( + ContractError::InvalidTransition { + from: PublicationState::Draft, + to: PublicationState::Published, + } + .to_string(), + "publication transition from Draft to Published is not permitted" + ); + } +} diff --git a/crates/conceptweave-domain/tests/publication_eligibility.rs b/crates/conceptweave-domain/tests/publication_eligibility.rs new file mode 100644 index 00000000..73cdd0a2 --- /dev/null +++ b/crates/conceptweave-domain/tests/publication_eligibility.rs @@ -0,0 +1,11 @@ +use conceptweave_domain::{CandidateKind, EvidenceReference, SemanticCandidate}; + +#[test] +fn draft_candidate_is_not_publishable_before_governance_review() { + let evidence = EvidenceReference::new("source-1", "sha256:abc", "schema.orders.total") + .expect("valid evidence fixture"); + let candidate = SemanticCandidate::new("candidate-1", CandidateKind::Concept, vec![evidence]) + .expect("valid semantic candidate fixture"); + + assert!(!candidate.is_publishable()); +} diff --git a/crates/conceptweave-domain/tests/publication_invariants.rs b/crates/conceptweave-domain/tests/publication_invariants.rs new file mode 100644 index 00000000..f9571a61 --- /dev/null +++ b/crates/conceptweave-domain/tests/publication_invariants.rs @@ -0,0 +1,49 @@ +use conceptweave_domain::{ + CandidateKind, ContractError, EvidenceReference, PublicationState, SemanticCandidate, +}; + +fn validated_candidate() -> SemanticCandidate { + let evidence = EvidenceReference::new("source-1", "sha256:abc", "public.orders").unwrap(); + let mut candidate = + SemanticCandidate::new("candidate-1", CandidateKind::Concept, vec![evidence]).unwrap(); + + candidate.transition(PublicationState::Proposed).unwrap(); + candidate.transition(PublicationState::Validated).unwrap(); + candidate +} + +#[test] +fn evidence_remains_read_only_before_governance_review() { + let candidate = validated_candidate(); + + assert_eq!(candidate.publication_state(), PublicationState::Validated); + assert_eq!(candidate.evidence().len(), 1); + assert!(!candidate.is_publishable()); +} + +#[test] +fn external_callers_cannot_enter_reviewed_state_without_governance_authority() { + let mut candidate = validated_candidate(); + + assert_eq!( + candidate.transition(PublicationState::Reviewed), + Err(ContractError::GovernanceAuthorizationRequired { + target: PublicationState::Reviewed, + }) + ); + assert_eq!(candidate.publication_state(), PublicationState::Validated); +} + +#[test] +fn external_callers_cannot_publish_by_skipping_governance() { + let mut candidate = validated_candidate(); + + assert_eq!( + candidate.transition(PublicationState::Published), + Err(ContractError::InvalidTransition { + from: PublicationState::Validated, + to: PublicationState::Published, + }) + ); + assert_eq!(candidate.publication_state(), PublicationState::Validated); +} diff --git a/crates/conceptweave-domain/tests/research_reference_traceability.rs b/crates/conceptweave-domain/tests/research_reference_traceability.rs new file mode 100644 index 00000000..c64825e2 --- /dev/null +++ b/crates/conceptweave-domain/tests/research_reference_traceability.rs @@ -0,0 +1,30 @@ +const REFERENCES: &str = include_str!("../../../docs/doctoring/REFERENCES.md"); +const TRACEABILITY: &str = + include_str!("../../../docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md"); + +#[test] +fn adopted_alignment_studies_have_authoritative_apa_bibliography_records() { + for (traceability_marker, authoritative_record, publication_year_marker) in [ + ( + "He, Chen, Dong, & Horrocks (2023)", + "https://ceur-ws.org/Vol-3632/ISWC2023_paper_427.pdf", + "(2023).", + ), + ( + "Amini, Saki Norouzi, Hitzler, & Amini (2024)", + "https://doi.org/10.1007/978-3-031-81221-7_2", + "(2025).", + ), + ] { + assert!( + TRACEABILITY.contains(traceability_marker), + "an adopted study must remain explicit in research-to-capability traceability: {traceability_marker}" + ); + assert!( + REFERENCES.split("\n\n").any(|entry| { + entry.contains(authoritative_record) && entry.contains(publication_year_marker) + }), + "an adopted study must have an authoritative APA publication record with the publication year bound to that record: {authoritative_record} {publication_year_marker}" + ); + } +} diff --git a/docs/CONTEXT_MAP.md b/docs/CONTEXT_MAP.md new file mode 100644 index 00000000..5ea47792 --- /dev/null +++ b/docs/CONTEXT_MAP.md @@ -0,0 +1,16 @@ +# Context Map + +## Internal relationships + +- Source Observation -> Semantic Discovery: **Customer/Supplier**; Discovery consumes immutable observation contracts. +- Semantic Discovery -> Model Validation: **Conformist to published candidate contract**; validation must not rewrite discovery evidence. +- Model Validation -> Governance & Publication: **Customer/Supplier**; governance consumes deterministic validation receipts. +- Governance & Publication -> Interoperability: **Published Language**; adapters consume immutable release contracts. + +## External relationships + +- contextual-orchestrator -> Semantic Discovery: **Anti-Corruption Layer**. Model/provider envelopes never enter the domain model directly. +- LineageWeave -> Source Observation: **Anti-Corruption Layer**. Inferred/proposed lineage remains explicitly non-authoritative until ConceptWeave governance evaluates it. +- context-graph-contracts <-> Interoperability: **Shared Kernel only for versioned public contracts**, kept minimal. +- semantic-data-portal <- Interoperability: **Published Language**. SDP consumes releases; ConceptWeave does not read SDP application tables. +- Keyverse -> future delivery layer: **Anti-Corruption Layer** for verified identity/tenant context. diff --git a/docs/PRD.md b/docs/PRD.md new file mode 100644 index 00000000..0e68c400 --- /dev/null +++ b/docs/PRD.md @@ -0,0 +1,81 @@ +# ConceptWeave Product Requirements Document + +## 1. Product statement + +ConceptWeave converts heterogeneous enterprise evidence into a governed ontology and semantic layer without collapsing observed facts, model inference, and human-approved meaning into the same truth state. + +## 2. Buyer problem + +Enterprise teams repeatedly hand-build business glossaries, ontologies, metric definitions, semantic mappings, and data relationships from database schemas, API contracts, documents, and tribal knowledge. The work is slow, inconsistent across tools, difficult to audit, and unsafe to delegate entirely to an LLM because inferred semantics can be plausible but wrong. + +## 3. Primary buyers and users + +- enterprise data architects and semantic-modeling teams; +- data governance and catalog teams; +- analytics/BI platform owners; +- AI/RAG platform teams that require governed machine-readable context; +- risk/compliance and enterprise-architecture teams that need traceable semantic models. + +## 4. Core job to be done + +Given an enterprise source estate, produce a **reviewable semantic model proposal** in which every concept, relationship, constraint, dimension, measure, and physical mapping is linked to exact evidence and can be validated, rejected, reviewed, published, superseded, and reproduced. + +## 5. Functional requirements + +### FR-1 Source observation + +Accept immutable snapshots or versioned contracts for relational schema, OpenAPI, AsyncAPI/event models, documents/glossaries, source-code structure, existing ontology/vocabulary, and lineage/provenance. Raw source authority remains with its owning system. + +### FR-2 Candidate discovery + +Produce candidates for concepts, taxonomies, non-taxonomic relations, semantic constraints, dimensions, measures, and physical-to-semantic mappings. Each candidate starts as inferred rather than authoritative. + +### FR-3 Evidence and provenance + +The current v0.1 candidate contract requires every candidate to retain exact source identity, source digest, and source location through `EvidenceReference`. Issue #2 must add immutable Source Observation and proposal-receipt contracts that also retain observation time, parser/extractor revision, and discovery method before the first Generation release. Until those receipt contracts exist, the Rust `SemanticCandidate` and `contracts/semantic-candidate.schema.json` must not be described as already carrying those deferred coordinates. Unsupported candidates fail closed. + +### FR-4 Deterministic validation + +Validate syntax, identifiers, relationship cardinality, mapping completeness, duplicate/contradictory definitions, ontology consistency where supported, semantic-measure contracts, and publication schema before review. + +### FR-5 Governed review + +A candidate cannot become authoritative solely because an LLM or automated extractor produced it. The publication lifecycle is Draft -> Proposed -> Validated -> Reviewed -> Published, with explicit rejection and supersession paths. + +### FR-6 Publication + +Publish versioned artifacts for ontology and semantic-layer consumers while retaining the exact input snapshot and proposal/review receipts that produced the release. + +### FR-7 Interoperability + +Support stable adapters for `semantic-data-portal`, `LineageWeave`, `context-graph-contracts`, and other CWL products without direct cross-service application-table SQL. + +### FR-8 LLM assistance + +All LLM-backed induction uses `contextual-orchestrator`. Model output is untrusted proposal data and may not skip deterministic validation or review. + +## 6. First vertical slice + +Relational schema snapshot -> observed tables/columns/foreign keys -> concept/relation/dimension/measure/mapping candidates -> evidence-bound validation report -> reviewable proposal package. + +## 7. Non-goals for v0.1 + +- replacing `semantic-data-portal` as the enterprise catalog; +- arbitrary write access to source systems; +- automatic publication without review; +- treating vector similarity as semantic truth; +- copying every external ontology into one CWL namespace; +- building a generic LLM gateway or browser crawler; +- claiming an emerging draft semantic-layer format is a stable standard. + +## 8. Acceptance criteria for the first commercial candidate + +- 100% owned production line/function/region and branch coverage where tooling exposes it; +- candidate-to-source provenance completeness of 100%; +- zero publication paths that bypass reviewed state; +- zero silent inferred-to-authoritative promotion; +- deterministic replay of the same immutable source snapshot and extraction configuration; +- cross-tenant access denial when tenancy is introduced; +- malformed/hostile source contracts rejected with bounded resource use; +- semantic-model release can be reproduced from source receipts and approved proposal receipts; +- buyer can inspect why each published artifact exists and which evidence supported it. diff --git a/docs/TRD.md b/docs/TRD.md new file mode 100644 index 00000000..ad5e1415 --- /dev/null +++ b/docs/TRD.md @@ -0,0 +1,58 @@ +# ConceptWeave Technical Requirements Document + +## 1. Architectural style + +ConceptWeave starts as a Rust-first modular monolith with explicit bounded contexts and ports. Network-service extraction is deferred until independent scaling, trust, or deployment boundaries are demonstrated. + +## 2. Bounded contexts + +1. **Source Observation** — immutable source snapshots and parser receipts. +2. **Semantic Discovery** — evidence-bound candidate generation. +3. **Model Validation** — deterministic structural, ontology, constraint, and semantic-model validation. +4. **Governance & Publication** — review decisions, immutable releases, supersession. +5. **Interoperability** — import/export adapters and CWL anti-corruption layers. + +The Core Domain is **Semantic Model Engineering**, represented by the discovery-to-publication lifecycle. Identity, LLM routing, outbound web access, observability, and catalog consumption are external/generic responsibilities. + +## 3. Dependency direction + +`domain <- application <- ports/contracts <- adapters <- delivery` + +Domain code must not import web frameworks, databases, provider SDKs, LLM SDKs, or another CWL product's internals. + +## 4. Source observation contract + +Every observed source will eventually carry at least: + +- source snapshot identifier; +- source kind; +- immutable content digest; +- source authority; +- observed/recorded time; +- parser/extractor version; +- tenant/workspace scope when tenancy exists; +- bounded source locations for extracted evidence. + +## 5. Candidate contract + +The initial Rust and JSON contracts cover candidate kind, truth status, publication state, and source evidence. Later revisions add ontology IRIs, language-tagged labels, relation endpoints, cardinality, units, measure expressions, physical mappings, confidence/evaluation receipts, and temporal validity without breaking v0.1 consumers. + +## 6. LLM boundary + +LLM calls go through `contextual-orchestrator`. The application sends bounded evidence/context and receives structured proposals. LLM output is never a database command, publication decision, validation result, or source-system mutation. Deterministic checks must be able to reject the output without another model call. + +## 7. Standards strategy + +Stable publication targets use stable recommendations first: RDF 1.1, OWL 2, SKOS, SHACL 1.0, JSON-LD 1.1, and PROV-O as applicable. RDF 1.2 and SHACL 1.2 are tracked as 2026 drafts/candidate work and are not silently treated as final standards. Apache Ossie (incubating; formerly OSI) is tracked as an emerging semantic-model exchange format for metrics, dimensions, relationships, and datasets. + +## 8. Persistence + +No durable product database is claimed by the foundation slice. When persistence is introduced it must be PostgreSQL, 3NF by default, use descriptive two-or-more-word `snake_case` objects, preserve business/effective time separately from system-recorded time when facts vary over time, enforce tenant-scoped references, and use explicit migration ownership rather than runtime DDL races. + +## 9. Security + +Source artifacts are untrusted input. Adapters must enforce source size/type bounds, parser timeouts, archive/decompression limits, SSRF-safe outbound access where external retrieval exists, and prompt-injection isolation for LLM-assisted extraction. Credentials and raw secrets never become semantic evidence. + +## 10. Evaluation + +Evaluation must separate extraction recall, semantic correctness, structural correctness, ontology consistency, mapping accuracy, measure correctness, and governance outcomes. Model-judge scores may supplement but never replace deterministic golden fixtures and human-reviewed expert cases. diff --git a/docs/UBIQUITOUS_LANGUAGE.md b/docs/UBIQUITOUS_LANGUAGE.md new file mode 100644 index 00000000..0c1d0c25 --- /dev/null +++ b/docs/UBIQUITOUS_LANGUAGE.md @@ -0,0 +1,18 @@ +# Ubiquitous Language + +| Term | Meaning | +| --- | --- | +| Source Snapshot | Immutable revision of source evidence observed by ConceptWeave. | +| Observation | Deterministically extracted fact from a Source Snapshot. | +| Evidence Reference | Stable source identity, digest, and location supporting a candidate. | +| Semantic Candidate | Evidence-bound proposal for a concept, relation, constraint, dimension, measure, or physical mapping. | +| Semantic Model Proposal | Versioned collection of candidates presented for validation/review. | +| Validation Report | Deterministic result describing structural or semantic contract validity; not a review decision. | +| Review Decision | Authorized accept/reject decision over validated candidates or a model proposal. | +| Semantic Model Release | Immutable governed publication artifact. | +| Truth Status | Epistemic classification: observed, inferred, proposed, authoritative, superseded, rejected. | +| Publication State | Governance workflow state: draft, proposed, validated, reviewed, published, superseded, rejected. | +| Physical Mapping | Mapping from a physical schema/API/event element to a semantic concept or field. | +| Dimension | Governed categorical or temporal axis used to group/filter analytical facts. | +| Measure | Governed calculation with explicit expression, grain, units, null semantics, and evidence. | +| Semantic Steward | Authorized reviewer responsible for accepting or rejecting semantic meaning. | diff --git a/docs/UML.md b/docs/UML.md new file mode 100644 index 00000000..a9559e7f --- /dev/null +++ b/docs/UML.md @@ -0,0 +1,40 @@ +# UML and lifecycle views + +## Candidate state machine + +```mermaid +stateDiagram-v2 + [*] --> Draft + Draft --> Proposed + Draft --> Rejected + Proposed --> Validated + Proposed --> Rejected + Validated --> Reviewed + Validated --> Rejected + Reviewed --> Published + Reviewed --> Rejected + Published --> Superseded + Rejected --> [*] + Superseded --> [*] +``` + +## Foundation sequence + +```mermaid +sequenceDiagram + participant Source + participant Observation + participant Discovery + participant Validator + participant Steward + participant Publisher + + Source->>Observation: immutable snapshot + Observation->>Discovery: observations + evidence refs + Discovery->>Validator: inferred candidate proposal + Validator-->>Discovery: validation report + Validator->>Steward: validated proposal + Steward->>Publisher: reviewed acceptance + Publisher-->>Source: no source mutation + Publisher-->>Steward: immutable release receipt +``` diff --git a/docs/adr/0001-product-boundary.md b/docs/adr/0001-product-boundary.md new file mode 100644 index 00000000..e4a0d6c4 --- /dev/null +++ b/docs/adr/0001-product-boundary.md @@ -0,0 +1,22 @@ +# ADR 0001 — Product and bounded-context boundary + +**Status:** Accepted + +## Context + +CWL already has products that reconstruct lineage, operate semantic catalogs, define shared graph contracts, and route LLM calls. Placing automatic semantic-model engineering inside any one of those products would blur system-of-record and reuse boundaries. + +## Decision + +ConceptWeave owns **Semantic Model Engineering**: observing source evidence, discovering semantic candidates, validating them, governing review, and publishing versioned ontology/semantic-layer releases. + +`semantic-data-portal` remains the consumer/catalog/governance plane for published semantic context. `LineageWeave` remains an inference/lineage evidence producer. `context-graph-contracts` remains a contract-only interoperability repository. `contextual-orchestrator` remains the LLM routing boundary. + +External code-analysis or graph-generation tools may be optional source adapters, but external forks are not ConceptWeave product authority or required internal dependencies. + +## Consequences + +- ConceptWeave can be used by GRC, EA, analytics, HR, billing, or other products without copying their authoritative data. +- No direct cross-service application-table SQL. +- Source systems remain authoritative for source facts. +- Published semantic releases are authoritative only within their explicitly reviewed semantic scope. diff --git a/docs/adr/0002-truth-publication-lifecycle.md b/docs/adr/0002-truth-publication-lifecycle.md new file mode 100644 index 00000000..ac35e119 --- /dev/null +++ b/docs/adr/0002-truth-publication-lifecycle.md @@ -0,0 +1,23 @@ +# ADR 0002 — Evidence, truth, and publication lifecycle + +**Status:** Accepted + +## Context + +Automatic ontology learning and LLM-assisted semantic modeling can produce plausible but incorrect concepts and relations. A single `confidence` number cannot establish authority. + +## Decision + +Separate epistemic truth status from governance publication state. Every semantic candidate requires exact source evidence. New candidates are inferred drafts. Publication requires the ordered lifecycle: + +`Draft -> Proposed -> Validated -> Reviewed -> Published` + +At Draft/Proposed/Validated/Reviewed stages the artifact is not authoritative. `Published` changes semantic truth to authoritative within the release scope. Rejection is explicit. Published facts are never overwritten; a replacement creates a new release and marks the old release/candidate superseded. + +No LLM, embedding similarity, graph centrality, or automated extractor may directly create authoritative semantic truth. + +## Consequences + +- Review and validation receipts become first-class future persistence objects. +- Replay can reconstruct why a semantic release exists. +- Consumers can filter `authoritative`, `observed`, `inferred`, and `proposed` data without conflating them. diff --git a/docs/adr/0003-standards-llm-boundary.md b/docs/adr/0003-standards-llm-boundary.md new file mode 100644 index 00000000..8a56cfa9 --- /dev/null +++ b/docs/adr/0003-standards-llm-boundary.md @@ -0,0 +1,19 @@ +# ADR 0003 — Standards and LLM engineering boundary + +**Status:** Accepted + +## Context + +ConceptWeave must publish portable semantic artifacts while standards evolve and LLM-based ontology engineering remains an active research area. + +## Decision + +1. Stable ontology publication targets are RDF 1.1, OWL 2, SKOS, SHACL 1.0, JSON-LD 1.1, and PROV-O as applicable. +2. RDF 1.2 and SHACL 1.2 are tracked as 2026 W3C in-progress work and may be implemented behind explicit experimental/versioned adapters; they are not labeled final standards. +3. Apache Ossie (incubating, formerly Open Semantic Interchange) is tracked as an emerging vendor-neutral semantic-model exchange format for datasets, fields/dimensions, relationships, and metrics. Any adapter is explicitly version-bound until the required specification subset is stable. +4. LLM-backed ontology learning, matching, labeling, and candidate generation must use `contextual-orchestrator` and produce structured proposals with evidence. LLMs do not validate, approve, or publish semantic truth. +5. Evaluation combines deterministic conformance/consistency checks, benchmark fixtures, and human-reviewed cases. Model-as-judge evidence is supplementary only. + +## Consequences + +ConceptWeave is standards-oriented without falsely promoting drafts to Recommendations, and model assistance can improve recall while governance remains fail-closed. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..291702a3 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,5 @@ +# Architecture Decision Records + +- [ADR 0001 — Product and bounded-context boundary](0001-product-boundary.md) +- [ADR 0002 — Evidence, truth, and publication lifecycle](0002-truth-publication-lifecycle.md) +- [ADR 0003 — Standards and LLM engineering boundary](0003-standards-llm-boundary.md) diff --git a/docs/doctoring/REFERENCES.md b/docs/doctoring/REFERENCES.md new file mode 100644 index 00000000..2cbca9b0 --- /dev/null +++ b/docs/doctoring/REFERENCES.md @@ -0,0 +1,100 @@ +# Standards and Research References + +This file records the evidence basis for ConceptWeave architecture decisions. Stable Recommendations and in-progress specifications are deliberately distinguished. The current paper-by-paper capability, rejection/adoption, owner, limitation, and benchmark mapping is maintained in `docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md`; bibliography alone does not count as product use. + +## Stable standards / recommendations + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS Simple Knowledge Organization System Reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ + +W3C OWL Working Group. (2012). *OWL 2 Web Ontology Language document overview (Second Edition)*. World Wide Web Consortium. https://www.w3.org/TR/owl2-overview/ + +W3C Provenance Working Group. (2013). *PROV-O: The PROV Ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +W3C RDF Working Group. (2014). *RDF 1.1 concepts and abstract syntax*. World Wide Web Consortium. https://www.w3.org/TR/rdf11-concepts/ + +W3C Data Shapes Working Group. (2017). *Shapes Constraint Language (SHACL)*. World Wide Web Consortium. https://www.w3.org/TR/shacl/ + +World Wide Web Consortium. (2020). *JSON-LD 1.1*. https://www.w3.org/TR/json-ld11/ + +## In-progress / emerging specifications tracked, not claimed as final standards + +W3C RDF-star Working Group. (2026). *RDF 1.2 concepts and abstract data model* (Candidate Recommendation Snapshot, April 7, 2026). World Wide Web Consortium. https://www.w3.org/TR/rdf12-concepts/ + +W3C Data Shapes Working Group. (2026). *SHACL 1.2 Core* (Working Draft, August 3, 2026). World Wide Web Consortium. https://www.w3.org/TR/2026/WD-shacl12-core-20260803/ + +Apache Software Foundation. (2026). *Apache Ossie (incubating)*. https://ossie.apache.org/ +Formerly Open Semantic Interchange (OSI); tracked as an emerging vendor-neutral semantic-model exchange specification rather than a W3C/ISO standard. + +## Generation research + +Babaei Giglou, H., D'Souza, J., & Auer, S. (2023). LLMs4OL: Large language models for ontology learning. In *The Semantic Web – ISWC 2023* (pp. 408–427). Springer. https://doi.org/10.1007/978-3-031-47240-4_22 + +Trajanoska, M., Stojanov, R., & Trajanov, D. (2023). *Enhancing knowledge graph construction using large language models* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/enhancing-knowledge-graph-construction-using-large-trajanoska-stojanov/80ffe83041735fdf94bf4b60dd32ba1a/?utm_source=chatgpt + +Shimizu, C., & Hitzler, P. (2024). *Accelerating knowledge graph and ontology engineering with large language models* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/accelerating-knowledge-graph-and-ontology-engineering-shimizu-hitzler/82d868ee8f7953108246241e28d5e339/?utm_source=chatgpt + +Lo, A., Jiang, A. Q., Li, W., & Jamnik, M. (2024). *End-to-end ontology learning with large language models* [Preprint]. arXiv. https://arxiv.org/abs/2410.23584 + +Babaei Giglou, H., D'Souza, J., & Auer, S. (2024). *LLMs4OL 2024 Overview: The 1st Large Language Models for Ontology Learning Challenge*. Consensus record: https://consensus.app/papers/llms4ol-2024-overview-the-1st-large-language-models-for-giglou-d’souza/3ee443141c0d51bf9a9a8f2257070f04/?utm_source=chatgpt + +Phuttaamart, T., Kertkeidkachorn, N., & Trongratsameethong, A. (2024). *The Ghost at LLMs4OL 2024 Task A: Prompt-Tuning-Based Large Language Models for Term Typing*. Consensus record: https://consensus.app/papers/the-ghost-at-llms4ol-2024-task-a-prompttuningbased-large-phuttaamart-kertkeidkachorn/9d0fce91b8ba550fa308ec889bd7056e/?utm_source=chatgpt + +Val-Calvo, M., Egaña Aranguren, M., Martínez-Hernández, J. M., Almagro-Hernández, G., Deshmukh, P., Bernabé-Díaz, J. A., Espinoza-Arias, P., Sánchez-Fernández, J., Mueller, J., & Fernández-Breis, J. (2025). OntoGenix: Leveraging large language models for enhanced ontology engineering from datasets. *Information Processing & Management, 62*, 104042. Consensus record: https://consensus.app/papers/ontogenix-leveraging-large-language-models-for-enhanced-val-calvo-aranguren/2c2771b0905a5292b6addb6b299bda17/?utm_source=chatgpt + +Zhang, Y., Dalal, A., Martin, C., Gadusu, S. R., & Mcginty, H. (2025). OLIVE: Ontology learning with integrated vector embeddings. *Applied Ontology, 20*, 36–53. Consensus record: https://consensus.app/papers/olive-ontology-learning-with-integrated-vector-zhang-dalal/1d96dab8b5c45b9fbbf19ecf9d39bc23/?utm_source=chatgpt + +Lippolis, A. S., Saeedizade, M. J., Keskisarkka, R., Zuppiroli, S., Ceriani, M., Gangemi, A., Blomqvist, E., & Nuzzolese, A. G. (2025). *Ontology generation using large language models* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/ontology-generation-using-large-language-models-lippolis-saeedizade/f3dd9e0944c253e3962b9ae9f4dc7867/?utm_source=chatgpt + +Giglou, H. B., D'Souza, J., Aioanei, A. C., Mihindukulasooriya, N., & Auer, S. (2026). *OntoLearner: A modular Python library for ontology learning with large language models*. Consensus record: https://consensus.app/papers/ontolearner-a-modular-python-library-for-ontology-giglou-d’souza/63f55ff320b759d0a5e6e2b79fe4e37a/?utm_source=chatgpt + +## Client, matching, and versioning research + +He, Y., Chen, J., Dong, H., & Horrocks, I. (2023). Exploring large language models for ontology alignment. In I. Fundulaki, K. Kozaki, D. Garijo, & J. M. Gomez-Perez (Eds.), *Proceedings of the ISWC 2023 Posters, Demos and Industry Tracks: From Novel Ideas to Industrial Practice* (CEUR Workshop Proceedings, Vol. 3632). CEUR-WS.org. https://ceur-ws.org/Vol-3632/ISWC2023_paper_427.pdf + +Amini, R., Saki Norouzi, S., Hitzler, P., & Amini, R. (2025). Towards complex ontology alignment using large language models. In S. Tiwari, B. Villazón-Terrazas, F. Ortiz-Rodríguez, & S. Sahri (Eds.), *Knowledge graphs and semantic web: 6th International Conference, KGSWC 2024, Paris, France, December 11–13, 2024, proceedings* (Lecture Notes in Computer Science, Vol. 15459, pp. 17–31). Springer Nature Switzerland. https://doi.org/10.1007/978-3-031-81221-7_2 + +Hertling, S., & Paulheim, H. (2023). OLaLa: Ontology matching with large language models. In *Proceedings of the 12th Knowledge Capture Conference 2023* (pp. 131–139). Association for Computing Machinery. https://doi.org/10.1145/3587259.3627571 + +Qiang, Z., Wang, W., & Taylor, K. L. (2023). Agent-OM: Leveraging LLM agents for ontology matching. *Proceedings of the VLDB Endowment, 18*, 516–529. Consensus record: https://consensus.app/papers/agentom-leveraging-llm-agents-for-ontology-matching-qiang-wang/1ff1e2abb0f255299ecb808951ceaf6b/?utm_source=chatgpt + +Babaei Giglou, H., D'Souza, J., Engel, F., & Auer, S. (2024). *LLMs4OM: Matching ontologies with large language models* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/llms4om-matching-ontologies-with-large-language-models-giglou-d’souza/a45561fbd0b25041a04df0f2fa49440b/?utm_source=chatgpt + +Qiang, Z., & Taylor, K. L. (2024). *OM4OV: Leveraging ontology matching for ontology versioning* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/om4ov-leveraging-ontology-matching-for-ontology-qiang-taylor/bc311ce1e88c52a1ad9485037371b2e0/?utm_source=chatgpt + +Qiang, Z., Taylor, K. L., & Wang, W. (2024). *How does a text preprocessing pipeline affect ontology syntactic matching?* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/how-does-a-text-preprocessing-pipeline-affect-ontology-qiang-taylor/b21e8fc85c665d108ec0022368151aca/?utm_source=chatgpt + +Sousa, G., Lima, R., & Trojahn, C. (2025). *Complex ontology matching with large language model embeddings* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/complex-ontology-matching-with-large-language-model-sousa-lima/7244613a2f595e9d9ded3c8e62300d99/?utm_source=chatgpt + +Taboada, M., Martínez, D., Arideh, M., & Mosquera, R. (2025). *Ontology matching with large language models and prioritized depth-first search* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/ontology-matching-with-large-language-models-and-taboada-martínez/7cff568231f455de89f26311b6be0d26/?utm_source=chatgpt + +Barcelos, E. I., French, R. H., & Wu, Y. (2025). *KROMA: Ontology matching with knowledge retrieval and large language models* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/kroma-ontology-matching-with-knowledge-retrieval-and-barcelos-french/4669219e2e1c54ea8af442fbc690f922/?utm_source=chatgpt + +Song, Y., Chen, J., & Schmidt, R. A. (2025). GenOM: Ontology matching with description generation and large language models. *World Wide Web, 29*. Consensus record: https://consensus.app/papers/genom-ontology-matching-with-description-generation-and-song-chen/8587c3ae332a516d8426504b1f64447c/?utm_source=chatgpt + +Khalov, A., & Ataeva, O. (2025). Automating ontology mapping in IT service management: A DOLCE and ITSMO integration. *Data Science Journal, 24*. Consensus record: https://consensus.app/papers/automating-ontology-mapping-in-it-service-management-a-khalov-ataeva/74e0b948f0a8599a8ba223f23eeac3cc/?utm_source=chatgpt + +## Bridge research + +Xiao, G., Ren, L., Qi, G., Xue, H., Di Panfilo, M., & Lanti, D. (2025). *LLM4VKG: Leveraging large language models for virtual knowledge graph construction*. Consensus record: https://consensus.app/papers/llm4vkg-leveraging-large-language-models-for-virtual-xiao-ren/c6486ba49d125d66ad70bb4f97df5dc7/?utm_source=chatgpt + +## Evaluation, hallucination, and governance research + +Qiang, Z., Taylor, K. L., Wang, W., & Jiang, J. (2024). *OAEI-LLM: A benchmark dataset for understanding large language model hallucinations in ontology matching* [Preprint]. arXiv. Consensus record: https://consensus.app/papers/oaeillm-a-benchmark-dataset-for-understanding-large-qiang-taylor/e71db19036e651e69c2b5cee75d36935/?utm_source=chatgpt + +Qiang, Z., Taylor, K. L., Wang, W., & Jiang, J. (2025). *OAEI-LLM-T: A TBox benchmark dataset for understanding large language model hallucinations in ontology matching*. Consensus record: https://consensus.app/papers/oaeillmt-a-tbox-benchmark-dataset-for-understanding-large-qiang-taylor/168a617397d8509ba9fe67e9889f2cab/?utm_source=chatgpt + +Qiang, Z., Wang, W., & Taylor, K. L. (2026). *Crowd-OM: Crowdsourcing for ontology matching validation*. Consensus record: https://consensus.app/papers/crowdom-crowdsourcing-for-ontology-matching-validation-qiang-wang/55e7bc49f40d56c7994ffb1e28d1e0fc/?utm_source=chatgpt + +Du, R., An, H., Wang, K., & Liu, W. (2024). *A short review for ontology learning: Stride to large language models trend*. Consensus record: https://consensus.app/papers/a-short-review-for-ontology-learning-stride-to-large-du-an/ad3e2c6bf660569ca1effb7b6d31a6f7/?utm_source=chatgpt + +Li, J., Garijo, D., & Poveda-Villalón, M. (2026). Large language models for ontology engineering: A systematic literature review. *Semantic Web, 17*(4), 1–45. https://doi.org/10.1177/22104968261465514 + +## Decision implications + +- LLMs can assist ontology learning, matching, modeling, and maintenance, but no retrieved study justifies automatic authority promotion. +- Generation is evaluated as distinct term-typing, taxonomy, non-taxonomic relation, mapping, structural, provenance, and abstention tasks; one aggregate ontology-quality score is insufficient. +- Client matching is retrieval/pruning/structural-evidence first. LLM prompting is a bounded candidate-ranking/explanation tool, not the source of truth. +- Ontology versioning and release compatibility are treated as distinct from ordinary matching; release diff must detect additions/removals/changes and explain affected client queries. +- OAEI-LLM/OAEI-LLM-T add LLM-specific hallucination categories to matching evaluation. GRC remains the enterprise round-trip fixture rather than the sole benchmark. +- Modular ontology engineering and explicit source provenance are preferred over one opaque prompt that attempts to generate an entire enterprise semantic layer in a single step. +- Human review remains mandatory before authority promotion. Scalable validation research may inform review mechanics but cannot replace domain-owner/steward authority. diff --git a/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md new file mode 100644 index 00000000..bde6ff96 --- /dev/null +++ b/docs/doctoring/RESEARCH_CAPABILITY_TRACEABILITY.md @@ -0,0 +1,100 @@ +# Research-to-capability traceability + +Snapshot: 2026-09-04 + +This register turns the accepted research set into product decisions. A paper is not considered "used" merely because it appears in the bibliography: it must be tied to a bounded context, an accepted or rejected design implication, and an executable evaluation family. Publication metadata follows the previously recorded Consensus set plus authoritative publication records re-verified on 2026-09-04; preprints are not silently promoted to peer-reviewed evidence. + +## Classification rules + +- `generation`: Source Observation, Semantic Discovery, or LLM Proposal capabilities that create ontology/semantic-model candidates. +- `client`: Model Alignment or Client Consumption capabilities that match, resolve, compare, explain, or consume governed releases. +- `bridge`: connects generated ontology/schema mappings to a downstream virtual/semantic consumption model. +- `cross_cutting`: evaluation, versioning, governance, reproducibility, or human-validation evidence shared by both tracks. +- `adopt`: becomes a product/test requirement. +- `adapt`: informs the design but is constrained by ConceptWeave authority/security contracts. +- `research_only`: retained as evidence but not adopted as a production rule. + +## Current research register + +| Study | Class | Product implication | Decision | Implementation owner | Evaluation / test family | Limitation carried into ConceptWeave | +| --- | --- | --- | --- | --- | --- | --- | +| Shimizu & Hitzler (2024), *Accelerating Knowledge Graph and Ontology Engineering with Large Language Models* | generation / cross_cutting | Keep ontology engineering modular: modeling, extension, population, alignment, disambiguation are separate operations rather than one opaque prompt. | adopt | Semantic Discovery, LLM Proposal, Model Alignment | per-operation fixtures; orchestration receipts; module-boundary fitness | Consensus currently records an arXiv publication; use as architecture evidence, not sole production truth. | +| Trajanoska, Stojanov, & Trajanov (2023), *Enhancing Knowledge Graph Construction Using Large Language Models* | generation | Compare LLM-assisted entity/relation extraction and ontology proposals with deterministic/specialized baselines. | adapt | Semantic Discovery, LLM Proposal | entity/relation precision-recall-F1; KG/ontology relevance; abstention | Demonstration domain is not enterprise GRC; no automatic generalization of reported accuracy. | +| Val-Calvo et al. (2025), *OntoGenix* | generation | Treat dataset-to-ontology work as staged preprocessing -> planning -> building -> refinement -> mapping, with explicit failure on complex modeling. | adopt | Source Observation, Semantic Discovery, Model Validation | stage receipts; coherent-model fixtures; complex-model abstention | Human modeling remains stronger for complex cases; model output stays proposed. | +| Lo, Jiang, Li, & Jamnik (2024), *End-to-End Ontology Learning with Large Language Models* | generation | Include end-to-end taxonomy generation as a benchmark strategy, not as the sole architecture. Measure semantic and structural similarity. | adapt | Semantic Discovery, Evaluation | taxonomy-edge P/R/F1; graph structural similarity; domain-transfer fixture | Consensus currently records arXiv; fine-tuned end-to-end generation does not remove governance review. | +| Giglou, D'Souza, & Auer (2023), *LLMs4OL* | generation | Term typing, taxonomy discovery, and non-taxonomic relation extraction are first-class Generation tasks. | adopt | Semantic Discovery, LLM Proposal | LLMs4OL-style task fixtures across heterogeneous domains | Zero-shot results vary by domain; no model family becomes canonical product truth. | +| Giglou, D'Souza, & Auer (2024), *LLMs4OL 2024 Overview* | generation / cross_cutting | Preserve challenge-style standardized task definitions and reusable benchmark splits. | adopt | Evaluation | challenge-compatible term typing / taxonomy / relation suites | Challenge evidence measures tasks, not enterprise governance or source authority. | +| Phuttaamart, Kertkeidkachorn, & Trongratsameethong (2024), *The Ghost at LLMs4OL 2024 Task A* | generation | Track prompt/prompt-tuning sensitivity explicitly for term typing. | adapt | LLM Proposal, Evaluation | term-typing per-domain accuracy/F1; prompt sensitivity | GeoNames degradation is a concrete warning against aggregate-only scores. | +| Zhang et al. (2025), *OLIVE: Ontology Learning With Integrated Vector Embeddings* | generation | Vector/LLM workflows may assist relationship discovery and OWL drafting, but vectors remain candidate evidence rather than semantic authority. | adapt | Semantic Discovery, publication adapters | candidate quality; OWL syntax/shape validation | Prompt-driven retrieval and vector similarity cannot define truth status. | +| Lippolis et al. (2025), *Ontology Generation using Large Language Models* | generation / cross_cutting | Competency questions and user stories can drive ontology drafts; assess multiple structural criteria plus expert qualitative review. | adopt | LLM Proposal, Model Validation | competency-question coverage; structural criteria; expert/steward edit distance | Reported quality varies by model/prompt; generated OWL remains draft/proposed. | +| Giglou et al. (2026), *OntoLearner* | generation / cross_cutting | Add cross-domain standardized benchmarking and measure failure against ontology complexity, not just model size. | adopt | Evaluation | multi-domain term/taxonomy/relation benchmark; complexity-stratified error analysis | Tool/library is research infrastructure, not a required runtime dependency. | +| Hertling & Paulheim (2023), *OLaLa* | client | Ontology matching needs explicit prompt representation, examples, existing correspondences, and candidate-generation choices. | adopt | Model Alignment | OAEI-style matching P/R/F1; zero/few-shot comparison | LLM result is a correspondence candidate, never automatic authoritative alignment. | +| Giglou, D'Souza, Engel, & Auer (2024), *LLMs4OM* | client | Retrieve first, then match; compare concept-only, parent-context, and child-context representations. | adopt | Model Alignment, Client Consumption | retrieval recall; matching P/R/F1 across representation variants | Consensus currently records arXiv; client must remain functional without LLM matching. | +| He, Chen, Dong, & Horrocks (2023), *Exploring Large Language Models for Ontology Alignment* | client | Compare concept-label-only matching with bounded structural-context matching instead of treating a richer prompt as automatically better. LLM output remains a correspondence candidate. | adapt | Model Alignment | OAEI Bio-ML hard subsets; concept-only vs structural-context ablation; zero-shot baseline | Peer-reviewed ISWC 2023 Posters/Demos evidence is explicitly preliminary; reported model gains do not establish enterprise-domain generality or publication authority. | +| Amini, Saki Norouzi, Hitzler, & Amini (2024), *Towards Complex Ontology Alignment Using Large Language Models* | client | Complex correspondence proposals may use bounded ontology modules/rich local context, with module/context size recorded as evidence rather than unconstrained prompt expansion. | adapt | Model Alignment | complex-alignment fixtures; module-size/context ablation; correspondence P/R/F1 | Refereed KGSWC 2024 evidence supports module-aware prompting, but rich context cannot bypass retrieval bounds, source provenance, abstention, or steward governance. | +| Sousa, Lima, & Trojahn (2025), *Complex Ontology Matching with Large Language Model Embeddings* | client | Support expressive correspondence proposals using local subgraph/neighborhood evidence, not label similarity alone. | adapt | Model Alignment | complex-correspondence F1; subgraph ablations | Embedding-space/model compatibility must be explicit; reported gains do not authorize cross-model vector comparison. | +| Taboada et al. (2025), MILA | client | Use programmed retrieval/search to prune candidates and reserve LLM calls for uncertain cases. | adopt | Model Alignment, Client Consumption | candidate recall; final P/R/F1; LLM-call reduction vs naive prompting | Consensus currently records arXiv; algorithmic search cannot bypass evidence/truth-state rules. | +| Barcelos, French, & Wu (2025), *KROMA* | client | Targeted knowledge retrieval, structural context, and refinement should precede context-augmented LLM matching. | adopt | Model Alignment | candidate pruning recall; prompt-enrichment ablation; communication cost | Consensus currently records arXiv; RAG context is not source authority. | +| Qiang, Wang, & Taylor (2023), *Agent-OM* | client | Separate retrieval and matching responsibilities and expose bounded matching tools rather than one monolithic agent prompt. | adapt | Model Alignment, contextual-orchestrator ACL | OAEI simple/complex/few-shot tracks; tool-call receipts | Agent autonomy does not include release publication or business authorization. | +| Song, Chen, & Schmidt (2025), *GenOM* | client | Generated concept descriptions can enrich retrieval/matching, but exact lexical evidence remains a useful deterministic precision signal. | adopt | Model Alignment | OAEI Bio-ML; definition-quality criteria; retrieval/matching ablations | Biomedical results require enterprise-domain replication before broader claims. | +| Qiang & Taylor (2024), *OM4OV* | client / cross_cutting | Ontology/release version comparison needs explicit update-entity detection and explanations; do not equate versioning with ordinary matching. | adopt | Release Contract, Client Consumption | release-diff correctness; added/removed/changed entity detection; false-match explanation | Consensus currently records arXiv; ConceptWeave needs its own compatibility semantics. | +| Qiang et al. (2024), *OAEI-LLM* | client / cross_cutting | LLM-specific ontology-matching hallucinations require a dedicated benchmark dimension. | adopt | Evaluation | OAEI-LLM hallucination categories; abstention quality | Benchmark does not replace enterprise GRC golden fixtures. | +| Qiang et al. (2025), *OAEI-LLM-T* | client / cross_cutting | Add TBox/schema hallucination tests for matching and alignment. | adopt | Evaluation | TBox hallucination leaderboard/categories | Duplicate preprint/proceedings variants count as one study in this register. | +| Qiang, Wang, & Taylor (2026), *Crowd-OM* | cross_cutting | Human validation quality needs explicit trust/coherence/history controls when review scales beyond one steward. | adapt | Governance & Publication | inter-reviewer disagreement; coherence; adjudication receipts | Crowdsourcing is optional; domain-owner/steward authority remains product policy. | +| Qiang, Taylor, & Wang (2024), *How Does A Text Preprocessing Pipeline Affect Ontology Syntactic Matching?* | client | Keep deterministic tokenization/normalization as inspectable evidence; avoid assuming stopword/stemming pipelines always improve matching; LLM repair is secondary. | adopt | Model Alignment | OAEI preprocessing ablations; false-mapping regressions | No generic stopword/stemming heuristic is promoted to semantic truth. | +| Khalov & Ataeva (2025), *Automating Ontology Mapping in IT Service Management* | client | Lexical, embeddings, graph structure, and LLM signals may be compared as candidate features. | research_only | Model Alignment research adapter | feature ablation if reproduced | Reported validation uses an LLM surrogate expert and no annotated gold; it cannot ground production acceptance. | +| Xiao et al. (2025), *LLM4VKG* | bridge | Schema analysis + ontology development + mapping creation must flow into a stable downstream consumption contract and tolerate incomplete ontology inputs without inventing truth. | adopt | Generation↔Client seam | RODI-style mapping F1; incomplete-ontology fixtures; GRC round-trip | VKG execution remains in consuming systems; ConceptWeave does not become their query/database authority. | +| Li, Garijo, & Poveda-Villalón (2026), systematic literature review | cross_cutting | Standardize task definitions, datasets, metrics, prompt/model receipts, and human-expert review; disclose reproducibility gaps. | adopt | all bounded contexts / Evaluation | reproducibility manifest; benchmark disclosure; provider/prompt sensitivity | Literature reports heterogeneous protocols; no paper/prompt becomes a universal algorithm. | +| Du, An, Wang, & Liu (2024), ontology-learning review | cross_cutting | Keep shallow/deep/LLM methods comparable rather than treating LLMs as the only valid generation family. | research_only | Evaluation | baseline taxonomy of method families | Secondary review; the 2026 systematic review is the stronger cross-cutting evidence base. | + +## GRC reference flow + +`ContextualWisdomLab/governance-risk-compliance` is the first enterprise golden/reference scenario, not a special-case algorithm. The same immutable GRC fixture must exercise both tracks: + +`GRC source contract -> observed facts -> generation candidates -> validation/steward review -> semantic_release -> client validation/resolution/diff/query-plan -> GRC deterministic calculation`. + +Acceptance must prove that ConceptWeave never becomes the GRC system of record, that proposed/inferred relations do not mutate authoritative GRC records, that release validation works offline, and that release upgrades identify affected GRC queries explicitly. Public OAEI/RODI/LLMs4OL-style benchmarks remain necessary because one enterprise fixture cannot establish general matching or learning performance. + +## Consensus records used in the prior accepted set + +The following canonical Consensus records were fetched before recording the corresponding product implications: + +- https://consensus.app/papers/accelerating-knowledge-graph-and-ontology-engineering-shimizu-hitzler/82d868ee8f7953108246241e28d5e339/?utm_source=chatgpt +- https://consensus.app/papers/enhancing-knowledge-graph-construction-using-large-trajanoska-stojanov/80ffe83041735fdf94bf4b60dd32ba1a/?utm_source=chatgpt +- https://consensus.app/papers/ontogenix-leveraging-large-language-models-for-enhanced-val-calvo-aranguren/2c2771b0905a5292b6addb6b299bda17/?utm_source=chatgpt +- https://consensus.app/papers/endtoend-ontology-learning-with-large-language-models-lo-jiang/c3543c4051ac5bd7bf6932020e8d5120/?utm_source=chatgpt +- https://consensus.app/papers/llms4ol-large-language-models-for-ontology-learning-giglou-d’souza/971c6331c7cd5e24a3a547d5a938b40d/?utm_source=chatgpt +- https://consensus.app/papers/llms4ol-2024-overview-the-1st-large-language-models-for-giglou-d’souza/3ee443141c0d51bf9a9a8f2257070f04/?utm_source=chatgpt +- https://consensus.app/papers/the-ghost-at-llms4ol-2024-task-a-prompttuningbased-large-phuttaamart-kertkeidkachorn/9d0fce91b8ba550fa308ec889bd7056e/?utm_source=chatgpt +- https://consensus.app/papers/olive-ontology-learning-with-integrated-vector-zhang-dalal/1d96dab8b5c45b9fbbf19ecf9d39bc23/?utm_source=chatgpt +- https://consensus.app/papers/ontology-generation-using-large-language-models-lippolis-saeedizade/f3dd9e0944c253e3962b9ae9f4dc7867/?utm_source=chatgpt +- https://consensus.app/papers/ontolearner-a-modular-python-library-for-ontology-giglou-d’souza/63f55ff320b759d0a5e6e2b79fe4e37a/?utm_source=chatgpt +- https://consensus.app/papers/olala-ontology-matching-with-large-language-models-hertling-paulheim/53331022346755a49bdbb5455ae13b8c/?utm_source=chatgpt +- https://consensus.app/papers/llms4om-matching-ontologies-with-large-language-models-giglou-d’souza/a45561fbd0b25041a04df0f2fa49440b/?utm_source=chatgpt +- https://consensus.app/papers/complex-ontology-matching-with-large-language-model-sousa-lima/7244613a2f595e9d9ded3c8e62300d99/?utm_source=chatgpt +- https://consensus.app/papers/ontology-matching-with-large-language-models-and-taboada-martínez/7cff568231f455de89f26311b6be0d26/?utm_source=chatgpt +- https://consensus.app/papers/kroma-ontology-matching-with-knowledge-retrieval-and-barcelos-french/4669219e2e1c54ea8af442fbc690f922/?utm_source=chatgpt +- https://consensus.app/papers/agentom-leveraging-llm-agents-for-ontology-matching-qiang-wang/1ff1e2abb0f255299ecb808951ceaf6b/?utm_source=chatgpt +- https://consensus.app/papers/genom-ontology-matching-with-description-generation-and-song-chen/8587c3ae332a516d8426504b1f64447c/?utm_source=chatgpt +- https://consensus.app/papers/om4ov-leveraging-ontology-matching-for-ontology-qiang-taylor/bc311ce1e88c52a1ad9485037371b2e0/?utm_source=chatgpt +- https://consensus.app/papers/oaeillm-a-benchmark-dataset-for-understanding-large-qiang-taylor/e71db19036e651e69c2b5cee75d36935/?utm_source=chatgpt +- https://consensus.app/papers/oaeillmt-a-tbox-benchmark-dataset-for-understanding-large-qiang-taylor/168a617397d8509ba9fe67e9889f2cab/?utm_source=chatgpt +- https://consensus.app/papers/crowdom-crowdsourcing-for-ontology-matching-validation-qiang-wang/55e7bc49f40d56c7994ffb1e28d1e0fc/?utm_source=chatgpt +- https://consensus.app/papers/how-does-a-text-preprocessing-pipeline-affect-ontology-qiang-taylor/b21e8fc85c665d108ec0022368151aca/?utm_source=chatgpt +- https://consensus.app/papers/automating-ontology-mapping-in-it-service-management-a-khalov-ataeva/74e0b948f0a8599a8ba223f23eeac3cc/?utm_source=chatgpt +- https://consensus.app/papers/llm4vkg-leveraging-large-language-models-for-virtual-xiao-ren/c6486ba49d125d66ad70bb4f97df5dc7/?utm_source=chatgpt +- https://consensus.app/papers/large-language-models-for-ontology-engineering-a-li-garijo/3087bb8f7cd0500d917f89d8a92559e5/?utm_source=chatgpt +- https://consensus.app/papers/a-short-review-for-ontology-learning-stride-to-large-du-an/ad3e2c6bf660569ca1effb7b6d31a6f7/?utm_source=chatgpt + +## Additional authoritative publication records verified on 2026-09-04 + +- He, Y., Chen, J., Dong, H., & Horrocks, I. (2023). *Exploring large language models for ontology alignment*. 22nd International Semantic Web Conference (ISWC 2023), Posters, Demos and Industry Tracks. Oxford University Research Archive peer-reviewed record: https://ora.ox.ac.uk/objects/uuid%3Ab0ecf14b-e9b9-4767-9fae-8a7adddd6fb6 +- Amini, R., Saki Norouzi, S., Hitzler, P., & Amini, R. (2024). *Towards complex ontology alignment using large language models*. In *Knowledge Graphs and Semantic Web: 6th International Conference, KGSWC 2024* (pp. 17–31). Springer. https://doi.org/10.1007/978-3-031-81221-7_2 + +## Next executable consequences + +1. Issue #2 Generation evaluation must expose task-level metrics rather than one aggregate "ontology quality" score. +2. Issue #3 Client work must implement retrieval-before-prompt, structural/neighborhood evidence, deterministic lexical evidence, explicit abstention, release diff/version compatibility, OAEI-LLM hallucination fixtures, concept-only vs structural-context ablations, and bounded-module complex-alignment fixtures. +3. GRC must remain the first enterprise round-trip fixture, while OAEI/RODI/LLMs4OL-style data guards against overfitting the general contract to GRC. +4. LLM calls remain behind `contextual-orchestrator`; model/provider/prompt changes require receipts and sensitivity evidence. +5. Human review remains mandatory before authority promotion; Crowd-OM is evidence for scalable validation mechanics, not permission to replace GRC/domain steward authority. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..5c31d18b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,23 @@ +# ConceptWeave + +ConceptWeave turns enterprise evidence into governed, reviewable semantic models while keeping source systems authoritative. + +## Product goal + +The first release target converts an immutable relational-schema snapshot into evidence-bound semantic candidates for concepts, relationships, dimensions, measures, constraints, and physical mappings. Candidates remain proposed until deterministic validation and authorized review permit publication. + +## Current status + +ConceptWeave is in foundation development. The active foundation work establishes its Rust domain model, lifecycle, public schema, architecture, security, test strategy, and operability baseline. Source adapters, persistence, model-assisted induction, steward interfaces, and publication adapters remain explicit gaps until implemented and verified. + +## Start here + +- [Repository overview](../README.md) +- [Product requirements](PRD.md) +- [Technical requirements](TRD.md) +- [Architecture](../ARCHITECTURE.md) +- [DeepWiki](https://deepwiki.com/ContextualWisdomLab/ConceptWeave) + +## Governance boundary + +Generated or LLM-assisted meaning is never authoritative by default. Every published semantic release must preserve source evidence and pass the product's deterministic validation and governance lifecycle. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 00000000..f7ba7dea --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,76 @@ +# Product / Technical Gap Baseline + +**Snapshot:** 2026-09-05 + +This file records code-current product and technical gaps. Exact PR/check/run coordinates are evidence snapshots, not mutable-head dependencies. Live protected-branch, PR, issue and workflow state wins whenever it advances after this snapshot. Because this documentation update creates a Foundation successor, the Foundation SHA below is the exact pre-refresh head; PR metadata must be refreshed to the resulting successor SHA. + +## Protected truth and active stack + +Protected/default `main` remains `f4f440dd58c77d7cd90dff8a1eb2eeb9a9940425`; only the bootstrap state is shipped there and no immutable ConceptWeave release exists. + +The active roots observed immediately before this baseline refresh are: + +1. Foundation PR #1 — pre-refresh exact head `5cdd319b9425989e632149b243a3308dd630c0ae`, Draft/open/mergeable. The current Foundation makes Product CI draft-aware while keeping Ready/non-Draft quality requirements intact. Product CI still cannot materialize from protected `main` because that branch does not yet contain `.github/workflows/product.yml`. +2. Product-CI bootstrap PR #35 — exact head `daa543ce2cc2b2eb6d35a7265abcf2a7466e7381`, open/non-Draft/mergeable. It adds only the pull-request form of Product CI so #1 can later be marked Ready without a no-op commit. Exact-head CodeQL PR, Security Scan and SAST Semgrep remain queued; `Security Scan / Detect changed scope` is pre-runner with no steps and no runner assignment, and no independent submitted review exists yet. +3. Client Consumption PR #5 — exact head `cbb9cda0c93d8b762195423834f1d6a27dbfa613`, Draft/open/mergeable. The current source retains language-neutral semantic-release admission, integrity, compatibility, diff/resolution and supersession validation. Previously valid review findings are source-repaired, but current protected evidence remains independently required. +4. Source Observation PR #6 — exact head `d255f5c08a621024809c7e076989eccf0662a330`, Draft/open/mergeable. PostgreSQL targeted `ON DELETE SET NULL (...)` / `SET DEFAULT (...)` column provenance and registry/ACL-resolved source identity are source-repaired. The next P0 slice is the concrete bounded read-only PostgreSQL adapter. +5. Zotero Research Classification root PR #9 — exact head `cda546672cd95b5f8bed7024f70e4e6b39a134c8`, Draft/open/mergeable. The dependent research/write-back stack remains proposal/review oriented and does not elevate local classifier output to semantic authority. + +Predecessor reviews/checks never transfer to successor heads. No force-push, destructive rebase, self-approval, fail-open scanner substitution or routine administrator bypass is acceptance evidence. + +## Foundation capability status + +| Area | Status | Evidence / next verification | +| --- | --- | --- | +| Product boundary | ACTIVE_PR | PRD/TRD/ADR/context map define ConceptWeave ownership of `observe -> discover -> propose -> align -> validate -> review -> publish`, governed immutable semantic releases and stable Client contracts. Foreign product truth remains behind released/versioned ports and ACLs. | +| Truth/publication lifecycle | REPAIRED_PENDING_CI | Rust and the public Draft 2020-12 semantic-candidate schema enforce compatible publication-state/truth-status semantics. Hosted exact-head Product evidence still requires the bootstrap workflow on protected `main`. | +| Source Observation | ACTIVE_CHILD | Immutable PostgreSQL table/column/PK/unique/FK/CHECK evidence, exact identifiers, targeted delete-column provenance, canonical snapshot digest syntax, UTC provenance, receipts, bounded request budgets/cancellation and registry-authorized opaque source identity exist. No live PostgreSQL adapter is claimed; ADR 0004 remains Proposed. | +| Client Consumption | ACTIVE_CHILD | Offline Published+Authoritative admission, compatibility, exact resolution/diff, canonical digest verification, detached artifact verification and explicit supersession validation exist. Current exact-head protected evidence and prerequisite integration remain outstanding. | +| Quality gate | ACTIVE_PR | Rust 1.98.0, unsafe forbidden, public docs required, exact checkout, fmt, Clippy, tests, rustdoc, owned 100% coverage, Draft-2020-12 schema fixtures, lock freshness and clean-tree checks. Every head movement requires fresh exact-head evidence. | +| Security / dependency review | CONSUMER_REVALIDATION_PENDING | The earlier public non-fork exact-range HTTP 403 was traced to an uninitialized repository dependency graph, not to a retryable central workflow defect. `.github#1873` was closed unmerged after enabling Dependabot vulnerability alerts initialized affected graphs and the same exact comparison returned HTTP 200. The hard gate remains fail closed; a current ConceptWeave head must still execute the pinned Dependency Review action successfully before acceptance. | +| Review / runner admission | BLOCKED_OWNER | #35's exact-head central runs are still queued before useful execution; `Detect changed scope` has no runner assignment or steps. Queueing blocks this validation lane only and is not a reason to stop Source Observation or other repository-owned work. | +| Standards / research | REPAIRED_PENDING_CI | Doctoring remains bound to authoritative standards/primary research and exact implementation contracts; hosted exact-head evidence remains independently required after head changes. | +| Release | NOT_STARTED | No immutable ConceptWeave release exists. Version/CHANGELOG/tag/package/semantic_release/SBOM/provenance/reproducibility/rollback are required on the exact protected release head. | + +## Dependency Review incident correction + +The prior Foundation predecessor exposed a real hosted failure: the authenticated Dependency Review compare preflight returned HTTP 403 for a public, non-fork ConceptWeave exact range. The initially proposed central repair retried the same token-bound request while retaining fail-closed behavior. + +Fresh owner RCA invalidated that causal hypothesis. The same authenticated exact-range request returned HTTP 200 for a repository whose dependency graph was initialized and HTTP 403 for affected repositories whose graph was not initialized. Enabling Dependabot vulnerability alerts initialized the dependency graph in ConceptWeave and pingora-gateway, after which the exact compare endpoint returned HTTP 200. Therefore `.github#1873` was correctly closed without merge: retries would extend queue occupancy but would not establish repository capability. + +Acceptance remains stricter than the RCA. HTTP 200 availability alone is not GREEN. A fresh exact ConceptWeave consumer run must reach and complete the pinned Dependency Review action; 403, transport failure, skipped substitution or a sibling scanner cannot satisfy the hard gate. + +## Central control-plane evidence + +Protected central source is `.github/main@b5efbc2762e472e4a380b0503b1f050f76fbb008` at this snapshot. This is evidence only, not a mutable ConceptWeave dependency. + +- The current central source includes queue/admission and changed-scope/review-runtime repairs already integrated through ordinary protected history. +- `.github#1873@41935494aa234eb458f1cc08f006daaa278b9760` is closed/unmerged because repository dependency-graph initialization, not its retry/sleep source delta, was the verified root cause of the observed public-repository 403. +- #35 remains an exact consumer canary for current runner admission and Dependency Review behavior. Its central workflows are queued, so no protected recovery or dependency-review success is inferred from repository settings alone. + +## P0 product gaps + +1. **Concrete Source Observation adapter** — maintained Rust PostgreSQL driver behind `conceptweave-source-port`; adapter-local registry/credential resolution; explicit read-only session/transaction; exact schema allowlist; total operation and statement deadlines; cancellation plus row/byte/concurrency budgets; complete immutable snapshot or fail closed; source-disappearance handling; deterministic replay against a frozen anonymized GRC-shaped fixture. +2. **Observed PostgreSQL surface completion** — domains/enums/indexes/comments, quoted identifiers and cross-schema collisions as generic observed evidence without importing source-system business truth. +3. **Ontology discovery** — deterministic term/concept/taxonomy/non-taxonomic-relation candidate generation with exact source receipts and abstention for unsupported semantics. +4. **Semantic-layer discovery** — dimensions, measures, grain, units, relationships and physical mappings with deterministic calculation contracts; do not infer business authority from relational structure alone. +5. **LLM Proposal** — every production model call through a released `contextual-orchestrator`; outputs remain proposed/inferred and preserve source/model/prompt/provenance evidence. +6. **Alignment / matching** — retrieval/pruning/structural evidence first, bounded optional LLM assistance, OAEI-style evaluation, deterministic reproducibility and steward-visible decisions. +7. **Validation engine** — RDF/OWL/SKOS/SHACL and semantic-layer validation, consistency/conflict/duplicate detection, bounded reasoning and explicit unsupported-feature failure. +8. **Governance persistence** — PostgreSQL 3NF candidates/evidence/validation/review/release/supersession receipts, transactional outbox and temporal history only where domain semantics require it. +9. **Review workflow** — Keyverse identity context, tenant/role/purpose authorization, steward decisions, maker-checker where required, stale-decision protection and immutable publication receipt. +10. **Publication adapters** — versioned OWL/RDFS/SKOS/SHACL/JSON-LD plus explicitly version-bound Apache Ossie export; draft/incubating formats cannot be presented as final standards. +11. **Client completion** — language-neutral release/supersession contract, provenance/signature verification, relation/mapping/dimension/measure resolution, compatibility/deprecation, match/explain/query-plan contracts while downstream products retain physical authorization/execution. +12. **CWL integration** — only released/versioned `semantic_release`/contract/ACL seams to `semantic-data-portal`, `context-graph-contracts`, GRC, EA and other consumers; no source copying, cross-service SQL or mutable supplier heads. +13. **Evaluation / multilingual** — reviewed golden fixtures, ontology-learning/matching metrics, source-evidence binding, abstention, reproducibility, KO/EN/JA/ZH/VI/ES/DE/FR labels, CJK/font/text-expansion checks where UI or published labels are material. +14. **Observability / recovery / release** — structured telemetry, security evidence, backup/restore, package/SBOM/provenance/signing, reproducible build and rollback proof before immutable release. + +## DDD fitness constraints + +- No generic `utils/helpers/services/common` domain buckets. +- Adapters remain outside the core domain model; external DTOs cross Anti-Corruption Layers. +- Source Observation facts are not source-system business truth, and relational constraints are not semantic authority by themselves. +- Client Consumption depends only on governed release contracts, never generator-private classes, prompts, persistence tables or orchestration state. +- `semantic-data-portal` remains catalog/governance/consumption rather than ConceptWeave persistence; `context-graph-contracts` owns interop contracts; `enterprise-architecture-core` owns EA; `contextual-orchestrator` owns provider routing. +- Consuming products retain tenant/purpose authorization and physical query execution. +- Published semantic truth is immutable; corrections create a new release plus supersession evidence rather than in-place overwrite. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..45626cc0 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["clippy", "llvm-tools-preview", "rustfmt"] diff --git a/scripts/check_ci_contract.py b/scripts/check_ci_contract.py new file mode 100644 index 00000000..f23b6c3f --- /dev/null +++ b/scripts/check_ci_contract.py @@ -0,0 +1,44 @@ +"""Fail closed when Product CI regresses on runner or coverage toolchain identity.""" + +from __future__ import annotations + +from pathlib import Path + + +WORKFLOW_PATH = Path(".github/workflows/product.yml") + + +def main() -> int: + """Validate queue-admission, supersession, and branch-coverage invariants.""" + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + + required_fragments = ( + "runs-on: ubuntu-24.04", + "types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, closed]", + "group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}", + "cancel-in-progress: ${{ github.event_name == 'pull_request' }}", + "if: ${{ github.event_name != 'pull_request' || (github.event.action != 'closed' && github.event.pull_request.draft == false) }}", + "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "COVERAGE_TOOLCHAIN: nightly-2026-08-20", + 'rustup toolchain install "$COVERAGE_TOOLCHAIN" --profile minimal --component llvm-tools-preview', + ) + missing = [fragment for fragment in required_fragments if fragment not in workflow] + if missing: + raise SystemExit( + "Product CI contract missing required fragment(s): " + ", ".join(missing) + ) + + if "runs-on: ubuntu-latest" in workflow: + raise SystemExit( + "Product CI must not use ubuntu-latest while current organization " + "evidence demonstrates selective floating-image starvation" + ) + + if "github.event.pull_request.number || github.ref" in workflow: + raise SystemExit("Product CI must isolate non-PR runs by run_id") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_coverage.sh b/scripts/check_coverage.sh new file mode 100755 index 00000000..2f691898 --- /dev/null +++ b/scripts/check_coverage.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +coverage_toolchain="${COVERAGE_TOOLCHAIN:-nightly-2026-08-20}" +trap 'rm -f coverage.json source-branches.json' EXIT + +cargo "+${coverage_toolchain}" llvm-cov \ + --workspace \ + --branch \ + --json \ + --output-path coverage.json + +jq '.data[0].totals' coverage.json +jq -r ' + .data[0].files[] + | select( + .summary.lines.percent != 100 + or .summary.functions.percent != 100 + or .summary.regions.percent != 100 + ) + | "COVERAGE_GAP file=\(.filename) lines=\(.summary.lines.percent) functions=\(.summary.functions.percent) regions=\(.summary.regions.percent)" +' coverage.json + +jq ' + [ + .data[0].files[] + | .filename as $file + | (.branches // [])[] + | { + file: $file, + line_start: .[0], + column_start: .[1], + line_end: .[2], + column_end: .[3], + true_count: .[4], + false_count: .[5] + } + ] + | sort_by(.file, .line_start, .column_start, .line_end, .column_end) + | group_by([.file, .line_start, .column_start, .line_end, .column_end]) + | map({ + file: .[0].file, + line_start: .[0].line_start, + column_start: .[0].column_start, + line_end: .[0].line_end, + column_end: .[0].column_end, + true_count: (map(.true_count) | add), + false_count: (map(.false_count) | add) + }) +' coverage.json > source-branches.json + +jq ' + { + count: (length * 2), + covered: ([.[] | (.true_count > 0), (.false_count > 0) | select(.)] | length), + notcovered: ([.[] | (.true_count == 0), (.false_count == 0) | select(.)] | length) + } + | .percent = (if .count == 0 then 100 else (.covered * 100 / .count) end) +' source-branches.json + +jq -r ' + .[] + | select(.true_count == 0 or .false_count == 0) + | "BRANCH_GAP file=\(.file) start=\(.line_start):\(.column_start) end=\(.line_end):\(.column_end) true_count=\(.true_count) false_count=\(.false_count)" +' source-branches.json + +jq -e ' + .data[0].totals.lines.percent == 100 and + .data[0].totals.functions.percent == 100 and + .data[0].totals.regions.percent == 100 +' coverage.json >/dev/null + +jq -e 'all(.[]; .true_count > 0 and .false_count > 0)' source-branches.json >/dev/null