From 6f1830a7d21f2ae853c0b50c70961ca9b97fa1fe Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Mon, 5 Oct 2026 21:31:04 -0700 Subject: [PATCH 1/2] chore(skills): sync vendored skill-lib skills to 9867ab3 Propagated with skill-lib tools/propagate_skills.py from The-Interdependency/skill-lib@9867ab33877f2b1f50f8a501cf379aa99360bd52 for exactly the canonical skills already vendored here: meta-module-build, msdmd, test-build. Source pin: c14ee9d -> 9867ab3. Verified with check_consumer_drift.py --strict-sha --require-vendored (clean). Local non-canonical skills are untouched. --- .agents/skills/README.md | 51 +- .agents/skills/meta-module-build/SKILL.md | 63 +- .agents/skills/msdmd/SKILL.md | 637 +++++---- .agents/skills/msdmd/collect.py | 813 ++++++++++-- .agents/skills/msdmd/collection.ts | 243 +++- .agents/skills/msdmd/formats.py | 189 +++ .agents/skills/msdmd/grammar_code.py | 136 ++ .../msdmd/module-projection.schema.json | 337 +++++ .agents/skills/msdmd/module_projection.py | 1180 +++++++++++++++++ .agents/skills/msdmd/native_code.py | 220 +++ .agents/skills/msdmd/package-lock.json | 26 + .agents/skills/msdmd/package.json | 6 + .agents/skills/msdmd/parsers/universal.py | 33 +- .agents/skills/msdmd/parsers/universal.ts | 29 +- .../skills/msdmd/python-module-reader.json | 81 ++ .agents/skills/msdmd/readers.py | 1009 ++++++++++++++ .../msdmd/references/implemented-readers.md | 79 ++ .../msdmd/references/metadata-conventions.md | 212 +++ .agents/skills/msdmd/requirements.txt | 9 + .agents/skills/msdmd/standards.py | 284 ++++ .agents/skills/msdmd/typescript-reader.cjs | 120 ++ .agents/skills/msdmd/visualize.py | 108 +- .agents/skills/test-build/SKILL.md | 170 +-- 23 files changed, 5531 insertions(+), 504 deletions(-) create mode 100644 .agents/skills/msdmd/formats.py create mode 100644 .agents/skills/msdmd/grammar_code.py create mode 100644 .agents/skills/msdmd/module-projection.schema.json create mode 100644 .agents/skills/msdmd/module_projection.py create mode 100644 .agents/skills/msdmd/native_code.py create mode 100644 .agents/skills/msdmd/package-lock.json create mode 100644 .agents/skills/msdmd/package.json create mode 100644 .agents/skills/msdmd/python-module-reader.json create mode 100644 .agents/skills/msdmd/readers.py create mode 100644 .agents/skills/msdmd/references/implemented-readers.md create mode 100644 .agents/skills/msdmd/references/metadata-conventions.md create mode 100644 .agents/skills/msdmd/requirements.txt create mode 100644 .agents/skills/msdmd/standards.py create mode 100644 .agents/skills/msdmd/typescript-reader.cjs diff --git a/.agents/skills/README.md b/.agents/skills/README.md index 6a821b81..ba32555b 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -3,13 +3,60 @@ This directory contains repo-local copies of canonical skills from `The-Interdependency/skill-lib`. -Source commit: `c14ee9d500579a4b5d6821f62c9d82ca96e73608` +Source commit: `9867ab33877f2b1f50f8a501cf379aa99360bd52` Repo-local copies are not the source of truth. Edit `skill-lib` first, then propagate from the canonical source. -Installed skills: +Skills refreshed from the source commit above: - `meta-module-build/` - `msdmd/` - `test-build/` + +Other installed skills (not refreshed by this propagation): + +Their existing files retain their own authority/provenance; the source +commit above does not assert that these copies were refreshed. + +- `a0-deep-research/` +- `a0-file-converter/` +- `a0-github-solution-finder/` +- `a0-infographic-builder/` +- `a0-website-design/` +- `a0p-fleet-runs/` +- `a0p-model-selector/` +- `a0p-module-doctrine/` +- `a0p-pcea-naming/` +- `a0p-self-declaring-modules/` +- `a0p-troubleshooter/` +- `agent-tools/` +- `anthropic-api/` +- `audit-website/` +- `better-auth-best-practices/` +- `brainstorming/` +- `browser-use/` +- `build-website/` +- `clean-build/` +- `distill-construction/` +- `distill-engineering/` +- `distill-general/` +- `distill-law/` +- `distill-medicine/` +- `find-skills/` +- `frontend-design/` +- `gemini-api/` +- `github-pages-management/` +- `gpt-5-api/` +- `grok-api/` +- `hmmm-disclosure/` +- `multimodel-handoff/` +- `next-best-practices/` +- `refuse-task/` +- `skill-creator/` +- `tool-use/` +- `ui-ux-pro-max/` +- `vercel-composition-patterns/` +- `vercel-react-best-practices/` +- `vercel-react-native-skills/` +- `web-design-guidelines/` diff --git a/.agents/skills/meta-module-build/SKILL.md b/.agents/skills/meta-module-build/SKILL.md index e9ca23f0..b3141f76 100644 --- a/.agents/skills/meta-module-build/SKILL.md +++ b/.agents/skills/meta-module-build/SKILL.md @@ -1,15 +1,21 @@ --- name: meta-module-build -description: Metadata-first module build skill built on msdmd. Use this when turning a capability idea into a bounded module manifest, file plan, public/internal surface, permission boundary, tests, docs, rollout, and rollback notes before implementation. +description: Native-first module planning built on msdmd. Consume existing manifests, schemas and design declarations; supplemental MODULE_BUILD blocks record remaining purpose, surface, boundary, test, rollout and rollback obligations. Load this when turning a capability idea into a bounded module manifest and file plan before implementation. --- -GPT generated; context, prompt Erin Spencer - # meta-module-build — Metadata-first module scaffolding -`meta-module-build` is an application of [msdmd](../msdmd/SKILL.md). It uses self-declared metadata to keep a proposed module's purpose, surfaces, dependencies, boundaries, tests, and rollout notes visible beside the files that implement it. +GPT generated; context, prompt Erin Spencer -Read `msdmd/SKILL.md` first. This skill inherits the block syntax, parser contract, and visible gap-reporting requirement. +`meta-module-build` is an application of [msdmd](../msdmd/SKILL.md). It +keeps proposed purpose, surfaces, dependencies, boundaries, tests and rollout +notes attributable to the sources that own them. + +Read `msdmd/SKILL.md` first. This skill inherits its source/provenance, native +reader support, supplemental block and information-coverage contracts. +Implementation status: this is a consuming-repo planning/runner contract. The +MSDMD schema-2 collector preserves JSON and TOML manifest trees, but this repo +does not ship a complete MODULE_BUILD obligation/policy runner. ## Doctrine @@ -21,9 +27,21 @@ intent -> manifest -> file plan -> tests -> scaffold -> reviewable change If a field is not known, write `hmmm`. Do not guess certainty into the manifest. +## Native-first coverage + +Consume existing package/module manifests, schemas, source-owned design +records and explicit rollout/test declarations before requesting supplemental +metadata. Preserve owning scope and source references. A package manifest is +not automatically a module plan: map only information it actually expresses. + +The build's semantic obligations remain required, but their native owners need +no MODULE_BUILD copies. Supplement genuinely unexpressed intent. Unsupported +mapping is unresolved coverage, not absence. Semantic planning mappings beyond +the shipped structured-source preservation remain implementation work. + ## The block -A module owns its build declaration in a `MODULE_BUILD` block: +A module may supplement its build declaration with a `MODULE_BUILD` block: ```python # === MODULE_BUILD === @@ -47,7 +65,9 @@ A module owns its build declaration in a `MODULE_BUILD` block: ## Field schema -Required: +This syntax governs supplemental MODULE_BUILD entries. Native sources retain +their own schemas and must satisfy the applicable planning obligations without +invented IDs or copied fields. Required for a supplemental entry: | Field | Meaning | |---|---| @@ -62,7 +82,7 @@ Required: | `rollout` | How the module becomes active. | | `rollback` | How to disable or remove it cleanly. | -Boundary fields are required because module generation often crosses hidden lines: +Boundary information is required because module generation often crosses hidden lines: | Field | Meaning | |---|---| @@ -139,21 +159,31 @@ Do not use raw factorization sentinels for A0-facing claims when a scoped envelo A `MODULE_BUILD` runner should: -1. parse all `MODULE_BUILD` blocks using the msdmd universal parser; -2. validate required fields; -3. report boundary fields visibly; -4. report modules without `MODULE_BUILD` as coverage gaps; -5. optionally fail in strict mode when required build metadata is missing; +1. extract supported native planning metadata and parse supplemental + `MODULE_BUILD` blocks using the msdmd universal parser; +2. validate native mappings and supplemental fields under their own schemas; +3. report boundary information and its owning sources visibly; +4. report genuinely absent required planning information only after capable + inspection; keep missing blocks separate as adoption observations; +5. fail strict mode on missing information, required unsupported extraction or + unresolved conflicts, rather than accepting incomplete coverage; 6. emit a review summary grouped by `module_kind` and boundary risk. +## Validation + +A supported native planning fixture must satisfy the information it actually +expresses without a MODULE_BUILD copy. Include missing rollback, unresolved +boundary, unsupported mapping and same-name/different-scope cases. + ## Anti-patterns - Building code first and writing the manifest after. -- Omitting boundary fields because the module feels small. +- Omitting boundary information because the module feels small. - Marking unknowns as solved instead of `hmmm`. - Using implementation-shaped ids that do not explain the protected capability. - Adding UI or route behavior without declaring metadata expectations. - Treating frontier-domain UCNS results as absolute claims. +- Demanding a second MODULE_BUILD copy of already-owned native information. ## Completion criteria @@ -169,10 +199,13 @@ or scaffold PR with tests and docs ``` -It is incomplete if it only produces an idea, a patch with no manifest, or a module with no boundary/test plan. +The manifest may be composed from source-linked native declarations and +supplemental entries. An idea alone, a patch with no manifest, or a module with +no boundary/test plan remains incomplete. ## hmmm +Native planning-obligation mappings remain consuming-repo implementation work. Default unresolved items for new modules: ```text diff --git a/.agents/skills/msdmd/SKILL.md b/.agents/skills/msdmd/SKILL.md index ffa10c40..fa8be158 100644 --- a/.agents/skills/msdmd/SKILL.md +++ b/.agents/skills/msdmd/SKILL.md @@ -1,323 +1,426 @@ --- name: msdmd -description: Module Self-Declared Metadata in Markdown — the foundational convention where each module declares its own structured metadata in a fenced comment block. Other skills in this lib (doc-build, cap-build, deps-build, owner-build, test-build, meta-module-build, risk-boundary-build, ratios, etc.) are thin applications on top of this convention. Load this when authoring a new metadata-driven skill, when extending the block schema, or when building a parser/executor for a new application. +description: Module Self-Declared Metadata in Markdown — native-first collection of metadata already expressed by code, documentation, manifests, schemas, tooling, and evidence formats, with MSDMD blocks only for otherwise unexpressed information. Load this when creating or revising metadata-driven skills, collecting repository metadata, integrating a metadata convention, building parsers or collection consumers, or auditing metadata coverage and provenance. --- -# msdmd — Module Self-Declared Metadata in Markdown +# msdmd — consume declarations where they already live -## The doctrine +## Contract and usage guidance -Every cross-cutting fact a module owns — its behavior obligations, -public documentation, declared capabilities, dependency edges, owner, -runtime boundaries, or executable evidence — should live **in the same -file as the module that owns that fact**, in a structured comment -block. A meta-runner walks the tree, parses every block, and acts on -it. +MSDMD consumes existing metadata conventions explicitly. It does not require +native declarations to be rewritten as MSDMD blocks. Its own blocks supplement +information that an owning source cannot already express adequately. -Modules without the relevant block surface as visible coverage gaps in -the runner output. Coverage is observable, not implicit. +Load this skill before changing metadata ingestion, coverage policy, or a +metadata-driven application. Read [the convention catalogue and reader +contract](references/metadata-conventions.md) for the families actually present +in the target repository. That catalogue is a discovery baseline, not a closed +allowlist and not a claim that every reader has been implemented. -This is the inverse of the conventional "keep your docs/tests/configs in -sync with code" approach, which fails because the contract and the -implementation live in different files. Anyone can delete the code and -forget the doc; the lie persists. msdmd makes the lie structurally -visible: when the implementation-owning file disappears, its owned -block disappears in the same diff. +This is the foundational metadata-block skill, expanded to native-first +interoperability; it owns the common ingestion contract, not every language's +syntax. Ordinary prose editing with no metadata contract is a non-trigger. -For tests, ownership is split rather than flattened: source modules own -`CONTRACTS` obligations; test modules own `CHECKS` evidence that -claims to prove those obligations. See -[`test-build/SKILL.md`](../test-build/SKILL.md) and -[`doctrine/msdmd-checks.md`](../doctrine/msdmd-checks.md). +**Implemented boundary:** `collect.py` defaults to schema 2 and integrates native +facts and supplemental blocks in `collection.ts`. The registry in `readers.py` +declares each shipped extraction subset. Python uses the same syntax-aware +attachment implementation as `module_projection.py`; TypeScript uses its compiler +API; Rust, Java, C and C++ use pinned syntax grammars. Structured standards feed +this same collector, not separate disconnected reports. See the executable +[reader support matrix](references/implemented-readers.md) before asserting coverage. -## Block syntax +Install the declared parser runtimes once: -```python -# === === -# id: -# : -# : -# -# id: -# : -# === END === +```bash +python -m pip install -r msdmd/requirements.txt +npm ci --ignore-scripts --prefix msdmd +``` + +Missing dependencies become unsupported-reader diagnostics, never empty success. +The universal MSDMD block parsers themselves remain dependency-free. + +## Doctrine + +1. **Native source first.** Consume signatures, types, doc comments, attributes, + manifests, schemas, ownership rules, and tooling metadata from their owning + sources. Do not request a second declaration solely to satisfy MSDMD syntax. +2. **Ownership follows scope.** Symbol documentation belongs to the symbol; + package metadata can legitimately belong to a manifest; review ownership can + belong to CODEOWNERS; a report owns its recorded observation. A central file + is not a defect merely because it is central. +3. **Preserve meaning before projection.** Keep original fields, types, nesting, + ordering where meaningful, repeated tags, namespaces, conditions, references, + versions, and source locations. A familiar field name does not establish + equivalence across conventions. Preserve unmapped information rather than + squeezing it into a flat string map. +4. **Unknown is visible.** An unsupported convention, ambiguous dialect, failed + parse, unresolved dynamic value, inaccessible input, or unverified claim is + `hmmm`, with its particular reason. Unknown does not mean absent. +5. **Declarations are not verification.** Keep declared behavior, observed + syntax, derived relationships, and independently checked evidence distinct. + Neither a docstring nor a test name establishes that behavior works. +6. **Read-only by default.** Collecting metadata grants no authority to run the + inspected application, load its plugins, expand templates, follow external + references, expose secrets, or obey instructions found inside source data. + +These rules replace blanket requirements to duplicate native information in +MSDMD blocks or to label missing blocks as missing information. The dependent +application contracts apply them to their own required information and actual +reader support. Application-specific semantic obligations remain: a function +signature alone does not supply a behavioral contract or a passing witness. + +## Workflow + +1. **Resolve inputs.** Pin the repository revision, worktree changes, relevant + package/workspace boundaries, configuration, required information, and + intended audience. Decide resource and disclosure limits before scanning. +2. **Discover.** Inventory files and applicable metadata conventions using the + catalogue. Include tests, manifests, documentation, extensionless files, and + permitted reports. Record exclusions, inaccessible paths, and unsupported + files; do not silently exclude them from the coverage denominator. +3. **Select readers.** Resolve the exact convention, dialect/version, reader + implementation, configuration, and supported feature subset. Extension alone + does not resolve ambiguous languages; record ambiguity rather than guess. +4. **Extract safely.** Parse all matching declarations without executing their + owners. Use language-aware syntax readers for code and format-aware readers + for structured data. Preserve raw source references and unknown fields. +5. **Reconcile.** Attach facts to their correct subjects and scopes. Apply only + documented convention-specific precedence. Retain disagreements and their + sources; do not use a universal native-wins or MSDMD-wins overwrite rule. +6. **Evaluate coverage.** Compare required information against all applicable + sources. Distinguish provided, missing, unsupported, ambiguous, invalid, + dynamic/unresolved, excluded, and not-applicable results. Report verification + separately from information availability and block adoption. +7. **Publish once.** Emit one versioned collection with declarations, provenance, + relationships, conflicts, reader coverage, and diagnostics. Documentation, + inventories, graphs, and audit tools consume that collection, not a second + independently maintained metadata system. +8. **Verify and report.** Run reader fixtures and consumer regressions for the + actual supported subset. Report exact inputs, commands, outcomes, changes, + and remaining `hmmm`. A catalogue entry alone earns no support claim. + +## The parser contract + +### Native readers and collection + +A native reader is a pure extraction boundary over supplied source bytes and +explicit context. Its manifest states detection, supported grammar/features, +source authority, scope/attachment rules, field mappings, unknown-field handling, +failure behavior, dependencies, and fixture-backed support status. Details and +required output fields are in the [reader contract](references/metadata-conventions.md#reader-contract). + +Each collected fact must remain attributable to an exact source identity, +location or structural pointer, subject, convention, and extraction method. +Generated identifiers are collector addresses, not falsely attributed native +IDs. Identical names in different packages, scopes, or revisions remain distinct. +Cross-revision identity requires an explicit mapping, not a line-number guess. + +Original syntax and an immutable source reference preserve lossless access; +normalized projections can be lossy only when labeled and linked back to that +source. Sensitive material stays access-controlled or explicitly redacted; +source preservation does not require publishing credentials or private content. + +Do not repurpose `MsdmdDeclaration.block` to mean JSDoc, TOML, or any other +non-block convention. `MsdmdCollectionV2` provides the native-capable schema +with explicit migration and consumer negotiation. The existing `MsdmdCollection` +name remains a schema-1 compatibility type, never a native-fact container. Refuse silent +projection when an old consumer would lose required information. + +### Python per-module projection + +`module_projection.py` owns the shared Python attachment reader. The schema-2 +collector invokes `project_python_bytes` on its bounded source buffer; the +standalone sidecar CLI remains an optional view of the same attachment facts. It parses supplied `.py` bytes with `ast` and +`tokenize` and never imports the inspected module. Its JSONL schema is +`module-projection.schema.json`; its machine-readable support manifest is +`python-module-reader.json`. The first record binds repository context, source +path and digest, optional revision, detected encoding, schema digest, reader +version, implementation digest, manifest digest, and effective Python/AST grammar. +Remaining records describe symbols, native docstrings, comments, source spans, +attachment methods, and parse diagnostics. + +Symbol IDs derive from repository, path, kind, and qualified name, with a +signature-derived disambiguator only for duplicate qualified declarations. Line +numbers are navigational facts, never identity. Attachment is structural: + +- a contiguous comment group immediately before a declaration at the same + lexical depth attaches as `leading_trivia` to that declaration, including + decorated declarations; +- every other comment inside a declaration, including comments before or + between its decorators and trailing indented suite comments before lexical + dedent, attaches to the nearest enclosing symbol; +- shebangs, encoding cookies, RATIOS seals, MSDMD fences, and otherwise + unattached comments remain module-scoped; and +- module, class, function, and method docstrings attach to their AST owner. + +Malformed MSDMD fences make the projection invalid and remain split rather +than silently spanning executable code or being accepted as a block: +interrupted (`msdmd_fence_interrupted`), unclosed (`msdmd_fence_unclosed`), +closed under a different name or containing a nested opening fence +(`msdmd_fence_mismatched`), and closed without any opening +(`msdmd_fence_unmatched_close`). A mismatch (wrong-name close or nested +opening) yields one diagnostic: one later closing fence per affected block name, +orphaned by that already-reported mismatch, is suppressed until the next opening +fence of that name; every other unpaired close is reported. This guarantee does +not cover interrupted fences, which may also report their later closing fence as +`msdmd_fence_unmatched_close`. +This projection is deliberately stricter than the universal block parser +(`parsers/universal.py`), which matches each requested block name independently +and silently ignores foreign, nested, or unpaired fences. + +Source lines split only at Python newlines (LF, CRLF, CR), so U+2028, form feed, +and similar separators never shift spans, and CR-only sources keep line numbers +and offsets aligned with the AST. Byte offsets index the UTF-8 re-encoding of the +decoded source text, not the raw file: for a BOM-prefixed or non-UTF-8 source +(for example a latin-1 file with a coding cookie) they differ from raw file byte +offsets; `source_sha256` identifies the raw bytes. Decorated-symbol spans begin +at the first decorator so normalized decorator facts retain an exact raw-source +reference through the pinned source digest. + +The projection is deterministic and disposable. Complete-tree writes prune stale +projection files; selected-file writes never prune outside their selection. +Writes use same-directory atomic replacement of exact UTF-8 bytes with LF +record terminators, so write and byte-exact `--check` converge on every platform. `--check` recomputes content and +fails on missing, stale, invalid, or—during a complete-tree check—unexpected +sidecars. Supply `--revision` when a repository revision is known; omission is +preserved as `hmmm` rather than guessed. + +```bash +python -m msdmd.module_projection --root . --repo example/repo \ + --revision --out-dir .msdmd/modules --write +python -m msdmd.module_projection --root . --repo example/repo \ + --revision --out-dir .msdmd/modules --check ``` -### Universal rules - -- **Fence**: `=== ===` opens, `=== END ===` - closes. Block name is uppercase snake_case (e.g. `CONTRACTS`, - `CHECKS`, `DOCS`, `CAPABILITIES`, `OWNERS`). -- **Comment marker**: whatever is idiomatic for the file's language. The - reference parsers auto-detect these line-comment families by extension: - `#` for Python, Ruby, Elixir, shell, Perl, R, Julia, PowerShell, Tcl, and - Raku; `//` for TypeScript/JavaScript, Rust, Go, Java, C, C++ (including - `.c+`, `.c++`, `.cxx`, and header variants), Swift, Kotlin, C#, - Objective-C++, Scala, Dart, Zig, Groovy, and PHP; `--` for SQL, Lua, - Haskell, Ada, VHDL, and Lean; `%` for Erlang and Prolog; `;` for - Clojure/Lisp/Scheme/Racket; `!` for Fortran; `'` for Visual Basic; and - `*>` for COBOL. The marker appears at the start of every line in the block. - `COMMENT_MARKERS` in both universal parsers is the exact extension registry. -- **Entry boundary**: every entry begins with `id:`. The id must be - unique within its block and stable across refactors (so it can be - referenced from external tooling). -- **Field lines**: indented one level beneath the id (two spaces of - visible indent inside the comment). Field names are lowercase - snake_case followed by `:` and a value. -- **Multiple blocks per file**: a module may declare more than one - block, of the same or different types. The parser concatenates - entries. -- **Multiple block types per file**: a module may declare both - `CONTRACTS` and `DOCS` (and any others). Each is parsed - independently by its respective application. - -### Example (Python source module) +Use repeatable `--source path/to/module.py` arguments for an incremental subset. +The standalone sidecar schema remains focused on attachment. The integrated +Python reader additionally extracts imports, literal exports, SPDX headers and +ReST/Google/NumPy docstring fields. Neither path supplies a call graph, evaluated +dynamic exports, cross-revision rename mapping or runtime proof. Stub syntax is +parsed without claiming complete `.pyi` semantics. + +## The runner protocol + +### Information coverage, not compulsory annotation + +A missing `DOCS` block is a **block-adoption observation**, not proof of missing +documentation. A missing `OWNERS` block does not establish an unowned file before +applicable native ownership rules are evaluated. A missing reader cannot earn +either a clean bill of health or a missing-information finding. + +A missing-information result requires an applicable obligation, a completed +search of its declared eligible sources, and capable readers that found no +satisfying declaration. Partially recovered metadata remains useful, but does +not justify a complete-coverage claim. Count unknown/excluded scope explicitly. + +Coverage output must name its denominator, eligibility rules, reader support, +exclusions, conflicts, and unresolved count. Separate at least information +availability, supported extraction scope, MSDMD-block adoption, and verified +behavior. A green aggregate cannot hide unsupported required inputs. + +Strict checks fail on missing required information, unresolved required scope, +parse/schema errors, or unresolved required-field conflicts. Optional unknowns +remain visible without necessarily blocking unrelated work. A non-strict +inventory may complete successfully while clearly reporting incomplete coverage. + +## Block syntax + +### Supplemental MSDMD declarations + +The existing block syntax remains supported; its purpose is supplementation, +not redeclaration of everything already present in native syntax. ```python # === CONTRACTS === -# id: chat_get_other_owner_404 -# given: GET /api/v1/conversations/{id} with x-user-id != row.user_id -# then: 404 (existence non-disclosure) +# id: other_owner_hidden +# given: a request for another owner's record +# then: return 404 without disclosing existence # class: security # === END CONTRACTS === ``` -### Example (Python test module) +Source modules own `CONTRACTS` obligations. Test modules own `CHECKS` witnesses; +`proves` produces `claims_proves`, not an automatic proof. Keep `call` with the +witness, not the source obligation. See [test-build](../test-build/SKILL.md) +and the [CONTRACTS/CHECKS doctrine](../doctrine/msdmd-checks.md). + +The block parser contract is unchanged: parse the requested block type from +text into all matching flat string-valued entries; preserve declared fields; +return an empty list when that block is absent; leave field semantics to the +application. The universal Python and TypeScript helpers live in +`parsers/universal.py` and `parsers/universal.ts` and remain dependency-free. + +Fences use uppercase snake-case block names. Every entry starts with `id:`; +field names use lowercase snake-case, allowing digits after the first character, +and field lines are indented beneath the ID. The authoring contract requires +IDs to be unique within one block type in one owning file; multiple matching +blocks concatenate. A conforming identity validator must diagnose conflicting +IDs and qualify collection addresses by repository, file, block and entry. + +The schema-2 collector diagnoses duplicate IDs within one file/block, qualifies +addresses by repository/revision/file/block/id and qualifies edge endpoints. +Ambiguous references remain unresolved with all candidate witnesses. The +visualizer uses those qualified addresses instead of collapsing same-name nodes. +The universal text parser alone still only parses syntax; it is not a validator. + +Use the helpers' matching `COMMENT_MARKERS` registries for supported repeated +line-comment syntax; do not duplicate their language lists in runners. Native +block comments, docstrings, XML documentation, or manifest syntax require their +own readers, not invalid adaptations of line-comment fences. Ambiguous suffixes +such as `.m` require explicit language context. Preserve valid first-line +shebangs and the separate [RATIOS boundary contract](../ratios/SKILL.md). + +## Field naming conventions + +Reserved fields retain their existing meanings: `id` identifies the entry; +`class` groups it; `summary` describes it; `call` addresses a witness; +`proves` names claimed obligations; `requires` is application-qualified; +`owner` declares responsibility; `since` records introduction; `deprecated` +records a retirement declaration. Observing a native deprecation grants no +permission to delete someone else's code. Authoring or retiring MSDMD mechanisms +requires the owning change and its supported replacement or explicit removal. + +## Worked example + +An unchanged repository contains: ```python -# === CHECKS === -# id: check_chat_get_other_owner_404_http -# proves: chat_get_other_owner_404 -# call: self::test_chat_get_other_owner_404_http -# requires: python3, posix_shell -# timeout: 20 -# mutates: db -# cleanup: transaction_rollback -# === END CHECKS === +# src/example/math.py +def double(value: int) -> int: + """Return twice the supplied value.""" + return value * 2 ``` -### Example (TypeScript source module) +```toml +# pyproject.toml +[project] +name = "example" +version = "1.0.0" +``` -```typescript -// === CONTRACTS === -// id: chat_input_send_disabled_while_pending -// given: a message is in flight -// then: send button is disabled and shows pending state -// class: ux_correctness -// === END CONTRACTS === +```text +# .github/CODEOWNERS +/src/example/ @example/maintainers ``` -### Example (Elixir) +With tested readers for those conventions, collect the signature and docstring +at symbol scope, package identity at package scope, and the applicable review +ownership rule at path scope. Retain every original source reference. No MSDMD +blocks need to be inserted. The return annotation is a declaration, not a test +result. CODEOWNERS review responsibility is not automatically authorship, +operational ownership, or proof of a team's live permissions. -```elixir -# === CAPABILITIES === -# id: agent_supervisor_dynamic_spawn -# summary: spawns child agents under a DynamicSupervisor with max_children=cap -# exposes: AgentSupervisor.start_child/1 -# === END CAPABILITIES === -``` +The integrated collector recovers all three native sources in this example. +Required information still needs an explicit policy; signatures, review rules +and package identity do not imply complete behavioral or operational coverage. -The block content is identical across languages — only the comment -marker changes. +## Repo collection point and visualizer -## The parser contract +### Native-capable runner usage -A msdmd parser is a pure function over file text: +`skills.json` identifies `msdmd/collect.py` as the native-and-supplemental runner. +The generated `_msdmd.ts` is disposable, never manually maintained. +Its `gaps` records supplemental block adoption, not native-information absence. +Every output includes the registry manifests, source digests, reader runs, +discovery ledger, facts, declarations, qualified edges, conflicts and diagnostics. -``` -parse(file_text: str, block_name: str) -> list[Entry] +```bash +python -m msdmd.collect --root . --repo example --out example_msdmd.ts --strict +python -m msdmd.visualize example_msdmd.ts --out example_msdmd.mmd ``` -where `Entry` is a flat `dict[str, str]` containing at minimum the -`id` field plus whatever fields the entry declared. The parser: - -- Returns all entries from all matching blocks (using - `re.finditer`-style iteration, not just the first block). -- Does not interpret or validate field semantics — that's the - application's job. An entry missing a required field surfaces as an - error in the executor, not in the parser. -- Does not fail on missing block type — returns empty list if no block - of that name exists. - -A reference implementation in pure stdlib Python lives at -`parsers/universal.py`; the TypeScript equivalent at `parsers/universal.ts`. -Both commit to zero non-stdlib dependencies so you can copy them into -any project. - -Extension detection refuses ambiguous suffixes rather than sniffing content. -For example, `.m` can mean Objective-C or MATLAB/Octave and therefore has no -automatic marker. A caller that already knows the language may still call -`parse_text` / `parseText` with an explicit marker. Languages that cannot carry -the msdmd shape as repeated line comments need a future versioned syntax -extension; they are not approximated with an invalid fence. - -The paired RATIOS helper preserves the interpreter boundary: a non-empty -line-1 shebang may precede opening RATIOS, which must then occupy line 2 with no -gap. See [`ratios/SKILL.md`](../ratios/SKILL.md) for the complete seal contract. - -## Repo collection point and visualizer +For a checked-in reproducible collection, use `--snapshot-identity` to bind facts +to the exact configured source-byte snapshot rather than the commit containing +the generated output. Excluded subtree rules, rather than transient cache +directory presence, are recorded in this snapshot mode. `--check` recomputes and compares bytes without writing: -Every consuming repo SHOULD maintain one repo-level collection point named -`_msdmd.ts` (for example, `a0_msdmd.ts`). This file is the -canonical aggregation surface for all parsed msdmd declarations in that -repo. It does not replace module-local blocks; it is generated from them -or maintained as a thin index over them. - -The collection point SHOULD use the shared shapes in `msdmd/collection.ts` -(or a verbatim copy in consuming repos) and export a `MsdmdCollection`: - -```typescript -import { defineMsdmdCollection } from "./.agents/skills/msdmd/collection"; - -export default defineMsdmdCollection({ - repo: "", - declarations: [ - { file: "path/to/module.py", block: "CONTRACTS", id: "...", fields: { summary: "..." } }, - { file: "tests/test_module.py", block: "CHECKS", id: "...", fields: { proves: "..." } }, - ], - gaps: [ - { file: "path/to/module.py", missing: ["CONTRACTS", "DOCS"] }, - ], - edges: [ - { from: "module_a", to: "module_b", kind: "requires", source_block: "DEPENDENCIES", source_id: "..." }, - { from: "check_module_a", to: "module_a_contract", kind: "claims_proves", source_block: "CHECKS", source_id: "..." }, - ], -}); - -export const declarations = []; -export const gaps = []; +```bash +python -m msdmd.collect --root . --repo The-Interdependency/skill-lib \ + --snapshot-identity --import-path ./msdmd/collection --out skill-lib_msdmd.ts --strict +python -m msdmd.collect --root . --repo The-Interdependency/skill-lib \ + --snapshot-identity --import-path ./msdmd/collection --out skill-lib_msdmd.ts --strict --check ``` -A repo-level msdmd visualizer SHOULD read `_msdmd.ts` and render -relationships between modules using the `MsdmdEdge` shape: -`DEPENDENCIES.requires`, `CAPABILITIES.exposes`, `OWNERS.owner`, -`BOUNDARIES` risk fields, `DOCS.covers`, `CHECKS.call`, -`CHECKS.proves` as `claims_proves`, and any `requires` edges shared -across application skills. The visualizer is a consumer of the -collection point, not a second metadata source. +Source revision, worktree state and byte digests remain separate. A snapshot is +not producer authentication. Excluded output paths are configured even before +their first write, preventing generation from changing its own source identity. -If a repo has no collection point or visualizer yet, record that as `hmmm` in -repo-local planning rather than pretending the graph exists. +### Required information and disclosure -A small stdlib generator prototype lives at `msdmd/collect.py`. Consuming repos -can run it directly or copy it as a starting point: +`--expected-block` measures block adoption only. `--require-source GLOB` demands +complete extraction for each matching file. Repeat `--require-fact` to require +source-linked witnesses from a named native convention and kind: ```bash -python -m msdmd.collect --root . --repo --out _msdmd.ts +python -m msdmd.collect --root . --repo example --json --strict \ + --require-source 'package.json' \ + --require-fact 'package.json::npm.package-json::dependency' ``` -The generator is intentionally conservative: it parses module-local blocks, -emits declarations, optional expected-block gaps, and simple relationship -edges from reserved fields. Repo-specific runners may enrich the output, but -should preserve the `MsdmdCollection` shape. +With a required policy, no matches, invalid sources, missing runtimes, ambiguous +syntax or unresolved extraction fail strict mode. Without a policy, optional +unknowns remain visible; a successful inventory is not a full-coverage verdict. +Parse errors and identity conflicts always invalidate strict collection. Native +facts and blocks remain separate; no universal precedence overwrites disagreements. -A minimal Mermaid visualizer prototype lives at `msdmd/visualize.py` and reads -raw JSON or generated TypeScript collection points: +Directory-descriptor discovery does not follow symlinks. It accounts for ignored +subtrees, unsupported inputs, byte limits and read errors. This safety path +requires POSIX no-follow directory-descriptor support. Secret-named files are +excluded; sensitive structured fields and URL credentials are redacted. This is +not a complete secret detector: public release still requires audience review. +The configured denominator never implies that excluded subtrees were inspected. -```bash -python -m msdmd.visualize _msdmd.ts --out _msdmd.mmd -``` +### Consumer transition -The visualizer is deliberately small: it renders declaration nodes, normalized -edge relationships, and visible gap nodes. Rich repo-specific UIs should consume -the same collection shape rather than re-parsing source files. +Schema 2 is the default. `--legacy-blocks-only` explicitly requests schema 1 for +existing block-only consumers; native requirements are rejected with that flag. +No native data is silently flattened into legacy block fields. The visualizer +supports both versions. Application-specific documentation renderers, ownership +policy and runtime witnesses retain their own semantics and acceptance gates. +## Validation and acceptance -## The runner protocol +The [acceptance matrix](references/metadata-conventions.md#acceptance-matrix) +defines the native-reader tests. Its decisive case is an unchanged repository +with supported native conventions and zero MSDMD blocks: metadata is collected +accurately, unknowns stay visible, and no false missing-information findings are +manufactured. Pair it with cases where required information really is missing. -A msdmd runner combines a parser and an executor: +For a skill/index edit, run the repository's editorial gates separately: -``` -walk(root: Path, block_name: str) -> Iterator[(file: Path, entries: list[Entry])] +```bash +python tools/build_codex_plugin_skills.py --apply +python tools/build_codex_plugin_skills.py --check +python tools/check_skill_lib_drift.py --warnings-fail +python tools/check_skill_compliance.py --warnings-fail +python -m unittest discover -s tests ``` -Implementation rules every runner MUST follow: - -1. **Walk the source tree** under a configurable root, skipping - conventional non-source paths (`__pycache__`, `node_modules`, - `.git`, build outputs, the runner's own test directory). -2. **Detect comment marker by extension**, not by content sniffing. Consume the - parser's `COMMENT_MARKERS` registry rather than maintaining a runner-local - language list. Python and TypeScript registries must remain identical. -3. **Parse all matching blocks** in each file. Multiple blocks of the - same type concatenate; entries from different blocks are - distinguishable only by id, not by source block. -4. **Visit modules without any block of the requested type** and emit - them as a separate "untested" / "undocumented" / "uncapable" gap - list. Truncate noise (e.g. show first 20, count the rest), but - never silently drop. Visibility is the whole point. -5. **Exit non-zero** when any entry fails the executor's check. The - gap list itself is informational unless the application opts in to - strict mode (in which case missing blocks are also a fail). +Update the canonical description, `skills.json`, generated Codex adapter, and +README together. Editorial tests passing do not establish native-reader support. +Load this skill with the applicable doc/cap/deps/owner/test/boundary/manifest/ +ratios/LLMS/frontend skill; preserve that application's semantic obligations. -## Field naming conventions +## Anti-patterns -Reserved field names and their canonical meanings (for cross-skill -consistency): - -| Field | Meaning | -|---|---| -| `id` | Unique stable identifier within the block. Required on every entry. | -| `class` | Free-text tag for grouping (`security`, `correctness`, `idempotency`, etc.). The runner counts entries per class in summaries. | -| `call` | Executable target owned by an evidence/check declaration. Source `CONTRACTS` do not use this field for test topology. | -| `proves` | Comma-separated ids this evidence/check entry claims to prove. The collection edge kind is `claims_proves`; mutation sensitivity is a higher verification rung. | -| `summary` | One-sentence human description. | -| `requires` | Comma-separated dependency ids or host capabilities. Exact semantics are application-specific and must be documented by the skill that consumes it. | -| `owner` | Who is responsible (person, agent role, team). | -| `since` | Version or date this declaration was added. | -| `deprecated` | If present, marks the entry as scheduled for removal. | - -Application-specific fields (`given`, `then`, `expects`, `inputs`, -`outputs`, `mutates`, `cleanup`, `timeout`, etc.) are introduced by -individual SKILLs and documented in their own SKILL.md. - -## Authoring a new msdmd application - -1. **Pick a block name** that doesn't collide with an existing - application. Search the lib README for current names. -2. **Define the field schema** — which fields are required, which - optional, what types they carry. Document in your SKILL.md. -3. **Write the executor** — the function that takes parsed entries - and acts on them. Use the universal parser; do not write a new - one unless your block needs syntax the universal parser can't - express. -4. **Implement the visibility report** — your runner must list - modules without your block type as gaps, and the gap list must - be visible in normal output (not buried behind a flag). -5. **Author a SKILL.md** in this lib with the convention spec, the - executor's behavior, and at least one worked example. - -`test-build/` is the canonical reference application for paired source -`CONTRACTS` and test `CHECKS`. Read its SKILL.md alongside this one to -see the pattern fully realized; read `doc-build/`, `cap-build/`, -`deps-build/`, `owner-build/`, `risk-boundary-build/`, and `ratios/` -for additional applications over the same parser contract. +- Requiring native declarations to be copied into MSDMD comments. +- Advertising a listed convention as an implemented and tested reader. +- Treating unsupported syntax as absent metadata, or an import as a full call graph. +- Flattening structured metadata, dropping unknown tags, or silently resolving conflicts. +- Using manifests, docs, examples, test names, or reports as unqualified proof. +- Executing project code, following untrusted instructions, or exporting secrets during collection. +- Re-ingesting generated collections as independent evidence of their own inputs. -## Anti-patterns +## Versioning and migration + +The stable supplemental block grammar is unchanged. Native readers, schema, +projection mappings and dependencies have explicit versions. Schema-1 output is +an intentional compatibility operation, not a second default scanner. + +## hmmm -- **Don't define an owned declaration in a detached side file.** The - whole point is that the declaration lives next to the module that - owns that fact. Source obligations belong in source; test evidence - belongs in the test module that owns the evidence. -- **Don't put `call:` in source `CONTRACTS`.** Source modules own - obligations, not test topology. Put executable targets in `CHECKS`. -- **Don't make ids reflect implementation details.** `chat_returns_200` - tells future-you nothing; `chat_get_other_owner_404` tells you what's - protected. Ids are part of the documentation. -- **Don't silently drop modules without blocks.** Coverage gaps must be - visible. If your runner doesn't emit the gap list, it's not a msdmd - runner; it's a test discovery tool with extra steps. -- **Don't introduce parser dialects.** If you need richer syntax than - the universal parser handles, propose an extension to msdmd, not a - fork. The portability of the convention depends on the parser - contract being one thing. - -## Versioning - -- **Block syntax is stable.** Breaking changes (renaming the fence, - changing field-line indentation rules, etc.) go through a major - version bump and a migration note in the lib README. -- **Reserved field names** above are stable. New reserved names are - additive only. -- **Application SKILLs** version independently in their own SKILL.md - files. +The shipped matrix defines extraction subsets, not every metadata standard. +Unimplemented conventions remain visible in discovery. `.h` language ambiguity, +macro/build/configuration expansion, full TSDoc validation, cross-source semantic +conflict resolution, ownership applicability and independent attestation/runtime +verification require their owning policies. None is inferred from a clean parse. diff --git a/.agents/skills/msdmd/collect.py b/.agents/skills/msdmd/collect.py index 74b52f6d..390afb6c 100644 --- a/.agents/skills/msdmd/collect.py +++ b/.agents/skills/msdmd/collect.py @@ -1,18 +1,72 @@ -# ratios: loc_comments=143:7 imports_exports=6:3 calls_definitions=35:6 -"""Generate repo-level msdmd collection-point TypeScript. +# ratios: loc_comments=706:54 imports_exports=17:5 calls_definitions=213:21 +# === DOCS === +# id: msdmd_foundational_contract +# source: msdmd/SKILL.md +# catalogue: msdmd/references/metadata-conventions.md +# summary: Native-first metadata collection with source-qualified facts, diagnostics, and coverage policy. Supplemental blocks express otherwise unexpressed information. +# === END DOCS === +# === CAPABILITIES === +# id: repo_collection_generator +# exposes: collect, render_typescript, collection_errors +# summary: Produces schema-2 collections from registered native readers and supplemental blocks without executing inspected sources. +# runner_scope: native-and-supplemental-blocks +# native_ingestion: implemented-tested-subsets +# limitations: Supported features and versions are reader-qualified; declarations and parsed reports do not prove runtime behavior. Required-source policy detects incomplete extraction. +# === END CAPABILITIES === +# === MODULE_BUILD === +# id: msdmd_collection_generator +# module_name: collect +# module_kind: instrument +# summary: produces versioned native-first MSDMD collections without executing inspected sources +# owner: The Interdependency skill-lib +# public_surface: collect, collect_legacy, collection_errors, render_typescript +# internal_surface: bounded discovery, block identity validation, edge reconciliation, coverage accounting +# auth_boundary: none +# storage_boundary: read +# network_boundary: none +# user_data_boundary: read +# admin_only: false +# tests: tests/test_collect.py, tests/test_native_collection.py +# rollout: schema 2 is the default CLI output; --legacy-blocks-only keeps explicit schema 1 compatibility +# rollback: use --legacy-blocks-only while preserving schema incompatibility visibility +# === END MODULE_BUILD === +"""Generate a versioned native-first MSDMD collection. -This is a small stdlib helper for consuming repos that want to generate a -`_msdmd.ts` aggregation file from module-local msdmd blocks. -It uses the universal parser and emits data shaped by `msdmd/collection.ts`. +Usage guidance: + python -m msdmd.collect --root . --repo owner/repo --out repo_msdmd.ts + python -m msdmd.collect --root . --repo owner/repo --strict + python -m msdmd.collect --root . --repo owner/repo --legacy-blocks-only + +The default schema-2 path statically reads supported native conventions and +supplemental MSDMD blocks. It never imports inspected code, executes scripts, +loads plugins, follows network references, or treats declarations as verified +behavior. The explicit legacy flag emits the previous block-only shape for old +consumers; it cannot silently represent native facts. """ from __future__ import annotations import argparse +import fnmatch +import tempfile +import hashlib import json +import os +import stat +import subprocess +import io +import tokenize +from collections import Counter, defaultdict from pathlib import Path -from typing import Iterable +from typing import Any, Iterable +from urllib.parse import quote + +from msdmd.parsers.universal import marker_for, parse_text +from msdmd.readers import READER_MANIFESTS, _redact_sensitive, read_native -from msdmd.parsers.universal import walk_tree +SCHEMA_ID = "the-interdependency.msdmd-collection" +SCHEMA_VERSION = "2.0.0" +LEGACY_SCHEMA_VERSION = "1.0.0" +DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 DEFAULT_BLOCK_NAMES = ( "DOCS", @@ -23,7 +77,6 @@ "CHECKS", "MODULE_BUILD", "BOUNDARIES", - "RATIOS", "LLMS", "FRONTEND_META", ) @@ -38,42 +91,327 @@ "boundaries": "risk", } +_DEFAULT_SKIP = { + ".git", + ".agents", + ".skill-lib", + ".venv", + ".mypy_cache", + ".pytest_cache", + ".tox", + "__pycache__", + "build", + "dist", + "node_modules", + "target", + "venv", +} + + +def _sha256(data: bytes | str) -> str: + payload = data if isinstance(data, bytes) else data.encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _quote(value: str, *, path: bool = False) -> str: + return quote(value, safe="/" if path else "") + def _split_targets(value: str) -> list[str]: return [part.strip() for part in value.split(",") if part.strip()] -def _declaration(file: Path, root: Path, block: str, entry: dict) -> dict: - fields = {str(key): str(value) for key, value in entry.items() if key != "id"} +def _block_source_text(path: Path, text: str) -> str: + """Return only real Python comments, or unchanged non-Python source. + + The universal block grammar remains language-neutral. Python collection + adds syntax awareness here so block-shaped examples inside strings are not + collected as declarations. + """ + if path.suffix.lower() not in {".py", ".pyi", ".pyw"}: + return text + text = text.replace("\r\n", "\n").replace("\r", "\n") + comments = [""] * len(text.split("\n")) + try: + for token in tokenize.generate_tokens(io.StringIO(text).readline): + if token.type == tokenize.COMMENT: + comments[token.start[0] - 1] = token.string + except (tokenize.TokenError, IndentationError): + return "" + return "\n".join(comments) + + +def _decode_block_source(path: Path, data: bytes) -> str: + """Decode Python by its declared source encoding; other sources as UTF-8.""" + if path.suffix.lower() in {".py", ".pyi", ".pyw"}: + encoding, _ = tokenize.detect_encoding(io.BytesIO(data).readline) + return data.decode(encoding) + return data.decode("utf-8") + + +def _block_address(repo: str, revision: str, file: str, block: str, entry_id: str) -> str: + return ( + f"msdmd://{_quote(repo)}@{_quote(revision)}/{_quote(file, path=True)}" + f"#block/{_quote(block)}/{_quote(entry_id)}" + ) + + +def _block_declaration( + *, + file: str, + repo: str, + revision: str, + content_sha256: str, + block: str, + entry: dict[str, str], +) -> dict[str, Any]: + entry_id = str(entry["id"]) return { - "file": file.relative_to(root).as_posix(), + "address": _block_address(repo, revision, file, block, entry_id), + "origin": "msdmd-block", + "standing": "declared", + "source": { + "repository": repo, + "revision": revision, + "file": file, + "content_sha256": content_sha256, + "location": {"pointer": f"block:{block}/id:{entry_id}"}, + }, + "file": file, "block": block, - "id": str(entry["id"]), - "fields": fields, + "id": entry_id, + "fields": {str(key): str(value) for key, value in entry.items() if key != "id"}, } -def _edges_for(declaration: dict) -> list[dict]: - edges: list[dict] = [] - fields = declaration["fields"] - source = declaration["id"] +def _raw_block_edges(declaration: dict[str, Any]) -> list[dict[str, Any]]: + edges: list[dict[str, Any]] = [] for field, kind in EDGE_FIELDS.items(): - value = fields.get(field) + value = declaration["fields"].get(field) if not value or value == "hmmm": continue for target in _split_targets(value): - edges.append( - { - "from": source, - "to": target, - "kind": kind, - "source_block": declaration["block"], - "source_id": source, - } - ) + edges.append({ + "from": declaration["address"], + "to": target, + "kind": kind, + "standing": "declared", + "source_block": declaration["block"], + "source_id": declaration["address"], + "source_entry_id": declaration["id"], + "target_resolution": "pending", + }) return edges +def _git_identity(root: Path, explicit_revision: str | None) -> tuple[str, bool | str, str | None]: + revision = explicit_revision + dirty: bool | str = "hmmm" + git_head: str | None = None + try: + git_head = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if revision is None: + revision = git_head + status_output = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain", "--untracked-files=all"], + check=True, + capture_output=True, + text=True, + ).stdout + input_changes = [] + for line in status_output.splitlines(): + changed_path = line[3:].split(" -> ")[-1].strip('"') if len(line) > 3 else "" + if not Path(changed_path).name.endswith("_msdmd.ts"): + input_changes.append(line) + dirty = bool(input_changes) + except (OSError, subprocess.CalledProcessError): + pass + return revision or "hmmm", dirty, git_head + + +def _discover(root: Path, max_file_bytes: int, generated_outputs: Iterable[str] = (), *, stable_exclusions: bool = False) -> tuple[list, list, list]: + """Read bounded files through directory descriptors; never follow symlinks.""" + if max_file_bytes < 1: + raise ValueError("max_file_bytes must be positive") + readable: list = [] + ledger: list = [] + diagnostics: list = [] + if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY"): + raise OSError("safe native discovery requires no-follow directory descriptor support") + + def excluded(relative: str, reason: str, *, entry_kind: str = "file", digest: str = "hmmm", size: int | None = None) -> None: + item = {"file": relative, "entry_kind": entry_kind, "status": "excluded", "reason": reason, + "reader_ids": [], "content_sha256": digest} + if size is not None: + item["size"] = size + ledger.append(item) + + if stable_exclusions: + for name in sorted(_DEFAULT_SKIP): + excluded("**/" + name + "/**", "configured-subtree", entry_kind="subtree") + generated_outputs = set(generated_outputs) + for output in sorted(generated_outputs): + excluded(output, "generated-collection-output", entry_kind="output") + + def failed(relative: str, exc: Exception, entry_kind: str = "file") -> None: + ledger.append({"file": relative, "entry_kind": entry_kind, "status": "unreadable", "reader_ids": [], "content_sha256": "hmmm"}) + diagnostics.append({"code": "unreadable_path", "severity": "error", "status": "unreadable", + "message": type(exc).__name__, "source": {"file": relative}, "reader_id": None}) + + def visit(directory_fd: int, parent: str = "") -> None: + try: + names = sorted(os.listdir(directory_fd)) + except OSError as exc: + failed(parent or ".", exc, "subtree") + return + for name in names: + relative = f"{parent}/{name}" if parent else name + try: + if relative in generated_outputs: + continue + mode = os.stat(name, dir_fd=directory_fd, follow_symlinks=False).st_mode + if stat.S_ISLNK(mode): + excluded(relative, "symlink-not-followed", digest=_sha256(os.readlink(name, dir_fd=directory_fd))) + continue + if stat.S_ISDIR(mode): + if name in _DEFAULT_SKIP: + if not stable_exclusions: + excluded(relative, "configured-subtree", entry_kind="subtree") + else: + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory_fd) + try: + visit(child, relative) + finally: + os.close(child) + continue + if not stat.S_ISREG(mode): + excluded(relative, "not-regular-file") + continue + if name.endswith("_msdmd.ts"): + excluded(relative, "generated-collection-output") + continue + if name.startswith(".env") or name.endswith((".pem", ".key")): + excluded(relative, "secret-file-disclosure-boundary") + continue + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd) + with os.fdopen(fd, "rb") as handle: + info = os.fstat(handle.fileno()) + if not stat.S_ISREG(info.st_mode): + excluded(relative, "not-regular-file") + continue + if info.st_size > max_file_bytes: + excluded(relative, "size-limit", size=info.st_size) + continue + data = handle.read(max_file_bytes + 1) + if len(data) > max_file_bytes: + excluded(relative, "size-limit", size=len(data)) + continue + readable.append({"file": relative, "data": data, "content_sha256": _sha256(data), "size": len(data)}) + except OSError as exc: + failed(relative, exc) + try: + root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError as exc: + failed(".", exc, "subtree") + else: + try: + visit(root_fd) + finally: + os.close(root_fd) + return readable, ledger, diagnostics + + +def _reconcile_block_edges( + raw_edges: list[dict[str, Any]], + declarations: list[dict[str, Any]], + diagnostics: list[dict[str, Any]], +) -> list[dict[str, Any]]: + by_id: dict[str, list[str]] = defaultdict(list) + for declaration in declarations: + by_id[declaration["id"]].append(declaration["address"]) + edges: list[dict[str, Any]] = [] + for edge in raw_edges: + candidates = by_id.get(edge["to"], []) + if len(candidates) == 1: + edge["to"] = candidates[0] + edge["target_resolution"] = "resolved-unique-entry-id" + elif len(candidates) > 1: + diagnostics.append({ + "code": "ambiguous_edge_target", + "severity": "warning", + "status": "ambiguous", + "message": f"target {edge['to']!r} matches {len(candidates)} qualified declarations", + "source": {"address": edge["source_id"]}, + "reader_id": "msdmd-block", + "candidates": sorted(candidates), + }) + edge["target_resolution"] = "ambiguous" + else: + edge["target_resolution"] = "external-or-unresolved" + edges.append(edge) + return edges + + +def _duplicate_diagnostics(declarations: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + for declaration in declarations: + groups[(declaration["file"], declaration["block"], declaration["id"])].append(declaration) + diagnostics: list[dict[str, Any]] = [] + conflicts: list[dict[str, Any]] = [] + for (file, block, entry_id), group in sorted(groups.items()): + if len(group) < 2: + continue + address = group[0]["address"] + diagnostics.append({ + "code": "duplicate_block_id", + "severity": "error", + "status": "invalid", + "message": f"{file} contains {len(group)} {block} entries with id {entry_id!r}", + "source": {"address": address, "file": file}, + "reader_id": "msdmd-block", + }) + conflicts.append({ + "kind": "identity-conflict", + "status": "unresolved", + "identity": address, + "witnesses": [item["source"] for item in group], + "reason": "duplicate entry id within one block type and owning file", + }) + return diagnostics, conflicts + + +def _native_identity_diagnostics(facts: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for fact in facts: + groups[fact["address"]].append(fact) + diagnostics: list[dict[str, Any]] = [] + conflicts: list[dict[str, Any]] = [] + for address, group in sorted(groups.items()): + if len(group) < 2: + continue + diagnostics.append({ + "code": "duplicate_native_fact_address", + "severity": "error", + "status": "invalid", + "message": f"collector generated {len(group)} native facts at one address", + "source": {"address": address, "file": group[0]["source"]["file"]}, + "reader_id": group[0]["extraction"]["reader_id"], + }) + conflicts.append({ + "kind": "collector-address-conflict", + "status": "unresolved", + "identity": address, + "witnesses": [item["source"] for item in group], + "reason": "multiple native facts received one generated collector address", + }) + return diagnostics, conflicts + + def collect( root: Path, repo: str, @@ -81,97 +419,402 @@ def collect( block_names: Iterable[str] = DEFAULT_BLOCK_NAMES, expected_blocks: Iterable[str] = (), source_commit: str | None = None, -) -> dict: - """Collect msdmd declarations and optional coverage gaps under ``root``.""" + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + include_native: bool = True, + snapshot_identity: bool = False, + generated_outputs: Iterable[str] = (), + required_sources: Iterable[str] = (), + required_facts: Iterable[dict[str, str]] = (), +) -> dict[str, Any]: + """Collect native facts and supplemental blocks under ``root`` as schema 2.""" root = root.resolve() block_names = tuple(block_names) expected_blocks = tuple(expected_blocks) + revision, dirty, git_head = _git_identity(root, source_commit) + outputs = set(generated_outputs) | {repo.rsplit("/", 1)[-1] + "_msdmd.ts"} + readable, discovery, diagnostics = _discover(root, max_file_bytes, outputs, stable_exclusions=snapshot_identity) + required_sources = tuple(required_sources) + required_facts = tuple(required_facts) + for requirement in required_facts: + if set(requirement) != {"file", "convention", "kind"} or not all(isinstance(v, str) and v for v in requirement.values()): + raise ValueError("fact requirements must contain nonempty file, convention and kind strings") + codeowners_source = next((name for name in (".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS") if any(item["file"] == name for item in readable)), None) + if snapshot_identity: + if source_commit is not None: + raise ValueError("snapshot identity and an explicit commit cannot both be selected") + source_bytes = "\n".join(item["file"] + "\0" + item["content_sha256"] for item in sorted(readable, key=lambda x: x["file"])) + revision = "snapshot:" + _sha256(source_bytes) + dirty, git_head = "hmmm", None + if source_commit is not None and git_head is not None and source_commit != git_head: + diagnostics.append({ + "code": "source_revision_mismatch", + "severity": "error", + "status": "invalid", + "message": f"declared source revision {source_commit} does not match git HEAD {git_head}", + "source": {"file": ".", "declared_revision": source_commit, "git_head": git_head}, + "reader_id": None, + }) - declarations: list[dict] = [] - missing_by_file: dict[str, set[str]] = {} + declarations: list[dict[str, Any]] = [] + facts: list[dict[str, Any]] = [] + native_edges: list[dict[str, Any]] = [] + raw_block_edges: list[dict[str, Any]] = [] + missing_by_file: dict[str, set[str]] = defaultdict(set) + reader_counts: Counter[str] = Counter() + reader_fact_counts: Counter[str] = Counter() + block_files: dict[str, set[str]] = defaultdict(set) - for block in block_names: - annotated, _ = walk_tree(root, block) - for file, entries in annotated: + for item in readable: + relative = item["file"] + relative_path = Path(relative) + context = { + "repo": repo, + "revision": revision, + "file": relative, + "content_sha256": item["content_sha256"], + "codeowners_source": codeowners_source, + } + reader_ids, file_facts, file_edges, file_diagnostics = ( + read_native(relative_path, item["data"], context) + if include_native + else ([], [], [], []) + ) + file_statuses = {diagnostic.get("status") for diagnostic in file_diagnostics} + if "invalid" in file_statuses: + status = "invalid" + elif file_statuses & {"unsupported", "ambiguous", "dynamic-unresolved"} and file_facts: + status = "partial" + elif "dynamic-unresolved" in file_statuses: + status = "partial" + elif file_statuses & {"unsupported", "ambiguous"}: + status = "unsupported" + else: + status = "supported" if reader_ids else "unsupported" + discovery.append({ + "file": relative, + "entry_kind": "file", + "status": status, + "reader_ids": reader_ids, + "content_sha256": item["content_sha256"], + "size": item["size"], + }) + for reader_id in reader_ids: + reader_counts[reader_id] += 1 + for fact in file_facts: + reader_fact_counts[fact["extraction"]["reader_id"]] += 1 + facts.extend(file_facts) + native_edges.extend(file_edges) + diagnostics.extend(file_diagnostics) + + marker = marker_for(relative_path) + if marker is None: + continue + try: + text = _decode_block_source(relative_path, item["data"]) + except (SyntaxError, UnicodeDecodeError) as exc: + diagnostics.append({ + "code": "undecodable_block_source", + "severity": "error", + "status": "invalid", + "message": str(exc), + "source": {"file": relative, "content_sha256": item["content_sha256"]}, + "reader_id": "msdmd-block", + }) + continue + block_text = _block_source_text(relative_path, text) + if include_native and relative_path.suffix.lower() in {".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".rs", ".java", ".c", ".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h"}: + comments_available = any(f['kind'] == 'comment' and f['extraction']['reader_id'] in {'typescript-compiler', 'native-grammars'} for f in file_facts) + if not comments_available and file_statuses & {'unsupported', 'ambiguous', 'invalid'}: + candidates = [] + for candidate_block in block_names: + for entry in parse_text(text, candidate_block, marker): + candidate, _ = _redact_sensitive({'block': candidate_block, 'entry': entry}) + candidates.append(candidate) + if candidates: + diagnostics.append({'code': 'supplemental_comment_extraction_unavailable', 'severity': 'error', 'status': 'unsupported', + 'message': 'block-shaped candidates retained without certifying comment origin; install or select the native parser', + 'source': {'file': relative, 'content_sha256': item['content_sha256'], 'candidate_standing': 'unverified', 'candidate_blocks': candidates}, + 'reader_id': 'msdmd-block'}) + lines = [""] * len(text.split("\n")) + for fact in file_facts: + if fact["kind"] == "comment" and fact["extraction"]["reader_id"] in {"typescript-compiler", "native-grammars"}: + for offset, line in enumerate(fact["native"]["value"]["text"].split("\n")): + index = fact["source"]["location"]["start_line"] - 1 + offset + if index < len(lines): + lines[index] = line + block_text = "\n".join(lines) + for block in block_names: + entries = parse_text(block_text, block, marker) + if entries: + block_files[block].add(relative) + elif block in expected_blocks: + missing_by_file[relative].add(block) for entry in entries: if "id" not in entry: continue - declarations.append(_declaration(file.resolve(), root, block, entry)) + declaration = _block_declaration( + file=relative, + repo=repo, + revision=revision, + content_sha256=item["content_sha256"], + block=block, + entry=entry, + ) + declarations.append(declaration) + raw_block_edges.extend(_raw_block_edges(declaration)) - for block in expected_blocks: - _, missing_files = walk_tree(root, block) - for file in missing_files: - relative = file.resolve().relative_to(root).as_posix() - missing_by_file.setdefault(relative, set()).add(block) - - declarations.sort(key=lambda item: (item["file"], item["block"], item["id"])) + declarations.sort(key=lambda item: (item["file"], item["block"], item["id"], json.dumps(item["fields"], sort_keys=True))) + facts.sort(key=lambda item: item["address"]) + duplicate_diagnostics, conflicts = _duplicate_diagnostics(declarations) + diagnostics.extend(duplicate_diagnostics) + native_diagnostics, native_conflicts = _native_identity_diagnostics(facts) + diagnostics.extend(native_diagnostics) + conflicts.extend(native_conflicts) + block_edges = _reconcile_block_edges(raw_block_edges, declarations, diagnostics) + edges = native_edges + block_edges + edges.sort(key=lambda item: (item["from"], item["kind"], item["to"])) gaps = [ - {"file": file, "missing": sorted(missing)} + {"file": file, "missing": sorted(missing), "kind": "block-adoption"} for file, missing in sorted(missing_by_file.items()) ] - edges = [edge for declaration in declarations for edge in _edges_for(declaration)] - edges.sort(key=lambda item: (item["source_block"], item["source_id"], item["kind"], item["to"])) + discovery.sort(key=lambda item: item["file"]) + diagnostics.sort(key=lambda item: (str(item.get("source", {}).get("file", item.get("source", {}).get("address", ""))), item["code"], item["message"])) - collection = { + snapshot_entries = [ + f"{item['file']}\0{item.get('content_sha256', 'hmmm')}\0{item['status']}" + for item in discovery + ] + snapshot_sha256 = _sha256("\n".join(snapshot_entries)) + statuses = Counter(item["status"] for item in discovery if item.get("entry_kind", "file") == "file") + for pattern in required_sources: + matches = [item for item in discovery if item.get("entry_kind", "file") == "file" and fnmatch.fnmatchcase(item["file"], pattern)] + bad = [item for item in matches if item["status"] != "supported"] + if not matches or bad: + diagnostics.append({"code": "required_source_not_supported", "severity": "error", "status": "unsupported", + "message": "required source pattern has no matches or incomplete extraction: " + pattern, + "source": {"file": pattern, "matches": [{"file": i["file"], "status": i["status"]} for i in matches]}, "reader_id": None}) + requirements = [] + for requirement in required_facts: + matches = [item for item in discovery if item.get("entry_kind", "file") == "file" and fnmatch.fnmatchcase(item["file"], requirement["file"])] + results = [] + for item in matches: + witnesses = [fact["address"] for fact in facts if fact["source"]["file"] == item["file"] and fact["convention"]["namespace"] == requirement["convention"] and fact["kind"] == requirement["kind"]] + status = "provided" if witnesses and item["status"] == "supported" else "unresolved" if item["status"] != "supported" else "not-found-in-extracted-subset" + results.append({"file": item["file"], "status": status, "witnesses": witnesses}) + status = "provided" if results and all(r["status"] == "provided" for r in results) else "unresolved" + requirements.append({**requirement, "status": status, "sources": results}) + if status != "provided": + diagnostics.append({"code": "required_fact_unresolved", "severity": "error", "status": "unsupported", + "message": "required native fact is not supported by complete source extraction", + "source": dict(requirement), "reader_id": None}) + reader_runs = [] + for manifest in READER_MANIFESTS: + reader_id = manifest["reader_id"] + matched = reader_counts[reader_id] + reader_runs.append({ + "reader_id": reader_id, + "reader_version": manifest["version"], + "support": manifest["support"], + "status": "applied" if matched else "not-applicable", + "files_matched": matched, + "facts_emitted": reader_fact_counts[reader_id], + }) + + return { + "schema": SCHEMA_ID, + "schema_version": SCHEMA_VERSION, + "capabilities": [ + "native-facts", + "supplemental-msdmd-blocks", + "source-qualified-identities", + "duplicate-id-diagnostics", + "discovery-accounting", + ], "repo": repo, + "source": { + "repository": repo, + "revision": revision, + "git_head": git_head or "hmmm", + "dirty_worktree": dirty, + "snapshot_sha256": snapshot_sha256, + "snapshot_complete": not any(item["status"] == "unreadable" or (item["status"] == "excluded" and item.get("reason") not in {"configured-subtree", "generated-collection-output"}) for item in discovery), + "revision_kind": "content-snapshot" if snapshot_identity else "git-or-declared", + "scope": "configured in-scope files; excluded subtrees are reported without traversing", + "exclusion_accounting": "configured-patterns" if snapshot_identity else "observed-paths", + }, + "requirements": requirements, + "reader_manifests": list(READER_MANIFESTS), + "reader_runs": reader_runs, + "discovery": discovery, "declarations": declarations, + "facts": facts, "gaps": gaps, "edges": edges, + "conflicts": conflicts, + "diagnostics": diagnostics, + "coverage": { + "denominator": {"kind": "discovered-files", "count": sum(statuses.values())}, + "excluded_subtrees": sum(i.get("entry_kind") == "subtree" and i["status"] == "excluded" for i in discovery), + "required_sources": list(required_sources), + "supported_files": statuses["supported"], + "partial_files": statuses["partial"], + "unsupported_files": statuses["unsupported"], + "invalid_files": statuses["invalid"], + "excluded_files": statuses["excluded"], + "unreadable_files": statuses["unreadable"], + "native_facts": len(facts), + "supplemental_declarations": len(declarations), + "block_adoption": {block: len(block_files[block]) for block in block_names}, + "information_availability": "policy-evaluated" if required_facts else "not-evaluated-without-obligation-policy", + "verified_behavior": "not-evaluated", + }, } - if source_commit: - collection["source_commit"] = source_commit - return collection -def render_typescript(collection: dict, *, import_path: str) -> str: - """Render a collection as a `_msdmd.ts` module.""" - payload = json.dumps(collection, indent=2, sort_keys=True) - return ( - f'import {{ defineMsdmdCollection }} from "{import_path}";\n\n' - f"export default defineMsdmdCollection({payload});\n" +def collect_legacy( + root: Path, + repo: str, + *, + block_names: Iterable[str] = DEFAULT_BLOCK_NAMES, + expected_blocks: Iterable[str] = (), + source_commit: str | None = None, +) -> dict[str, Any]: + """Emit explicit schema-1 block-only compatibility output.""" + current = collect( + root, + repo, + block_names=block_names, + expected_blocks=expected_blocks, + source_commit=source_commit, + include_native=False, ) + declarations = [ + {"file": item["file"], "block": item["block"], "id": item["id"], "fields": item["fields"]} + for item in current["declarations"] + ] + legacy_by_address = {item["address"]: item["id"] for item in current["declarations"]} + edges = [] + for edge in current["edges"]: + if "source_block" not in edge: + continue + edges.append({ + "from": legacy_by_address.get(edge["from"], edge["from"]), + "to": legacy_by_address.get(edge["to"], edge["to"]), + "kind": edge["kind"], + "source_block": edge["source_block"], + "source_id": edge["source_entry_id"], + }) + result: dict[str, Any] = { + "schema_version": LEGACY_SCHEMA_VERSION, + "repo": repo, + "declarations": declarations, + "gaps": [{"file": item["file"], "missing": item["missing"]} for item in current["gaps"]], + "edges": edges, + } + if source_commit: + result["source_commit"] = source_commit + return result + + +def collection_errors(collection: dict[str, Any]) -> list[dict[str, Any]]: + """Return diagnostics that invalidate collection identity or syntax.""" + return [item for item in collection.get("diagnostics", []) if item.get("severity") == "error"] + + +def render_typescript(collection: dict[str, Any], *, import_path: str) -> str: + """Render a schema-1 or schema-2 collection as a TypeScript module.""" + helper = "defineMsdmdCollectionV2" if collection.get("schema_version") == SCHEMA_VERSION else "defineMsdmdCollection" + imports = helper + if helper == "defineMsdmdCollectionV2" and len(collection.get("facts", [])) > 64: + # Bound TypeScript contextual inference without casts or losing type checks. + imports += ", mergeMsdmdFactChunks" + parts = [] + for key, value in sorted(collection.items()): + if key == "facts": + chunks = [json.dumps(value[i:i + 64], indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + for i in range(0, len(value), 64)] + rendered = "mergeMsdmdFactChunks(\n" + ",\n".join(chunks) + "\n)" + else: + rendered = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + parts.append(" " + json.dumps(key) + ": " + rendered.replace("\n", "\n ")) + payload = "{\n" + ",\n".join(parts) + "\n}" + else: + payload = json.dumps(collection, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + return f'import {{ {imports} }} from "{import_path}";\n\nexport default {helper}({payload});\n' def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path("."), help="repo root to scan") + parser.add_argument("--root", type=Path, default=Path("."), help="repository root to scan") parser.add_argument("--repo", required=True, help="repository slug for the collection") parser.add_argument("--out", type=Path, help="output .ts path; stdout when omitted") - parser.add_argument( - "--block", - action="append", - dest="blocks", - help="block name to collect; may be repeated; defaults to all known blocks", - ) - parser.add_argument( - "--expected-block", - action="append", - default=[], - help="block expected on every source file for gap reporting; may be repeated", - ) - parser.add_argument( - "--import-path", - default="./.agents/skills/msdmd/collection", - help="TypeScript import path for defineMsdmdCollection", - ) - parser.add_argument("--source-commit", help="source commit SHA to record") + parser.add_argument("--block", action="append", dest="blocks", help="supplemental block name to collect; may be repeated") + parser.add_argument("--expected-block", action="append", default=[], help="block expected on every supported line-comment file for adoption reporting") + parser.add_argument("--import-path", default="./.agents/skills/msdmd/collection", help="TypeScript import path for the schema helper") + parser.add_argument("--source-commit", help="source commit SHA to record; current git HEAD is detected when omitted") + parser.add_argument("--max-file-bytes", type=int, default=DEFAULT_MAX_FILE_BYTES, help="per-file read boundary") + parser.add_argument("--legacy-blocks-only", action="store_true", help="emit explicit schema-1 block-only compatibility output") + parser.add_argument("--snapshot-identity", action="store_true", help="bind generated output to source bytes rather than a circular output commit") + parser.add_argument("--require-source", action="append", default=[], help="glob that must have matched, fully extracted inputs; repeat as needed") + parser.add_argument("--require-fact", action="append", default=[], help="required fileglob::convention::kind, repeated as needed") + parser.add_argument("--json", action="store_true", help="emit raw collection JSON instead of TypeScript") + parser.add_argument("--check", action="store_true", help="compare --out without writing; nonzero on drift") + parser.add_argument("--strict", action="store_true", help="exit nonzero after rendering if invalid identity or syntax diagnostics exist") args = parser.parse_args() - collection = collect( - args.root, - args.repo, - block_names=args.blocks or DEFAULT_BLOCK_NAMES, - expected_blocks=args.expected_block, - source_commit=args.source_commit, - ) - rendered = render_typescript(collection, import_path=args.import_path) - if args.out: - args.out.write_text(rendered, encoding="utf-8") + required_facts = [] + for value in args.require_fact: + parts = value.split("::") + if len(parts) != 3 or not all(parts): + parser.error("--require-fact must be fileglob::convention::kind") + required_facts.append(dict(zip(("file", "convention", "kind"), parts))) + if args.legacy_blocks_only and (args.require_source or required_facts or args.snapshot_identity): + parser.error("legacy output cannot represent native coverage policies or snapshot identity") + if args.legacy_blocks_only: + collection = collect_legacy( + args.root, + args.repo, + block_names=args.blocks or DEFAULT_BLOCK_NAMES, + expected_blocks=args.expected_block, + source_commit=args.source_commit, + ) + else: + collection = collect( + args.root, + args.repo, + block_names=args.blocks or DEFAULT_BLOCK_NAMES, + expected_blocks=args.expected_block, + source_commit=args.source_commit, + max_file_bytes=args.max_file_bytes, + snapshot_identity=args.snapshot_identity, + generated_outputs=[args.out.resolve().relative_to(args.root.resolve()).as_posix()] if args.out and args.out.resolve().is_relative_to(args.root.resolve()) else [], + required_sources=args.require_source, + required_facts=required_facts, + ) + rendered = json.dumps(collection, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" if args.json else render_typescript(collection, import_path=args.import_path) + if args.check: + if not args.out: + parser.error("--check requires --out") + if not args.out.exists() or args.out.read_bytes() != rendered.encode("utf-8"): + print("collection drift: " + str(args.out)) + return 1 + elif args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=".msdmd-", dir=args.out.parent) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(rendered.encode("utf-8")) + os.replace(temporary, args.out) + finally: + if os.path.exists(temporary): + os.unlink(temporary) else: print(rendered, end="") + if args.strict and not args.legacy_blocks_only and collection_errors(collection): + return 2 return 0 if __name__ == "__main__": raise SystemExit(main()) -# ratios: loc_comments=143:7 imports_exports=6:3 calls_definitions=35:6 +# ratios: loc_comments=706:54 imports_exports=17:5 calls_definitions=213:21 diff --git a/.agents/skills/msdmd/collection.ts b/.agents/skills/msdmd/collection.ts index 00195df4..9d13533b 100644 --- a/.agents/skills/msdmd/collection.ts +++ b/.agents/skills/msdmd/collection.ts @@ -1,12 +1,18 @@ // ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm /** - * Shared TypeScript shapes for repo-level msdmd collection points. + * Versioned TypeScript contracts for MSDMD collection points. * - * A consuming repo's `_msdmd.ts` file may import or copy these - * types, then export a `MsdmdCollection` generated from module-local msdmd - * blocks. This file is type-only: it does not parse source files or validate - * declarations. + * Usage guidance: + * - New native-capable generators call `defineMsdmdCollectionV2`. + * - Existing block-only files may keep `defineMsdmdCollection` while they + * explicitly remain schema 1. + * - Consumers must negotiate `schema_version`; schema 1 cannot represent + * native facts and must never receive a silent lossy projection. */ + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + export type MsdmdBlockName = | "DOCS" | "CAPABILITIES" @@ -22,54 +28,229 @@ export type MsdmdBlockName = export type MsdmdFieldMap = Record; -export interface MsdmdDeclaration { - /** Repository-relative source file that owns the declaration. */ +/** Schema-1 block declaration retained only for explicit legacy consumers. */ +export interface MsdmdDeclarationV1 { file: string; - /** msdmd application block name, such as CONTRACTS, CHECKS, or DOCS. */ block: MsdmdBlockName; - /** Stable entry id declared inside the block. */ id: string; - /** Flat parsed fields, excluding id unless a generator intentionally repeats it. */ fields: MsdmdFieldMap; } -export interface MsdmdGap { - /** Repository-relative source file with missing expected block coverage. */ +export interface MsdmdGapV1 { file: string; - /** Block types expected by local policy but absent from this file. */ - missing: MsdmdBlockName[]; - /** Optional explanation from the collector or policy layer. */ + missing: string[]; reason?: string; } -export interface MsdmdEdge { - /** Source declaration id or file path. */ +export interface MsdmdEdgeV1 { from: string; - /** Target declaration id, capability id, owner, route, file, or external system. */ to: string; - /** Relationship kind: requires, exposes, owns, covers, calls, claims_proves, risk, etc. */ kind: string; - /** Block that produced this edge. */ source_block: MsdmdBlockName; - /** Entry id that produced this edge. */ source_id: string; } -export interface MsdmdCollection { - /** Repository slug, for example a0 or skill-lib. */ +/** Block-only schema. It cannot represent native facts. */ +export interface MsdmdCollectionV1 { + schema_version?: "1.0.0"; repo: string; - /** Parsed module-local msdmd entries. */ - declarations: MsdmdDeclaration[]; - /** Visible coverage gaps emitted by collectors or local policy. */ - gaps: MsdmdGap[]; - /** Optional normalized relationship graph for visualizers. */ - edges?: MsdmdEdge[]; - /** Optional collector metadata. */ + declarations: MsdmdDeclarationV1[]; + gaps: MsdmdGapV1[]; + edges?: MsdmdEdgeV1[]; generated_at?: string; source_commit?: string; } -export function defineMsdmdCollection(collection: MsdmdCollection): MsdmdCollection { +/** Explicit names retained for schema-1 source compatibility; never used for native facts. */ +export type MsdmdDeclaration = MsdmdDeclarationV1; +export type MsdmdGap = MsdmdGapV1; +export type MsdmdEdge = MsdmdEdgeV1; +export type MsdmdCollection = MsdmdCollectionV1; + +export type MsdmdSourceLocation = + | { pointer: string } + | { start_line: number; end_line: number; span?: JsonValue; start_byte?: number; end_byte?: number }; + +export interface MsdmdSourceReference { + repository: string; + revision: string; + file: string; + content_sha256: string; + location: MsdmdSourceLocation; +} + +export interface MsdmdDeclarationV2 { + address: string; + origin: "msdmd-block"; + standing: "declared"; + source: MsdmdSourceReference; + file: string; + block: MsdmdBlockName; + id: string; + fields: MsdmdFieldMap; +} + +export interface MsdmdNativeFact { + address: string; + origin: "native"; + kind: string; + source: MsdmdSourceReference; + subject: { + address: string; + scope: string; + identity: string; + }; + convention: { + namespace: string; + version: string; + dialect: string; + }; + native: { + id: string | null; + value: JsonValue; + }; + standing: "declared" | "syntactically-observed" | "derived" | "reported-evidence"; + extraction: { + reader_id: string; + reader_version: string; + support: "implemented-and-tested" | "partial" | "specified-only" | "unsupported"; + configuration_sha256: string; + implementation_sha256?: string; + }; + projection?: { + mapping_version: string; + canonical_field?: string; + relation?: string; + loss: string; + }; +} + +export interface MsdmdReaderManifest { + reader_id: string; + version: string; + entry_point: string; + dependencies: string[]; + convention: string; + specification: string; + supported_versions: string[]; + feature_subset: string[]; + detection: string[]; + scope_attachment: string; + unknown_field_policy: string; + execution_policy: string; + support: "implemented-and-tested" | "partial" | "specified-only" | "unsupported"; + known_limitations?: string[]; +} + +export interface MsdmdReaderRun { + reader_id: string; + reader_version: string; + support: MsdmdReaderManifest["support"]; + status: "applied" | "not-applicable"; + files_matched: number; + facts_emitted: number; +} + +export interface MsdmdDiscoveryEntry { + file: string; + status: "supported" | "partial" | "unsupported" | "invalid" | "excluded" | "unreadable"; + reader_ids: string[]; + content_sha256: string; + entry_kind?: "file" | "subtree" | "output"; + size?: number; + reason?: string; +} + +export interface MsdmdEdgeV2 { + from: string; + to: string; + kind: string; + standing: "declared" | "syntactically-observed" | "derived" | "reported-evidence"; + target_resolution: "resolved-unique-entry-id" | "external-or-unresolved" | "ambiguous"; + source_block?: MsdmdBlockName; + source_id?: string; + source_entry_id?: string; +} + +export interface MsdmdDiagnostic { + code: string; + severity: "error" | "warning" | "info"; + status: string; + message: string; + source: JsonValue; + reader_id: string | null; + candidates?: string[]; +} + +export interface MsdmdConflict { + kind: string; + status: "unresolved" | "resolved"; + identity: string; + witnesses: MsdmdSourceReference[]; + reason: string; +} + +export interface MsdmdGapV2 { + file: string; + missing: MsdmdBlockName[]; + kind: "block-adoption"; +} + +export interface MsdmdCollectionV2 { + schema: "the-interdependency.msdmd-collection"; + schema_version: "2.0.0"; + capabilities: string[]; + repo: string; + source: { + repository: string; + revision: string; + git_head: string; + dirty_worktree: boolean | "hmmm"; + snapshot_sha256: string; + snapshot_complete: boolean; + revision_kind?: "content-snapshot" | "git-or-declared"; + exclusion_accounting?: "configured-patterns" | "observed-paths"; + scope?: string; + }; + requirements?: JsonValue[]; + reader_manifests: MsdmdReaderManifest[]; + reader_runs: MsdmdReaderRun[]; + discovery: MsdmdDiscoveryEntry[]; + declarations: MsdmdDeclarationV2[]; + facts: MsdmdNativeFact[]; + gaps: MsdmdGapV2[]; + edges: MsdmdEdgeV2[]; + conflicts: MsdmdConflict[]; + diagnostics: MsdmdDiagnostic[]; + coverage: { + denominator: { kind: "discovered-files"; count: number }; + excluded_subtrees?: number; + required_sources?: string[]; + supported_files: number; + partial_files: number; + unsupported_files: number; + invalid_files: number; + excluded_files: number; + unreadable_files: number; + native_facts: number; + supplemental_declarations: number; + block_adoption: Record; + information_availability: "not-evaluated-without-obligation-policy" | "policy-evaluated"; + verified_behavior: "not-evaluated"; + }; +} + +/** Explicit compatibility helper for block-only schema-1 collection points. */ +export function defineMsdmdCollection(collection: MsdmdCollectionV1): MsdmdCollectionV1 { return collection; } + +/** Native-capable schema-2 helper. Consumers must opt into this exact version. */ +export function defineMsdmdCollectionV2(collection: MsdmdCollectionV2): MsdmdCollectionV2 { + return collection; +} +/** Keep large generated literals type-checkable without any/unknown casts. */ +export function mergeMsdmdFactChunks(...chunks: MsdmdNativeFact[][]): MsdmdNativeFact[] { + return chunks.flat(); +} // ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm diff --git a/.agents/skills/msdmd/formats.py b/.agents/skills/msdmd/formats.py new file mode 100644 index 00000000..eed8a5b8 --- /dev/null +++ b/.agents/skills/msdmd/formats.py @@ -0,0 +1,189 @@ +# ratios: loc_comments=151:14 imports_exports=9:5 calls_definitions=78:11 +"""Bounded, non-executing parsers shared by the native metadata adapters. + +Usage: parse_json(bytes), parse_yaml(bytes), parse_xml(bytes). No includes, +constructors, network references or target code are executed. YAML uses PyYAML's +syntax parser with explicit YAML 1.2 core scalar resolution and typed scalar keys. +Repeated keys, cyclic aliases, custom tags, merge keys and excessive expansion +are errors, never silently flattened. XML supports UTF-8, without DTD/entities. +""" +from __future__ import annotations + +import json +from decimal import Decimal +import math +import re +import xml.etree.ElementTree as ET +from typing import Any +from urllib.parse import urlsplit + +def is_json_schema_uri(value: Any) -> bool: + """Recognize the exact JSON Schema URI authority without fetching it. + + Usage: pass a native $schema value, without coercing its type. This + recognizes the owning namespace, not a supported draft or trusted URL. + Userinfo, explicit ports, lookalike hosts and control characters are + outside this recognition subset and remain unclassified source data. + """ + if not isinstance(value, str) or any(ord(char) <= 32 or ord(char) == 127 for char in value): + return False + try: + uri = urlsplit(value) + except ValueError: + return False + return uri.scheme in {'http', 'https'} and uri.netloc.lower() == 'json-schema.org' + + +MAX_NODES = 100_000 +MAX_DEPTH = 128 + + +def check_tree(value: Any, depth: int = 0, budget: list[int] | None = None) -> Any: + budget = [MAX_NODES] if budget is None else budget + budget[0] -= 1 + if depth > MAX_DEPTH or budget[0] < 0: + raise ValueError('structured input exceeds node/depth limit') + if isinstance(value, dict): + for item in value.values(): + check_tree(item, depth + 1, budget) + elif isinstance(value, list): + for item in value: + check_tree(item, depth + 1, budget) + return value + + +def parse_json(data: bytes) -> Any: + def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise ValueError('duplicate JSON object key') + result[key] = value + return result + + def constant(value: str) -> Any: + raise ValueError('non-finite JSON number') + + def integer(value: str) -> Any: + result = int(value) + if abs(result) > 2 ** 53 - 1 or value == '-0': + return {'$type': 'json-number', 'lexeme': value} + return result + + def number(value: str) -> Any: + result = float(value) + if not math.isfinite(result): + raise ValueError('JSON number exceeds finite numeric representation') + original = Decimal(value) + if original != Decimal.from_float(result) or (result == 0 and original.is_signed()): + return {'$type': 'json-number', 'lexeme': value} + return result + + return check_tree(json.loads(data, object_pairs_hook=pairs, parse_constant=constant, parse_int=integer, parse_float=number)) + + +def parse_yaml(data: bytes) -> Any: + import yaml # Optional reader dependency; missing installation is diagnostic. + + class CoreLoader(yaml.BaseLoader): + yaml_implicit_resolvers: dict = {} + CoreLoader.add_implicit_resolver('tag:yaml.org,2002:null', re.compile(r'^(?:~|null|Null|NULL|)$'), ['~', 'n', 'N', '']) + CoreLoader.add_implicit_resolver('tag:yaml.org,2002:bool', re.compile(r'^(?:true|True|TRUE|false|False|FALSE)$'), list('tTfF')) + CoreLoader.add_implicit_resolver('tag:yaml.org,2002:int', re.compile(r'^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$'), list('-+0123456789')) + CoreLoader.add_implicit_resolver('tag:yaml.org,2002:float', re.compile(r'^[-+]?(?:(?:[0-9]+\.[0-9]*|\.[0-9]+|[0-9]+[eE][-+]?[0-9]+)(?:[eE][-+]?[0-9]+)?|\.(?:inf|Inf|INF|nan|NaN|NAN))$'), list('-+.0123456789')) + node = yaml.compose(data.decode('utf-8-sig'), Loader=CoreLoader) + active: set[int] = set() + budget = [MAX_NODES] + + def scalar(text: str, tag: str) -> Any: + if tag not in {'tag:yaml.org,2002:str', 'tag:yaml.org,2002:bool', + 'tag:yaml.org,2002:null', 'tag:yaml.org,2002:int', + 'tag:yaml.org,2002:float'}: + raise ValueError('unsupported YAML tag') + if tag == 'tag:yaml.org,2002:str': + return text + if tag.endswith(':null') and text in {'', '~', 'null', 'Null', 'NULL'}: + return None + if tag.endswith(':bool') and text in {'true', 'True', 'TRUE', 'false', 'False', 'FALSE'}: + return text.lower() == 'true' + if tag.endswith(':int'): + if re.fullmatch(r'[-+]?[0-9]+', text): + return int(text, 10) + if re.fullmatch(r'0o[0-7]+', text): + return int(text[2:], 8) + if re.fullmatch(r'0x[0-9a-fA-F]+', text): + return int(text[2:], 16) + if tag.endswith(':float'): + if text.lower().lstrip('+-') in {'.inf', '.nan'}: + return {'$type': 'yaml-nonfinite', 'value': text} + if re.fullmatch(r'[-+]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][-+]?[0-9]+)?', text): + value = float(text) + if not math.isfinite(value): + raise ValueError('YAML number exceeds finite numeric representation') + return value + raise ValueError('invalid explicitly tagged YAML scalar') + + def visit(item: Any, depth: int = 0) -> Any: + if item is None: + return None + budget[0] -= 1 + if depth > MAX_DEPTH or budget[0] < 0: + raise ValueError('YAML alias expansion exceeds node/depth limit') + if id(item) in active: + raise ValueError('cyclic YAML alias') + active.add(id(item)) + try: + if isinstance(item, yaml.ScalarNode): + if item.tag == 'tag:yaml.org,2002:str' and item.style is not None: + return item.value + return scalar(item.value, item.tag) + if isinstance(item, yaml.SequenceNode): + if item.tag != 'tag:yaml.org,2002:seq': + raise ValueError('unsupported YAML sequence tag') + return [visit(child, depth + 1) for child in item.value] + if isinstance(item, yaml.MappingNode): + if item.tag != 'tag:yaml.org,2002:map': + raise ValueError('unsupported YAML mapping tag') + entries: list[dict] = [] + seen: set[str] = set() + string_keys = True + for key, value in item.value: + if not isinstance(key, yaml.ScalarNode): + raise ValueError('non-scalar YAML key is outside metadata mapping subset') + name = visit(key, depth + 1) + if name == '<<': + raise ValueError('YAML merge keys require a declared merge dialect') + signature = key.tag + ':' + json.dumps(name, sort_keys=True) + if signature in seen: + raise ValueError('duplicate YAML mapping key') + seen.add(signature) + string_keys = string_keys and isinstance(name, str) + entries.append({'key': name, 'tag': key.tag, 'value': visit(value, depth + 1)}) + if string_keys: + return {entry['key']: entry['value'] for entry in entries} + # JSON object keys cannot encode typed YAML keys without loss. + return {'$type': 'yaml-mapping', 'entries': entries} + raise ValueError('unknown YAML node') + finally: + active.remove(id(item)) + + return visit(node) + + +def parse_xml(data: bytes) -> ET.Element: + text = data.decode('utf-8-sig') + if '\x00' in text or re.search(r' MAX_DEPTH or count > MAX_NODES: + raise ValueError('XML exceeds node/depth limit') + stack.extend((child, depth + 1) for child in node) + return root +# ratios: loc_comments=151:14 imports_exports=9:5 calls_definitions=78:11 diff --git a/.agents/skills/msdmd/grammar_code.py b/.agents/skills/msdmd/grammar_code.py new file mode 100644 index 00000000..b2ba5be2 --- /dev/null +++ b/.agents/skills/msdmd/grammar_code.py @@ -0,0 +1,136 @@ +# ratios: loc_comments=116:9 imports_exports=8:1 calls_definitions=41:4 +"""Source metadata from pinned Rust, Java, C and C++ grammars. + +Usage: read_grammar(path, supplied_bytes, context) through msdmd.collect. +No build, macro expansion, annotation processing or target import occurs. .h +requires explicit language selection and is reported ambiguous. Documentation +retains raw comments and repeated tags; it does not run rustdoc or Doxygen. +""" +from __future__ import annotations + +import hashlib +import importlib +import re +from pathlib import Path +from typing import Any + +GRAMMARS = {'.rs': ('rust', '0.24.2'), '.java': ('java', '0.23.5'), + '.c': ('c', '0.24.2'), '.cpp': ('cpp', '0.23.4'), + '.cc': ('cpp', '0.23.4'), '.cxx': ('cpp', '0.23.4'), + '.hpp': ('cpp', '0.23.4'), '.hxx': ('cpp', '0.23.4')} +DECLARATIONS = {'function_item', 'function_signature_item', 'struct_item', + 'enum_item', 'trait_item', 'type_item', 'mod_item', 'const_item', 'static_item', + 'class_declaration', 'interface_declaration', 'enum_declaration', 'record_declaration', + 'method_declaration', 'constructor_declaration', 'annotation_type_declaration', + 'function_definition', 'class_specifier', 'struct_specifier', 'enum_specifier', + 'namespace_definition', 'declaration'} + + +def read_grammar(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + from tree_sitter import Language, Parser + from msdmd.readers import _fact, _diagnostic + rid = 'native-grammars' + if path.suffix == '.h': + return [], [], [_diagnostic(context, reader_id=rid, code='ambiguous_header_language', + message='.h can be C or C++; supply an unambiguous .c/.hpp source representation', status='ambiguous')] + language, version = GRAMMARS[path.suffix.lower()] + # Only hard-coded installed grammar modules are loaded, never target modules. + grammar = importlib.import_module('tree_sitter_' + language) + tree = Parser(Language(grammar.language())).parse(data) + namespace = {'rust': 'rustdoc', 'java': 'javadoc', 'c': 'doxygen', 'cpp': 'doxygen'}[language] + context = dict(context, convention_version=f'{language}-grammar@{version}', dialect=language) + facts: list = [] + diagnostics: list = [] + nodes: list = [] + stack = [tree.root_node] + while stack: + node = stack.pop() + nodes.append(node) + stack.extend(reversed(node.named_children)) + if tree.root_node.has_error: + diagnostics.append(_diagnostic(context, reader_id=rid, code='source_grammar_error', + message='source contains grammar errors; recovered facts are partial', status='invalid', severity='error')) + declarations: list[tuple[Any, str]] = [] + counts: dict[str, int] = {} + + def text(node: Any) -> str: + return data[node.start_byte:node.end_byte].decode('utf-8') + + def name_of(node: Any) -> str | None: + name = node.child_by_field_name('name') + declarator = node.child_by_field_name('declarator') + while name is None and declarator is not None: + if declarator.type in {'identifier', 'field_identifier', 'qualified_identifier'}: + name = declarator + break + declarator = declarator.child_by_field_name('declarator') + return text(name) if name is not None else None + + def emit(node: Any, kind: str, identity: str, value: Any, *, scope: str = 'symbol', convention: str | None = None) -> dict: + key = kind + ':' + identity + ordinal = counts.get(key, 0) + counts[key] = ordinal + 1 + fact = _fact(context, reader_id=rid, kind=kind, scope=scope, identity=identity, + convention=convention or language + '.source-metadata', + location={'start_line': node.start_point.row + 1, 'end_line': node.end_point.row + 1, + 'start_byte': node.start_byte, 'end_byte': node.end_byte}, value=value) + fact['address'] = fact['subject']['address'] + '/fact/' + kind + '/' + str(ordinal) + facts.append(fact) + return fact + + for node in nodes: + if node.type in DECLARATIONS and (name := name_of(node)): + parents = [(parent, identity) for parent, identity in declarations + if parent.start_byte <= node.start_byte and parent.end_byte >= node.end_byte] + parent = parents[-1][1] if parents else '' + # Rust impl blocks have no named declaration; retain their type context. + ancestor = node.parent + impl = None + while ancestor is not None and ancestor.type not in DECLARATIONS: + if ancestor.type == 'impl_item': + impl = ancestor.child_by_field_name('type') + break + ancestor = ancestor.parent + if impl is not None: + parent = '.'.join(filter(None, (parent, text(impl)))) + identity = '.'.join(filter(None, (parent, name))) + body = node.child_by_field_name('body') + signature = data[node.start_byte:body.start_byte if body else node.end_byte].decode('utf-8').strip() + declarations.append((node, identity)) + emit(node, 'symbol-declaration', identity, {'name': name, 'qualified_name': identity, + 'syntax_kind': node.type, 'signature': signature, 'condition_resolution': 'not-evaluated'}) + if node.type in {'use_declaration', 'import_declaration', 'package_declaration', 'preproc_include', 'attribute_item', 'inner_attribute_item'}: + emit(node, 'source-declaration', context['file'], {'syntax_kind': node.type, 'text': text(node)}, scope='module') + if node.type in {'macro_invocation', 'preproc_if', 'preproc_ifdef'}: + diagnostics.append(_diagnostic(context, reader_id=rid, code='unresolved_compile_context', + message='macro/preprocessor context is preserved but not evaluated', status='dynamic-unresolved')) + for node in nodes: + if node.type not in {'line_comment', 'block_comment', 'comment'}: + continue + raw = text(node) + enclosing = [(owner, ident) for owner, ident in declarations + if owner.start_byte <= node.start_byte and owner.end_byte >= node.end_byte] + identity = enclosing[-1][1] if enclosing else context['file'] + scope = 'symbol' if enclosing else 'module' + doc = raw.startswith(('/**', '/*!', '///', '//!')) + attachment = 'enclosing' if enclosing else 'module' + if doc and not raw.startswith(('//!', '/*!')): + following = [(owner, ident) for owner, ident in declarations if owner.start_byte >= node.end_byte] + if following: + owner, candidate = following[0] + between = data[node.end_byte:owner.start_byte].decode('utf-8') + # Only comments/attributes can bridge leading documentation. + scrubbed = re.sub(r'/\*.*?\*/|//[^\n]*|#\[[^\]]*\]', '', between, flags=re.S) + if not scrubbed.strip(): + identity, scope, attachment = candidate, 'symbol', 'leading-documentation' + emit(node, 'comment', identity, {'text': raw, 'attachment': attachment}, scope=scope) + if doc: + tags = [{'name': match[1], 'text': match[2].strip()} + for match in re.finditer(r'(?:^|\n)\s*(?:\*|///|//!)?\s*[@\\]([A-Za-z]+)\s*([^\n]*)', raw.removeprefix('/**').removesuffix('*/'))] + emit(node, 'documentation-comment', identity, {'text': raw, 'tags': tags, + 'attachment': attachment, 'expansion': 'not-performed'}, scope=scope, convention=namespace) + for match in re.finditer(r'SPDX-(License-Identifier|FileCopyrightText):\s*([^\n*]+)', raw): + emit(node, 'license-declaration', context['file'], {'tag': 'SPDX-' + match[1], 'value': match[2].strip()}, + scope='module', convention='spdx.source-header') + return facts, [], diagnostics +# ratios: loc_comments=116:9 imports_exports=8:1 calls_definitions=41:4 diff --git a/.agents/skills/msdmd/module-projection.schema.json b/.agents/skills/msdmd/module-projection.schema.json new file mode 100644 index 00000000..86843f7d --- /dev/null +++ b/.agents/skills/msdmd/module-projection.schema.json @@ -0,0 +1,337 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:the-interdependency:msdmd:module-projection:1", + "title": "MSDMD per-module projection record", + "description": "Schema for each line in a deterministic MSDMD Python module JSONL projection. The first line must be a module record; following lines are symbol, metadata, or diagnostic records.", + "oneOf": [ + { + "$ref": "#/$defs/module" + }, + { + "$ref": "#/$defs/symbol" + }, + { + "$ref": "#/$defs/metadata" + }, + { + "$ref": "#/$defs/diagnostic" + } + ], + "$defs": { + "position": { + "type": "object", + "additionalProperties": false, + "required": [ + "line", + "column", + "byte" + ], + "properties": { + "line": { + "type": "integer", + "minimum": 1 + }, + "column": { + "type": "integer", + "minimum": 0, + "description": "Zero-based Unicode character offset within the line." + }, + "byte": { + "type": "integer", + "minimum": 0, + "description": "Zero-based UTF-8 byte offset in the decoded source-text projection; source_sha256 identifies the authoritative raw bytes." + } + } + }, + "span": { + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "$ref": "#/$defs/position" + }, + "end": { + "$ref": "#/$defs/position" + } + } + }, + "module": { + "type": "object", + "additionalProperties": false, + "required": [ + "record_type", + "schema", + "schema_version", + "schema_sha256", + "repo", + "source_revision", + "source_path", + "source_sha256", + "source_encoding", + "reader", + "reader_version", + "reader_sha256", + "reader_manifest_sha256", + "python_implementation", + "python_version", + "ast_feature_version", + "freshness_key_sha256", + "module_id", + "status", + "hmmm" + ], + "properties": { + "record_type": { + "const": "module" + }, + "schema": { + "const": "urn:the-interdependency:msdmd:module-projection:1" + }, + "schema_version": { + "const": "1.0.0" + }, + "schema_sha256": { + "$ref": "#/$defs/sha256" + }, + "repo": { + "type": "string", + "minLength": 1 + }, + "source_revision": { + "type": "string", + "minLength": 1 + }, + "source_path": { + "type": "string", + "minLength": 1 + }, + "source_sha256": { + "$ref": "#/$defs/sha256" + }, + "source_encoding": { + "type": "string", + "minLength": 1 + }, + "reader": { + "const": "the-interdependency.msdmd.python-ast-tokenize" + }, + "reader_version": { + "const": "1.0.0" + }, + "reader_sha256": { + "$ref": "#/$defs/sha256" + }, + "reader_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "python_implementation": { + "type": "string", + "minLength": 1 + }, + "python_version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "ast_feature_version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+$" + }, + "freshness_key_sha256": { + "$ref": "#/$defs/sha256" + }, + "module_id": { + "type": "string", + "minLength": 1 + }, + "status": { + "enum": [ + "complete", + "invalid" + ] + }, + "hmmm": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "symbol": { + "type": "object", + "additionalProperties": false, + "required": [ + "record_type", + "id", + "kind", + "qualified_name", + "parent", + "signature", + "signature_sha256", + "decorators", + "standing", + "source_span" + ], + "properties": { + "record_type": { + "const": "symbol" + }, + "id": { + "type": "string", + "minLength": 1 + }, + "kind": { + "enum": [ + "class", + "function", + "async_function", + "method", + "async_method" + ] + }, + "qualified_name": { + "type": "string", + "minLength": 1 + }, + "parent": { + "type": [ + "string", + "null" + ] + }, + "signature": { + "type": "string", + "minLength": 1 + }, + "signature_sha256": { + "$ref": "#/$defs/sha256" + }, + "decorators": { + "type": "array", + "items": { + "type": "string" + } + }, + "standing": { + "const": "syntactically_observed" + }, + "source_span": { + "$ref": "#/$defs/span" + } + } + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "required": [ + "record_type", + "metadata_kind", + "convention", + "subject", + "subject_kind", + "attachment", + "standing", + "text", + "source_span" + ], + "properties": { + "record_type": { + "const": "metadata" + }, + "metadata_kind": { + "type": "string", + "minLength": 1 + }, + "convention": { + "type": "string", + "minLength": 1 + }, + "subject": { + "type": "string", + "minLength": 1 + }, + "subject_kind": { + "type": "string", + "minLength": 1 + }, + "attachment": { + "enum": [ + "native_docstring", + "leading_trivia", + "nearest_enclosing_symbol", + "module_convention", + "module_scope" + ] + }, + "standing": { + "enum": [ + "declared", + "syntactically_observed" + ] + }, + "text": { + "type": "string" + }, + "source_span": { + "$ref": "#/$defs/span" + } + } + }, + "diagnostic": { + "type": "object", + "required": [ + "record_type", + "code", + "severity", + "message", + "source_path" + ], + "properties": { + "record_type": { + "const": "diagnostic" + }, + "code": { + "type": "string", + "minLength": 1 + }, + "severity": { + "enum": [ + "error", + "warning", + "hmmm" + ] + }, + "message": { + "type": "string" + }, + "source_path": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "column": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + }, + "additionalProperties": false + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/.agents/skills/msdmd/module_projection.py b/.agents/skills/msdmd/module_projection.py new file mode 100644 index 00000000..7c848911 --- /dev/null +++ b/.agents/skills/msdmd/module_projection.py @@ -0,0 +1,1180 @@ +# ratios: loc_comments=910:128 imports_exports=14:10 calls_definitions=259:54 +# === MODULE_BUILD === +# id: msdmd_python_module_projection +# module_name: module_projection +# module_kind: instrument +# summary: derives deterministic per-module JSONL metadata with structural symbol attachment from Python source without executing it +# owner: The Interdependency skill-lib +# public_surface: project_python_module, render_jsonl, project_tree, write_projections, check_projections +# internal_surface: Python symbol discovery, comment grouping, structural attachment, freshness computation +# auth_boundary: none +# storage_boundary: write +# network_boundary: none +# user_data_boundary: read +# admin_only: false +# tests: tests/test_module_projection.py +# rollout: explicit python -m msdmd.module_projection --write invocation +# rollback: remove the runner and generated projection directories; native sources remain authoritative +# === END MODULE_BUILD === +# === CAPABILITIES === +# id: python_module_metadata_projection +# exposes: deterministic JSONL projection, Python symbol identities, native docstrings, structurally attached comments, freshness key +# summary: creates one generated metadata sidecar per Python module while keeping source locations navigational rather than identity-bearing +# limitations: Python source only; call graphs, runtime behavior, cross-revision rename identity, and non-Python readers remain hmmm +# === END CAPABILITIES === +# === CONTRACTS === +# id: module_projection_line_shift_stability +# given: blank or comment lines are inserted without changing declaration structure +# then: metadata remains attached to the same qualified symbol identity after regeneration +# class: identity +# +# id: module_projection_never_executes_source +# given: inspected Python source contains executable top-level code +# then: projection parses bytes without importing or executing the inspected module +# class: safety +# +# id: module_projection_freshness_binds_source_and_reader +# given: source bytes, schema, reader implementation, support manifest, or effective Python AST grammar changes +# then: the deterministic projection freshness key changes +# class: provenance +# === END CONTRACTS === +"""Generate deterministic, structurally attached Python module metadata. + +This runner is the first executable native-reader slice of MSDMD. Native +source remains authoritative; generated JSONL files are disposable projections. +Each projection binds the source digest, reader digest, schema version, effective +Python AST grammar, symbols, docstrings, comments, source spans, and attachment +method. + +Usage:: + + python -m msdmd.module_projection \ + --root . --repo The-Interdependency/example \ + --out-dir .msdmd/modules --write + + python -m msdmd.module_projection \ + --root . --repo The-Interdependency/example \ + --out-dir .msdmd/modules --check + +Use ``--source path/to/module.py`` one or more times to limit the selected +modules. A complete-tree ``--write`` replaces declared projection files and +removes stale JSONL projections below ``--out-dir``; a selected write never +prunes files outside its selection. ``--check`` performs no writes and fails +when a selected projection is missing, stale, or invalid. Complete-tree checks +also reject unexpected projections. No inspected module is imported or +executed. + +Attachment rules: + +* a comment group immediately preceding a declaration at the same lexical + depth is leading trivia for that declaration; +* otherwise a comment inside a declaration, including comments before or + between its decorators and trailing indented suite comments before lexical + dedent, belongs to the nearest enclosing declaration; +* headers, MSDMD fences, RATIOS, encoding cookies, and unattached comments stay + at module scope; +* native module/class/function docstrings attach directly to their AST owner. + +Interrupted, unclosed, or name-mismatched MSDMD fences, opening fences inside +an open block, and closing fences without an opening produce diagnostics instead +of spanning intervening code or being accepted as a block. A mismatch +(wrong-name close or nested opening) yields one diagnostic: one later close per +affected block name, orphaned by that already-reported mismatch, is suppressed +until the next opening of that name. An interrupted fence may also report its +later closing fence as unmatched. This is stricter than the +universal block parser, which matches each requested block name independently. + +Source lines are split only at Python newlines (LF, CRLF, and CR); U+2028, NEL, +form feed, and other Unicode separators remain inside their physical line. +Line and byte ranges are navigational facts for the pinned source digest. They +never form a symbol identity, so inserting lines cannot change ownership. Byte +offsets index the UTF-8 re-encoding of the decoded source text, so they differ +from raw file offsets for BOM-prefixed or non-UTF-8 sources. +Projection files are written as exact UTF-8 bytes with LF record terminators on +every platform. +""" +from __future__ import annotations + +import argparse +import ast +import codecs +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import sys +import tempfile +import tokenize +from typing import Iterable, Iterator, Sequence + + +SCHEMA_ID = "urn:the-interdependency:msdmd:module-projection:1" +SCHEMA_VERSION = "1.0.0" +READER_ID = "the-interdependency.msdmd.python-ast-tokenize" +READER_VERSION = "1.0.0" +PROJECTION_SUFFIX = ".msdmd.jsonl" + +DEFAULT_SKIP_DIRS = frozenset( + { + ".git", + ".agents", + ".venv", + "__pycache__", + "build", + "dist", + "node_modules", + "target", + "venv", + } +) + +_COOKIE_RE = re.compile(br"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") +_BLANK_OR_COMMENT_RE = re.compile(br"^[ \t\f]*(?:[#\r\n]|$)") +_FENCE_RE = re.compile(r"^===\s+(END\s+)?([A-Z][A-Z0-9_]*)\s+===$") +# Python recognizes only LF, CRLF, and CR as source newlines. str.splitlines() +# additionally splits on U+2028, U+2029, NEL, form feed, and other separators, +# which would desynchronize line indexes from AST and tokenize positions. +_PYTHON_NEWLINE_RE = re.compile(r"\r\n|\r|\n") +_PYTHON_NEWLINE_BYTES_RE = re.compile(br"\r\n|\r|\n") + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _canonical_json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _reader_sha256() -> str: + return _sha256(Path(__file__).read_bytes()) + + +def _schema_sha256() -> str: + return _sha256(Path(__file__).with_name("module-projection.schema.json").read_bytes()) + + +def _reader_manifest_sha256() -> str: + return _sha256(Path(__file__).with_name("python-module-reader.json").read_bytes()) + + +def _runtime_identity() -> dict[str, str]: + return { + "python_implementation": sys.implementation.name, + "python_version": ".".join(str(value) for value in sys.version_info[:3]), + "ast_feature_version": f"{sys.version_info.major}.{sys.version_info.minor}", + } + + +def _python_source_lines(text: str) -> list[str]: + """Split decoded source at Python newlines only, keeping line endings.""" + + lines: list[str] = [] + start = 0 + for match in _PYTHON_NEWLINE_RE.finditer(text): + lines.append(text[start:match.end()]) + start = match.end() + if start < len(text): + lines.append(text[start:]) + return lines + + +def _python_source_byte_lines(source: bytes) -> list[bytes]: + """Split raw source bytes at Python newlines only, keeping line endings.""" + + lines: list[bytes] = [] + start = 0 + for match in _PYTHON_NEWLINE_BYTES_RE.finditer(source): + lines.append(source[start:match.end()]) + start = match.end() + if start < len(source): + lines.append(source[start:]) + return lines + + +def _byte_line_reader(source: bytes): + """Return a readline callable that yields Python-newline-delimited lines.""" + + iterator = iter(_python_source_byte_lines(source)) + + def readline() -> bytes: + return next(iterator, b"") + + return readline + + +def _line_start_bytes(lines: Sequence[str]) -> list[int]: + offsets = [0] + total = 0 + for line in lines: + total += len(line.encode("utf-8")) + offsets.append(total) + return offsets + + +def _byte_offset(lines: Sequence[str], starts: Sequence[int], line: int, column: int) -> int: + if line < 1: + return 0 + line_index = min(line - 1, max(len(lines) - 1, 0)) + prefix = lines[line_index][:column] if lines else "" + return starts[line_index] + len(prefix.encode("utf-8")) + + +def _span( + lines: Sequence[str], + starts: Sequence[int], + start_line: int, + start_column: int, + end_line: int, + end_column: int, +) -> dict: + return { + "start": { + "line": start_line, + "column": start_column, + "byte": _byte_offset(lines, starts, start_line, start_column), + }, + "end": { + "line": end_line, + "column": end_column, + "byte": _byte_offset(lines, starts, end_line, end_column), + }, + } + + +def _node_span(node: ast.AST, lines: Sequence[str], starts: Sequence[int]) -> dict: + def character_column(line: int, utf8_byte_column: int) -> int: + if not lines: + return 0 + line_index = min(max(line - 1, 0), len(lines) - 1) + prefix = lines[line_index].encode("utf-8")[:utf8_byte_column] + return len(prefix.decode("utf-8")) + + start_line = int(getattr(node, "lineno", 1)) + end_line = int(getattr(node, "end_lineno", start_line)) + return _span( + lines, + starts, + start_line, + character_column(start_line, int(getattr(node, "col_offset", 0))), + end_line, + character_column(end_line, int(getattr(node, "end_col_offset", 0))), + ) + + +def _module_address(repo: str, source_path: str) -> str: + return f"python:{repo}:{source_path}::" + + +@dataclass(frozen=True) +class Symbol: + node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef + qualified_name: str + kind: str + depth: int + header_line: int + identity: str + signature: str + signature_sha256: str + parent_identity: str | None + + @property + def start_line(self) -> int: + return int(self.node.lineno) + + @property + def end_line(self) -> int: + return int(self.node.end_lineno or self.node.lineno) + + @property + def column(self) -> int: + return int(self.node.col_offset) + + +@dataclass(frozen=True) +class CommentGroup: + tokens: tuple[tokenize.TokenInfo, ...] + + @property + def start_line(self) -> int: + return self.tokens[0].start[0] + + @property + def start_column(self) -> int: + return self.tokens[0].start[1] + + @property + def end_line(self) -> int: + return self.tokens[-1].end[0] + + @property + def end_column(self) -> int: + return self.tokens[-1].end[1] + + @property + def text_lines(self) -> tuple[str, ...]: + values = [] + for token in self.tokens: + text = token.string[1:] + if text.startswith(" "): + text = text[1:] + values.append(text) + return tuple(values) + + +@dataclass(frozen=True) +class CommentDiagnostic: + code: str + message: str + line: int | None = None + + +def _signature(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) -> tuple[str, str]: + type_params = [ast.unparse(item) for item in getattr(node, "type_params", [])] + type_suffix = f"[{', '.join(type_params)}]" if type_params else "" + if isinstance(node, ast.ClassDef): + arguments = [ast.unparse(base) for base in node.bases] + arguments.extend(ast.unparse(keyword) for keyword in node.keywords) + text = ( + f"class {node.name}{type_suffix}({', '.join(arguments)})" + if arguments + else f"class {node.name}{type_suffix}" + ) + identity_tree = { + "kind": "class", + "bases": [ast.dump(base, include_attributes=False) for base in node.bases], + "keywords": [ast.dump(item, include_attributes=False) for item in node.keywords], + "type_params": [ + ast.dump(item, include_attributes=False) + for item in getattr(node, "type_params", []) + ], + } + else: + prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def" + arguments = ast.unparse(node.args) + returns = f" -> {ast.unparse(node.returns)}" if node.returns is not None else "" + text = f"{prefix} {node.name}{type_suffix}({arguments}){returns}" + identity_tree = { + "kind": "async_function" if isinstance(node, ast.AsyncFunctionDef) else "function", + "arguments": ast.dump(node.args, include_attributes=False), + "returns": ast.dump(node.returns, include_attributes=False) if node.returns else None, + "type_params": [ + ast.dump(item, include_attributes=False) + for item in getattr(node, "type_params", []) + ], + } + return text, _sha256(_canonical_json(identity_tree).encode("utf-8")) + + +def _kind(node: ast.AST, parent_kind: str | None) -> str: + if isinstance(node, ast.ClassDef): + return "class" + if parent_kind == "class": + return "async_method" if isinstance(node, ast.AsyncFunctionDef) else "method" + return "async_function" if isinstance(node, ast.AsyncFunctionDef) else "function" + + +def _header_line(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) -> int: + decorators = [int(item.lineno) for item in node.decorator_list] + return min([int(node.lineno), *decorators]) + + +def _discover_symbols(tree: ast.Module, repo: str, source_path: str) -> list[Symbol]: + discovered: list[tuple[ast.AST, str, str, int, int, str, str, str | None]] = [] + + def visit_node( + node: ast.AST, + names: tuple[str, ...], + parent_kind: str | None, + parent_key: str | None, + depth: int, + ) -> None: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + qualified = ".".join((*names, node.name)) + kind = _kind(node, parent_kind) + signature, signature_digest = _signature(node) + base = f"python:{repo}:{source_path}::{kind}:{qualified}" + discovered.append( + (node, qualified, kind, depth, _header_line(node), signature, signature_digest, parent_key) + ) + for child in node.body: + visit_node(child, (*names, node.name), kind, base, depth + 1) + return + + # Control-flow nodes and their non-statement suite containers do not + # create lexical symbol scopes. Recurse through match_case, + # ExceptHandler, and equivalent AST nodes while stopping at each + # discovered declaration boundary above. + for child in ast.iter_child_nodes(node): + visit_node(child, names, parent_kind, parent_key, depth) + + for statement in tree.body: + visit_node(statement, (), None, None, 0) + + counts: dict[str, int] = {} + for _, qualified, kind, _, _, _, _, _ in discovered: + base = f"python:{repo}:{source_path}::{kind}:{qualified}" + counts[base] = counts.get(base, 0) + 1 + + occurrence: dict[tuple[str, str], int] = {} + identities_by_base: dict[str, list[str]] = {} + symbols: list[Symbol] = [] + for node, qualified, kind, depth, header, signature, signature_digest, parent_base in discovered: + base = f"python:{repo}:{source_path}::{kind}:{qualified}" + if counts[base] == 1: + identity = base + else: + duplicate_key = (base, signature_digest) + ordinal = occurrence.get(duplicate_key, 0) + 1 + occurrence[duplicate_key] = ordinal + suffix = signature_digest[:12] + identity = f"{base}~{suffix}" if ordinal == 1 else f"{base}~{suffix}.{ordinal}" + identities_by_base.setdefault(base, []).append(identity) + parent_identity = None + if parent_base is not None: + candidates = identities_by_base.get(parent_base, []) + parent_identity = candidates[-1] if candidates else parent_base + symbols.append( + Symbol( + node=node, + qualified_name=qualified, + kind=kind, + depth=depth, + header_line=header, + identity=identity, + signature=signature, + signature_sha256=signature_digest, + parent_identity=parent_identity, + ) + ) + return symbols + + +def _encoding_cookie_line(source: bytes) -> int | None: + reader = _byte_line_reader(source) + first = reader() + if first.startswith(codecs.BOM_UTF8): + first = first[len(codecs.BOM_UTF8):] + if _COOKIE_RE.match(first): + return 1 + if not _BLANK_OR_COMMENT_RE.match(first): + return None + return 2 if _COOKIE_RE.match(reader()) else None + + +def _comment_groups( + lines: Sequence[str], + encoding_cookie_line: int | None, +) -> tuple[list[CommentGroup], list[CommentDiagnostic]]: + comments: list[tokenize.TokenInfo] = [] + diagnostics: list[CommentDiagnostic] = [] + # Tokenize the decoded Python-newline line map with each terminator + # normalized to LF. A raw byte readline would deliver a CR-only file as one + # line, desynchronizing comment positions from AST line numbers. Line + # numbers and in-line columns are unchanged by terminator normalization. + normalized = iter(_PYTHON_NEWLINE_RE.sub("\n", line) for line in lines) + try: + for token in tokenize.generate_tokens(lambda: next(normalized, "")): + if token.type == tokenize.COMMENT: + comments.append(token) + except (SyntaxError, tokenize.TokenError) as exc: + diagnostics.append( + CommentDiagnostic( + code="python_tokenize_error", + message=str(exc), + ) + ) + + groups: list[list[tokenize.TokenInfo]] = [] + current: list[tokenize.TokenInfo] = [] + in_msdmd_fence = False + open_fence_name: str | None = None + # Closing fences left orphaned by an already-diagnosed mismatch are + # follow-on noise: suppress one close per affected block name, until the + # next opening of that name, so a mismatch yields one diagnostic. + # Interrupted fences are not covered. The projection remains invalid. + suppressed_closes: dict[str, int] = {} + + def flush() -> None: + nonlocal current + if current: + groups.append(current) + current = [] + + for token in comments: + payload = token.string[1:].lstrip() + fence = _FENCE_RE.match(payload) + is_fence = fence is not None + is_end_fence = bool(fence and fence.group(1)) + fence_name = fence.group(2) if fence else None + is_singleton_convention = ( + payload.startswith("ratios:") + or (token.start[0] == 1 and token.string.startswith("#!")) + or token.start[0] == encoding_cookie_line + ) + + if in_msdmd_fence: + contiguous = ( + bool(current) + and token.start[0] == current[-1].end[0] + 1 + and token.start[1] == current[-1].start[1] + ) + if contiguous: + current.append(token) + if is_fence and (not is_end_fence or fence_name != open_fence_name): + # A closing fence must name the open block; an opening + # fence inside an open block is equally unpaired. Neither + # may be accepted as a well-formed MSDMD block. + start_line = current[0].start[0] + closing = "closing" if is_end_fence else "opening" + diagnostics.append( + CommentDiagnostic( + code="msdmd_fence_mismatched", + message=( + f"MSDMD fence {open_fence_name} starting on line {start_line} " + f"meets {closing} fence {fence_name} on line {token.start[0]}" + ), + line=token.start[0], + ) + ) + affected = [open_fence_name] if is_end_fence else [open_fence_name, fence_name] + for name in affected: + if name is not None: + suppressed_closes[name] = suppressed_closes.get(name, 0) + 1 + flush() + in_msdmd_fence = False + open_fence_name = None + elif is_fence: + flush() + in_msdmd_fence = False + open_fence_name = None + continue + start_line = current[0].start[0] + diagnostics.append( + CommentDiagnostic( + code="msdmd_fence_interrupted", + message=f"MSDMD fence starting on line {start_line} is interrupted or unclosed", + line=start_line, + ) + ) + flush() + in_msdmd_fence = False + open_fence_name = None + if is_fence and not is_end_fence: + flush() + current = [token] + in_msdmd_fence = True + open_fence_name = fence_name + # A new opening of this name ends any suppression left by an + # earlier mismatch, so later genuine orphans remain visible. + suppressed_closes.pop(fence_name, None) + continue + if is_fence and fence_name is not None and suppressed_closes.get(fence_name, 0): + suppressed_closes[fence_name] -= 1 + flush() + groups.append([token]) + continue + if is_fence: + diagnostics.append( + CommentDiagnostic( + code="msdmd_fence_unmatched_close", + message=( + f"MSDMD closing fence {fence_name} on line {token.start[0]} " + "has no matching opening fence" + ), + line=token.start[0], + ) + ) + flush() + groups.append([token]) + continue + if is_singleton_convention: + flush() + groups.append([token]) + continue + if ( + current + and token.start[0] == current[-1].end[0] + 1 + and token.start[1] == current[-1].start[1] + ): + current.append(token) + else: + flush() + current = [token] + if in_msdmd_fence: + start_line = current[0].start[0] + diagnostics.append( + CommentDiagnostic( + code="msdmd_fence_unclosed", + message=f"MSDMD fence starting on line {start_line} is unclosed", + line=start_line, + ) + ) + flush() + return [CommentGroup(tuple(group)) for group in groups], diagnostics + + +def _special_comment_kind( + group: CommentGroup, + encoding_cookie_line: int | None, +) -> str | None: + lines = group.text_lines + if group.start_line == 1 and group.tokens[0].string.startswith("#!"): + return "shebang" + if group.start_line == encoding_cookie_line: + return "encoding_cookie" + if any(line.strip().startswith("ratios:") for line in lines): + return "msdmd_ratios" + if any(_FENCE_RE.match(line.strip()) for line in lines): + return "msdmd_block" + return None + + +def _leading_subject(group: CommentGroup, symbols: Sequence[Symbol]) -> Symbol | None: + candidates = [ + symbol + for symbol in symbols + if symbol.column == group.start_column and symbol.header_line == group.end_line + 1 + ] + if not candidates: + return None + return min(candidates, key=lambda symbol: (symbol.header_line, -symbol.depth)) + + +def _lexical_end_line(symbol: Symbol, lines: Sequence[str]) -> int: + end_line = symbol.end_line + for line_number in range(symbol.end_line + 1, len(lines) + 1): + raw = lines[line_number - 1] + if not raw.strip(): + continue + indentation = len(raw) - len(raw.lstrip(" \t\f")) + if indentation <= symbol.column: + break + end_line = line_number + return end_line + + +def _enclosing_subject( + group: CommentGroup, + symbols: Sequence[Symbol], + lines: Sequence[str], +) -> Symbol | None: + # A declaration's lexical region begins at its first decorator, so comments + # before or between decorators (and inside decorator expressions) belong to + # the decorated declaration rather than drifting to an outer scope. + candidates = [ + symbol + for symbol in symbols + if symbol.header_line <= group.start_line + and group.end_line <= _lexical_end_line(symbol, lines) + and ( + group.start_line <= symbol.end_line + or group.start_column > symbol.column + ) + ] + if not candidates: + return None + return max(candidates, key=lambda symbol: (symbol.depth, symbol.start_line, -symbol.end_line)) + + +def _symbol_record(symbol: Symbol, lines: Sequence[str], starts: Sequence[int]) -> dict: + decorators = [ast.unparse(item) for item in symbol.node.decorator_list] + source_span = _node_span(symbol.node, lines, starts) + if symbol.node.decorator_list: + source_span["start"] = _span( + lines, + starts, + symbol.header_line, + symbol.column, + symbol.header_line, + symbol.column, + )["start"] + return { + "record_type": "symbol", + "id": symbol.identity, + "kind": symbol.kind, + "qualified_name": symbol.qualified_name, + "parent": symbol.parent_identity, + "signature": symbol.signature, + "signature_sha256": symbol.signature_sha256, + "decorators": decorators, + "standing": "syntactically_observed", + "source_span": source_span, + } + + +def _docstring_record( + owner: ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef, + subject: str, + subject_kind: str, + lines: Sequence[str], + starts: Sequence[int], +) -> dict | None: + if not owner.body: + return None + statement = owner.body[0] + if not ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, str) + ): + return None + return { + "record_type": "metadata", + "metadata_kind": "documentation", + "convention": "python.docstring.pep257", + "subject": subject, + "subject_kind": subject_kind, + "attachment": "native_docstring", + "standing": "declared", + "text": statement.value.value, + "source_span": _node_span(statement, lines, starts), + } + + +def _comment_record( + group: CommentGroup, + subject: str, + subject_kind: str, + attachment: str, + metadata_kind: str, + lines: Sequence[str], + starts: Sequence[int], +) -> dict: + return { + "record_type": "metadata", + "metadata_kind": metadata_kind, + "convention": "python.line-comment", + "subject": subject, + "subject_kind": subject_kind, + "attachment": attachment, + "standing": "syntactically_observed", + "text": "\n".join(group.text_lines), + "source_span": _span( + lines, + starts, + group.start_line, + group.start_column, + group.end_line, + group.end_column, + ), + } + + +def _freshness_key( + repo: str, + source_path: str, + source_sha256: str, + reader_sha256: str, + schema_sha256: str, + reader_manifest_sha256: str, + source_revision: str, + runtime_identity: dict[str, str], +) -> str: + identity = { + "schema": SCHEMA_ID, + "schema_version": SCHEMA_VERSION, + "repo": repo, + "source_path": source_path, + "source_sha256": source_sha256, + "source_revision": source_revision, + "reader": READER_ID, + "reader_version": READER_VERSION, + "reader_sha256": reader_sha256, + "schema_sha256": schema_sha256, + "reader_manifest_sha256": reader_manifest_sha256, + **runtime_identity, + } + return _sha256(_canonical_json(identity).encode("utf-8")) + + +def project_python_module( + path: Path, + *, + root: Path, + repo: str, + revision: str | None = None, +) -> list[dict]: + """Return deterministic projection records for one Python source file. + + ``path`` must resolve within ``root``. Invalid syntax produces a module + record plus a diagnostic record; the source is never imported or executed. + """ + + root = root.resolve() + resolved = path.resolve() + try: + relative = resolved.relative_to(root).as_posix() + except ValueError as exc: + raise ValueError(f"source escapes root: {path}") from exc + + return project_python_bytes(resolved.read_bytes(), source_path=relative, repo=repo, revision=revision) + + +def project_python_bytes( + source: bytes, *, source_path: str, repo: str, revision: str | None = None, +) -> list[dict]: + """Project an already-bounded immutable buffer; never reopen the inspected file. + + The path is provenance only. Both the standalone file runner and unified + collector call this same syntax/attachment implementation. + """ + relative = Path(source_path).as_posix() + if Path(relative).is_absolute() or ".." in Path(relative).parts: + raise ValueError("source_path must be repository-relative") + source_sha256 = _sha256(source) + reader_sha256 = _reader_sha256() + schema_sha256 = _schema_sha256() + reader_manifest_sha256 = _reader_manifest_sha256() + runtime_identity = _runtime_identity() + source_revision = revision or "hmmm" + header = { + "record_type": "module", + "schema": SCHEMA_ID, + "schema_version": SCHEMA_VERSION, + "schema_sha256": schema_sha256, + "repo": repo, + "source_revision": source_revision, + "source_path": relative, + "source_sha256": source_sha256, + "source_encoding": "hmmm", + "reader": READER_ID, + "reader_version": READER_VERSION, + "reader_sha256": reader_sha256, + "reader_manifest_sha256": reader_manifest_sha256, + **runtime_identity, + "freshness_key_sha256": _freshness_key( + repo, + relative, + source_sha256, + reader_sha256, + schema_sha256, + reader_manifest_sha256, + source_revision, + runtime_identity, + ), + "module_id": _module_address(repo, relative), + "status": "complete", + "hmmm": [] if revision else ["repository revision not supplied"], + } + + try: + source_encoding = tokenize.detect_encoding(_byte_line_reader(source))[0] + header["source_encoding"] = source_encoding + decoded = source.decode(source_encoding) + tree = ast.parse( + decoded, + filename=relative, + type_comments=True, + feature_version=(sys.version_info.major, sys.version_info.minor), + ) + except (SyntaxError, UnicodeDecodeError) as exc: + header["status"] = "invalid" + header["hmmm"].append("source could not be parsed as Python") + diagnostic = { + "record_type": "diagnostic", + "code": "python_parse_error", + "severity": "error", + "message": str(exc), + "source_path": relative, + "line": getattr(exc, "lineno", None), + "column": getattr(exc, "offset", None), + } + return [header, diagnostic] + + lines = _python_source_lines(decoded) or [""] + starts = _line_start_bytes(lines) + module_id = header["module_id"] + symbols = _discover_symbols(tree, repo, relative) + records: list[dict] = [header] + records.extend(_symbol_record(symbol, lines, starts) for symbol in symbols) + + module_doc = _docstring_record(tree, module_id, "module", lines, starts) + if module_doc is not None: + records.append(module_doc) + for symbol in symbols: + doc = _docstring_record( + symbol.node, + symbol.identity, + symbol.kind, + lines, + starts, + ) + if doc is not None: + records.append(doc) + + encoding_cookie_line = _encoding_cookie_line(source) + groups, comment_diagnostics = _comment_groups(lines, encoding_cookie_line) + for diagnostic in comment_diagnostics: + header["status"] = "invalid" + hmmm = ( + "comments could not be tokenized completely" + if diagnostic.code == "python_tokenize_error" + else "MSDMD comment fence is malformed" + ) + if hmmm not in header["hmmm"]: + header["hmmm"].append(hmmm) + records.append( + { + "record_type": "diagnostic", + "code": diagnostic.code, + "severity": "error", + "message": diagnostic.message, + "source_path": relative, + "line": diagnostic.line, + } + ) + + for group in groups: + special = _special_comment_kind(group, encoding_cookie_line) + if special is not None: + records.append( + _comment_record( + group, + module_id, + "module", + "module_convention", + special, + lines, + starts, + ) + ) + continue + leading = _leading_subject(group, symbols) + if leading is not None: + records.append( + _comment_record( + group, + leading.identity, + leading.kind, + "leading_trivia", + "comment", + lines, + starts, + ) + ) + continue + enclosing = _enclosing_subject(group, symbols, lines) + if enclosing is not None: + records.append( + _comment_record( + group, + enclosing.identity, + enclosing.kind, + "nearest_enclosing_symbol", + "comment", + lines, + starts, + ) + ) + continue + records.append( + _comment_record( + group, + module_id, + "module", + "module_scope", + "comment", + lines, + starts, + ) + ) + + return records + + +def render_jsonl(records: Sequence[dict]) -> str: + """Render records as deterministic JSON Lines text with LF terminators. + + Callers persist ``content.encode("utf-8")`` exactly; see + ``write_projections``. + """ + + return "".join(f"{_canonical_json(record)}\n" for record in records) + + +def _iter_python_sources(root: Path, selected: Sequence[Path] = ()) -> Iterator[Path]: + root = root.resolve() + if selected: + resolved = sorted({path.resolve() for path in selected}) + for path in resolved: + path.relative_to(root) + if path.suffix != ".py" or not path.is_file(): + raise ValueError(f"selected source is not a Python file: {path}") + yield path + return + + def walk(directory: Path) -> Iterator[Path]: + for child in sorted(directory.iterdir(), key=lambda item: item.name): + if child.is_symlink(): + continue + if child.is_dir(): + if child.name in DEFAULT_SKIP_DIRS: + continue + yield from walk(child) + elif child.is_file() and child.suffix == ".py": + yield child + + yield from walk(root) + + +def project_tree( + root: Path, + repo: str, + *, + sources: Sequence[Path] = (), + revision: str | None = None, +) -> dict[str, str]: + """Return ``relative sidecar path -> deterministic JSONL`` for Python files.""" + + root = root.resolve() + projections: dict[str, str] = {} + for source in _iter_python_sources(root, sources): + relative = source.relative_to(root).as_posix() + projections[f"{relative}{PROJECTION_SUFFIX}"] = render_jsonl( + project_python_module(source, root=root, repo=repo, revision=revision) + ) + return projections + + +def _projection_target(out_dir: Path, relative: str) -> Path: + if not relative.endswith(PROJECTION_SUFFIX): + raise ValueError(f"projection path must end with {PROJECTION_SUFFIX}: {relative}") + target = (out_dir / relative).resolve() + try: + target.relative_to(out_dir) + except ValueError as exc: + raise ValueError(f"projection path escapes output directory: {relative}") from exc + return target + + +def write_projections( + out_dir: Path, + projections: dict[str, str], + *, + prune: bool = False, +) -> list[Path]: + """Atomically write projections, optionally pruning a complete output set.""" + + out_dir = out_dir.resolve() + out_dir.mkdir(parents=True, exist_ok=True) + expected = {_projection_target(out_dir, relative) for relative in projections} + if prune: + for stale in sorted(out_dir.rglob(f"*{PROJECTION_SUFFIX}")): + if stale not in expected: + stale.unlink() + written: list[Path] = [] + for relative, content in sorted(projections.items()): + target = _projection_target(out_dir, relative) + target.parent.mkdir(parents=True, exist_ok=True) + content_bytes = content.encode("utf-8") + if not target.exists() or target.read_bytes() != content_bytes: + temporary: Path | None = None + try: + # Binary mode: text mode would translate LF to os.linesep + # (CRLF on Windows) and break byte-exact write/check convergence. + with tempfile.NamedTemporaryFile( + mode="wb", + dir=target.parent, + prefix=f".{target.name}.", + suffix=".tmp", + delete=False, + ) as stream: + stream.write(content_bytes) + stream.flush() + os.fsync(stream.fileno()) + temporary = Path(stream.name) + temporary.replace(target) + temporary = None + finally: + if temporary is not None and temporary.exists(): + temporary.unlink() + written.append(target) + return written + + +def check_projections( + out_dir: Path, + projections: dict[str, str], + *, + check_unexpected: bool = False, +) -> list[str]: + """Return deterministic freshness and diagnostic findings without writing.""" + + out_dir = out_dir.resolve() + findings: list[str] = [] + expected = {_projection_target(out_dir, relative) for relative in projections} + actual = set(out_dir.rglob(f"*{PROJECTION_SUFFIX}")) if out_dir.exists() else set() + for missing in sorted(expected - actual): + findings.append(f"missing:{missing.relative_to(out_dir).as_posix()}") + if check_unexpected: + for unexpected in sorted(actual - expected): + findings.append(f"unexpected:{unexpected.relative_to(out_dir).as_posix()}") + for relative, expected_content in sorted(projections.items()): + target = _projection_target(out_dir, relative) + if target.exists() and target.read_bytes() != expected_content.encode("utf-8"): + findings.append(f"stale:{relative}") + first = json.loads(expected_content.split("\n", 1)[0]) + if first["status"] != "complete": + findings.append(f"invalid:{relative}") + return findings + + +def _selected_sources(root: Path, values: Iterable[str]) -> list[Path]: + return [root / value for value in values] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path("."), help="authorized source root") + parser.add_argument("--repo", required=True, help="repository authority slug") + parser.add_argument("--revision", help="exact source revision; omitted remains visible as hmmm") + parser.add_argument("--out-dir", type=Path, required=True, help="generated projection root") + parser.add_argument( + "--source", + action="append", + default=[], + help="repository-relative Python source to project; repeat to select several", + ) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--write", action="store_true", help="write fresh projections") + mode.add_argument("--check", action="store_true", help="fail on missing, stale, unexpected, or invalid projections") + args = parser.parse_args(argv) + + root = args.root.resolve() + projections = project_tree( + root, + args.repo, + sources=_selected_sources(root, args.source), + revision=args.revision, + ) + complete_scope = not args.source + if args.write: + written = write_projections(args.out_dir, projections, prune=complete_scope) + invalid = [ + relative + for relative, content in projections.items() + if json.loads(content.split("\n", 1)[0])["status"] != "complete" + ] + print(f"msdmd module projections: wrote {len(written)}") + for relative in invalid: + print(f"INVALID {relative}") + return 1 if invalid else 0 + + findings = check_projections( + args.out_dir, + projections, + check_unexpected=complete_scope, + ) + status = "pass" if not findings else "fail" + print(f"msdmd module projections: {status} ({len(projections)} modules, {len(findings)} findings)") + for finding in findings: + print(finding) + return 1 if findings else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +# ratios: loc_comments=910:128 imports_exports=14:10 calls_definitions=259:54 diff --git a/.agents/skills/msdmd/native_code.py b/.agents/skills/msdmd/native_code.py new file mode 100644 index 00000000..ba276be4 --- /dev/null +++ b/.agents/skills/msdmd/native_code.py @@ -0,0 +1,220 @@ +# ratios: loc_comments=195:14 imports_exports=15:3 calls_definitions=83:3 +"""Syntax-aware code readers for the unified collection. + +Usage: registry calls read_python/read_typescript with bounded bytes and context. +Python delegates symbol/comment attachment to module_projection. The TypeScript +compiler is a library worker, not the inspected program. Neither reader resolves +imports, loads target configuration, or establishes runtime behavior. +""" +from __future__ import annotations + +import ast +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +from msdmd.module_projection import project_python_bytes, _byte_line_reader +import tokenize + + +def read_python(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + from msdmd.readers import _fact, _diagnostic, _subject_address, _edge + context = dict(context, convention_version=f'Python {sys.version_info.major}.{sys.version_info.minor}', dialect='python') + rid = 'python-ast' + records = project_python_bytes(data, source_path=context['file'], repo=context['repo'], revision=context['revision']) + header = records[0] + facts: list = [] + edges: list = [] + diagnostics: list = [] + symbols = {r['id']: r for r in records if r['record_type'] == 'symbol'} + identities = {key: value['qualified_name'] for key, value in symbols.items()} + # Only duplicate qualified names need the attachment reader's signature suffix. + counts: dict[str, int] = {} + for name in identities.values(): + counts[name] = counts.get(name, 0) + 1 + identities = {key: name if counts[name] == 1 else key for key, name in identities.items()} + identities[header['module_id']] = context['file'] + metadata_counts: dict[tuple[str, str], int] = {} + + def location(record: dict) -> dict: + span = record['source_span'] + return {'start_line': span['start']['line'], 'end_line': span['end']['line']} + + for record in records[1:]: + if record['record_type'] == 'diagnostic': + diagnostics.append(_diagnostic(context, reader_id=rid, code=record['code'], + message=record['message'], status='invalid', severity='error')) + continue + if record['record_type'] == 'symbol': + identity = identities[record['id']] + kind = 'class-declaration' if record['kind'] == 'class' else 'callable-declaration' + fact = _fact(context, reader_id=rid, kind=kind, scope='symbol', identity=identity, + location=location(record), native_id=record['qualified_name'], value={ + 'name': record['qualified_name'].split('.')[-1], 'qualified_name': record['qualified_name'], + 'kind': record['kind'], 'signature': record['signature'], 'decorators': record['decorators'], + 'attachment_identity': record['id'], 'parent_identity': record['parent']}, + projection={'mapping_version': 'python-surfaces@2', 'canonical_field': 'declared_surface', 'loss': 'behavior not inferred'}) + # Stable semantic pointer; exact byte/line span is provenance, not identity. + fact['address'] = fact['subject']['address'] + '/fact/declaration' + fact['source']['location']['span'] = record['source_span'] + facts.append(fact) + continue + if record['record_type'] != 'metadata': + continue + identity = identities.get(record['subject'], record['subject']) + scope = 'module' if record['subject'] == header['module_id'] else 'symbol' + kind = 'documentation' if record['metadata_kind'] == 'documentation' else 'comment' + value = {'text': record['text'], 'attachment': record['attachment'], + 'attachment_identity': record['subject'], 'metadata_kind': record['metadata_kind']} + if kind == 'documentation': + try: + import docstring_parser + parsed = docstring_parser.parse(record['text']) + value['format'] = parsed.style.name.lower() + value['fields'] = [dict(vars(field)) for field in parsed.meta] + value['summary'] = parsed.short_description + value['description'] = parsed.long_description + except ImportError: + diagnostics.append(_diagnostic(context, reader_id=rid, code='missing_docstring_parser', + message='install msdmd/requirements.txt to extract structured docstring fields', status='unsupported')) + except Exception as exc: + # A parser rejection must not discard the source-owned docstring. + diagnostics.append(_diagnostic(context, reader_id=rid, code='docstring_parse_error', + message=type(exc).__name__, status='unsupported')) + key = (record['subject'], kind) + ordinal = metadata_counts.get(key, 0); metadata_counts[key] = ordinal + 1 + fact = _fact(context, reader_id=rid, kind=kind, scope=scope, identity=identity, + location=location(record), value=value, + convention='python.pep257' if kind == 'documentation' else 'python.line-comment', standing='declared') + fact['address'] = fact['subject']['address'] + f'/fact/{kind}/{ordinal}' + fact['source']['location']['span'] = record['source_span'] + facts.append(fact) + if kind == 'comment': + for line_ordinal, line in enumerate(record['text'].splitlines()): + for tag in ('SPDX-License-Identifier', 'SPDX-FileCopyrightText'): + if tag + ':' in line: + facts.append(_fact(context, reader_id=rid, kind='license-declaration', scope=scope, + identity=identity, location=location(record), value={'tag': tag, 'value': line.split(tag + ':', 1)[1].strip()}, + native_id=f'{tag}:{ordinal}:{line_ordinal}', convention='spdx.file-header', standing='declared')) + if header['status'] == 'invalid': + return facts, edges, diagnostics + encoding = tokenize.detect_encoding(_byte_line_reader(data))[0] + tree = ast.parse(data.decode(encoding), type_comments=True) + import_ordinals: dict[str, int] = {} + for node in ast.walk(tree): + if not isinstance(node, (ast.Import, ast.ImportFrom)): + continue + owners = [r for r in symbols.values() if r['source_span']['start']['line'] <= node.lineno <= r['source_span']['end']['line']] + owner = min(owners, key=lambda r: r['source_span']['end']['line'] - r['source_span']['start']['line']) if owners else None + identity = identities[owner['id']] if owner else context['file'] + scope = 'symbol' if owner else 'module' + for alias in node.names: + module = alias.name if isinstance(node, ast.Import) else '.' * node.level + (node.module or '') + item = {'kind': 'import' if isinstance(node, ast.Import) else 'from_import', + 'module': module, 'name': alias.name, 'alias': alias.asname, + 'level': getattr(node, 'level', 0)} + ordinal = import_ordinals.get(identity, 0); import_ordinals[identity] = ordinal + 1 + fact = _fact(context, reader_id=rid, kind='dependency', scope=scope, identity=identity, + location={'start_line': node.lineno, 'end_line': node.end_lineno}, value=item, + native_id=f'{module}:{alias.name}:{ordinal}', projection={ + 'mapping_version': 'python-imports@2', 'relation': 'imports', 'loss': 'conditions retained in source; resolution not executed'}) + fact['address'] = fact['subject']['address'] + f'/fact/import/{ordinal}' + facts.append(fact) + edges.append(_edge(fact['subject']['address'], 'python-module:' + module, 'imports')) + # A literal initializer is not a complete export set after another write, + # mutation, conditional assignment, or escape through an alias/call. + # Only one simple module-level literal assignment with no subsequent uses + # earns static standing. Other syntax remains visible without execution. + assignments = [] + allowed_names: set[int] = set() + for node in tree.body: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] if isinstance(node, ast.AnnAssign) else [] + if len(targets) == 1 and isinstance(targets[0], ast.Name) and targets[0].id == '__all__': + assignments.append(node) + allowed_names.add(id(targets[0])) + export_uses = [node for node in ast.walk(tree) if isinstance(node, ast.Name) and node.id == '__all__'] + unresolved = len(assignments) != 1 or any(id(node) not in allowed_names for node in export_uses) + if assignments: + node = assignments[-1] + value = node.value + literal = isinstance(value, (ast.List, ast.Tuple)) and all( + isinstance(item, ast.Constant) and isinstance(item.value, str) for item in value.elts) + unresolved = unresolved or not literal + names = [item.value for item in value.elts] if literal else None + facts.append(_fact(context, reader_id=rid, kind='exports', scope='module', identity=context['file'], + location={'start_line': node.lineno, 'end_line': node.end_lineno}, native_id='__all__', + value={'names': names if not unresolved else None, + 'declared_names': names, + 'resolution': 'dynamic-unresolved' if unresolved else 'static'})) + if export_uses and unresolved: + diagnostics.append(_diagnostic(context, reader_id=rid, code='dynamic_python_exports', + message='__all__ has a nonliteral, repeated, conditional, mutated or escaped declaration; final exports not inferred', + status='dynamic-unresolved')) + return facts, edges, diagnostics + + +def read_typescript(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + from msdmd.readers import _fact, _diagnostic, _edge, _subject_address + rid = 'typescript-compiler' + helper = Path(__file__).with_name('typescript-reader.cjs') + env = {key: value for key, value in os.environ.items() if key not in {'NODE_OPTIONS', 'NODE_PATH'}} + # Package resolution is rooted at the trusted helper, never the inspected repo. + result = subprocess.run(['node', str(helper)], input=json.dumps({'path': str(path), 'text': data.decode('utf-8-sig')}), + capture_output=True, text=True, cwd=helper.parent, env=env, check=False) + if result.returncode: + return [], [], [_diagnostic(context, reader_id=rid, code='typescript_reader_unavailable', + message='trusted TypeScript worker failed; install Node and run npm ci --prefix msdmd', status='unsupported')] + parsed = json.loads(result.stdout) + context = dict(context, convention_version=parsed['version'], dialect='typescript-compiler-jsdoc') + facts: list = [] + edges: list = [] + diagnostics = [_diagnostic(context, reader_id=rid, code=d['code'], + message=d.get('message', 'dynamic module loading is not resolved'), status=d['status'], + severity='error' if d['status'] == 'invalid' else 'warning', + location={'start_line': d['start_line'], 'end_line': d['end_line']}) for d in parsed['diagnostics']] + for index, item in enumerate(parsed['declarations']): + fact = _fact(context, reader_id=rid, kind='exported-declaration' if item['exported'] else 'symbol-declaration', + scope='symbol', identity=item['identity'], native_id=None if item.get('anonymous') else item['qualified_name'], value=item, + location={'start_line': item['start_line'], 'end_line': item['end_line']}) + fact['address'] = fact['subject']['address'] + '/fact/declaration' + facts.append(fact) + if item['exported']: + edges.append(_edge(_subject_address(context, 'module', context['file']), fact['subject']['address'], 'exports')) + for index, item in enumerate(parsed['docs']): + fact = _fact(context, reader_id=rid, kind='documentation', scope='symbol', identity=item['identity'], + value=item, location={'start_line': item['start_line'], 'end_line': item['end_line']}, + convention='typescript.documentation-comment', standing='declared') + fact['address'] = fact['subject']['address'] + f'/fact/documentation/{index}' + facts.append(fact) + for index, item in enumerate(sorted(parsed['comments'], key=lambda c: c['start_line'])): + scope = 'symbol' if item.get('identity') else 'module' + identity = item.get('identity') or context['file'] + fact = _fact(context, reader_id=rid, kind='comment', scope=scope, identity=identity, + value=item, location={'start_line': item['start_line'], 'end_line': item['end_line']}, native_id=str(index)) + facts.append(fact) + for tag in ('SPDX-License-Identifier', 'SPDX-FileCopyrightText'): + for match_ordinal, match in enumerate(re.finditer(re.escape(tag) + r':\s*([^\r\n]*?)(?:\*/|$)', item['text'], re.M)): + facts.append(_fact(context, reader_id=rid, kind='license-declaration', scope='module', identity=context['file'], + value={'tag': tag, 'value': match[1].strip()}, native_id=f'{tag}:{index}:{match_ordinal}', + location={'start_line': item['start_line'], 'end_line': item['end_line']}, convention='spdx.file-header', standing='declared')) + for index, item in enumerate(parsed['imports']): + fact = _fact(context, reader_id=rid, kind='dependency', scope='module', identity=context['file'], + value=item, native_id=str(index), location={'start_line': item['start_line'], 'end_line': item['end_line']}) + facts.append(fact) + edges.append(_edge(fact['subject']['address'], 'ecma-module:' + item['module'], item['kind'])) + for index, item in enumerate(parsed.get('exports', [])): + fact = _fact(context, reader_id=rid, kind='local-export', scope='module', identity=context['file'], + value=item, native_id=item['exported_name'], location={'start_line': item['start_line'], 'end_line': item['end_line']}) + fact['address'] = fact['subject']['address'] + f'/fact/local-export/{index}' + facts.append(fact) + targets = [_subject_address(context, 'symbol', identity) for identity in item['declaration_identities']] + fallback = 'ecma-local:' + item['local_name'] if item.get('local_name') else fact['address'] + for target in targets or [fallback]: + edges.append(_edge(fact['subject']['address'], target, 'exports:' + item['exported_name'])) + return facts, edges, diagnostics +# ratios: loc_comments=195:14 imports_exports=15:3 calls_definitions=83:3 diff --git a/.agents/skills/msdmd/package-lock.json b/.agents/skills/msdmd/package-lock.json new file mode 100644 index 00000000..a364aaa2 --- /dev/null +++ b/.agents/skills/msdmd/package-lock.json @@ -0,0 +1,26 @@ +{ + "name": "msdmd-native-reader-runtime", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "msdmd-native-reader-runtime", + "dependencies": { + "typescript": "5.8.3" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/.agents/skills/msdmd/package.json b/.agents/skills/msdmd/package.json new file mode 100644 index 00000000..2e4ce030 --- /dev/null +++ b/.agents/skills/msdmd/package.json @@ -0,0 +1,6 @@ +{ + "name": "msdmd-native-reader-runtime", + "private": true, + "description": "Pinned syntax-only TypeScript parser for MSDMD; never loads inspected projects", + "dependencies": {"typescript": "5.8.3"} +} diff --git a/.agents/skills/msdmd/parsers/universal.py b/.agents/skills/msdmd/parsers/universal.py index 204f533f..9ce8367f 100644 --- a/.agents/skills/msdmd/parsers/universal.py +++ b/.agents/skills/msdmd/parsers/universal.py @@ -1,4 +1,27 @@ -# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 +# ratios: loc_comments=161:80 imports_exports=4:7 calls_definitions=55:10 +# === MODULE_BUILD === +# id: msdmd_python_reference_parser +# module_name: universal +# module_kind: instrument +# summary: parses canonical line-comment metadata without executing inspected source +# owner: The Interdependency skill-lib +# public_surface: COMMENT_MARKERS, RATIO_IDS, marker_for, parse_text, parse_file, walk_tree, parse_ratios, parse_ratios_file, ratios_placement +# internal_surface: marker and block matching helpers +# auth_boundary: none +# storage_boundary: read +# network_boundary: none +# user_data_boundary: read +# admin_only: false +# tests: tests/test_universal_parser.py +# rollout: exact-pinned reference parser propagation +# rollback: restore a previously accepted exact parser identity +# === END MODULE_BUILD === +# === CONTRACTS === +# id: msdmd_python_parser_preserves_field_names +# given: a valid metadata entry uses lowercase snake-case field names containing digits +# then: parsed entries retain those field names and string values without executing the inspected source +# class: evidence +# === END CONTRACTS === """Universal msdmd parser — pure stdlib. Implements the parser contract from ``msdmd/SKILL.md``: extracts every @@ -69,7 +92,7 @@ } _DEFAULT_SKIP = ( - "__pycache__", "node_modules", ".git", ".venv", "venv", + "__pycache__", "node_modules", ".git", ".agents", ".venv", "venv", "dist", "build", ".next", ".nuxt", "target", ".pytest_cache", ".mypy_cache", ".tox", ) @@ -100,7 +123,7 @@ def parse_text(text: str, block_name: str, marker: str = "#") -> list[dict]: block_re = _block_regex(block_name, marker) m = re.escape(marker) id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") - field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_]+):\s*(?P.+?)\s*$") + field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_][a-z0-9_]*):\s*(?P.+?)\s*$") entries: list[dict] = [] for block in block_re.finditer(text): @@ -189,7 +212,7 @@ def iter_source_files(path: Path) -> Iterable[Path]: # boundary to literal line 2: # ratios: loc_comments=N:M imports_exports=N:M calls_definitions=N:M RATIO_IDS = ("loc_comments", "imports_exports", "calls_definitions") -_RATIOS_TOKEN_RE = re.compile(r"(?P[a-z_]+)=(?P\S+)") +_RATIOS_TOKEN_RE = re.compile(r"(?P[a-z_][a-z0-9_]*)=(?P\S+)") def _ratios_line_re(marker: str) -> re.Pattern[str]: @@ -253,4 +276,4 @@ def ratios_placement(text: str, marker: str = "#") -> tuple[bool, bool]: last_ok = bool(line_re.match(raw.rstrip())) break return (opening_ok, last_ok) -# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 +# ratios: loc_comments=161:80 imports_exports=4:7 calls_definitions=55:10 diff --git a/.agents/skills/msdmd/parsers/universal.ts b/.agents/skills/msdmd/parsers/universal.ts index 141f46c5..619e1a89 100644 --- a/.agents/skills/msdmd/parsers/universal.ts +++ b/.agents/skills/msdmd/parsers/universal.ts @@ -1,4 +1,27 @@ // ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm +// === MODULE_BUILD === +// id: msdmd_typescript_reference_parser +// module_name: universal +// module_kind: instrument +// summary: parses canonical line-comment metadata without executing inspected source +// owner: The Interdependency skill-lib +// public_surface: Entry, WalkOptions, COMMENT_MARKERS, RATIO_IDS, markerFor, parseText, parseFile, walkTree, parseRatios, parseRatiosFile, ratiosPlacement +// internal_surface: marker and block matching helpers +// auth_boundary: none +// storage_boundary: read +// network_boundary: none +// user_data_boundary: read +// admin_only: false +// tests: tests/test_universal_parser.py::test_typescript_parser_field_contract +// rollout: exact-pinned reference parser propagation +// rollback: restore a previously accepted exact parser identity +// === END MODULE_BUILD === +// === CONTRACTS === +// id: msdmd_typescript_parser_preserves_field_names +// given: a valid metadata entry uses lowercase snake-case field names containing digits +// then: parsed entries retain those field names and string values without executing the inspected source +// class: evidence +// === END CONTRACTS === /** * Universal msdmd parser — pure Node stdlib (fs, path). * @@ -61,7 +84,7 @@ export const COMMENT_MARKERS: Record = { }; const DEFAULT_SKIP = new Set([ - "__pycache__", "node_modules", ".git", ".venv", "venv", + "__pycache__", "node_modules", ".git", ".agents", ".venv", "venv", "dist", "build", ".next", ".nuxt", "target", ".pytest_cache", ".mypy_cache", ".tox", ]); @@ -86,7 +109,7 @@ export function parseText( "gm", ); const idRe = new RegExp(`^\\s*${m}\\s*id:\\s*(\\S+)\\s*$`); - const fieldRe = new RegExp(`^\\s*${m}\\s+([a-z_]+):\\s*(.+?)\\s*$`); + const fieldRe = new RegExp(`^\\s*${m}\\s+([a-z_][a-z0-9_]*):\\s*(.+?)\\s*$`); const entries: Entry[] = []; let match: RegExpExecArray | null; @@ -185,7 +208,7 @@ function ratiosLineRe(marker: string): RegExp { export function parseRatios(text: string, marker: string = "#"): Entry[] { const lineRe = ratiosLineRe(marker); - const tokenRe = /([a-z_]+)=(\S+)/g; + const tokenRe = /([a-z_][a-z0-9_]*)=(\S+)/g; const out: Entry[] = []; for (const raw of text.split("\n")) { const lm = lineRe.exec(raw.replace(/\s+$/, "")); diff --git a/.agents/skills/msdmd/python-module-reader.json b/.agents/skills/msdmd/python-module-reader.json new file mode 100644 index 00000000..c4bfbd16 --- /dev/null +++ b/.agents/skills/msdmd/python-module-reader.json @@ -0,0 +1,81 @@ +{ + "reader_id": "the-interdependency.msdmd.python-ast-tokenize", + "reader_version": "1.0.0", + "status": "partial", + "entry_point": "msdmd/module_projection.py", + "output_schema": "msdmd/module-projection.schema.json", + "dependencies": [ + "python-stdlib:ast", + "python-stdlib:tokenize" + ], + "supported_grammar": { + "selection": "The grammar of the executing Python runtime, passed explicitly to ast.parse as feature_version.", + "projection_fields": [ + "python_implementation", + "python_version", + "ast_feature_version" + ], + "freshness_binding": "All three runtime fields participate in freshness_key_sha256." + }, + "conventions": [ + "python.ast.declaration", + "python.docstring.pep257", + "python.line-comment", + "python.encoding-cookie", + "msdmd.fenced-block", + "msdmd.ratios" + ], + "detection": { + "suffixes": [ + ".py" + ], + "language_context": "Explicit Python source selection or .py tree discovery.", + "ambiguity": "Files without a .py suffix and Python-compatible embedded languages are not guessed." + }, + "supported_subset": [ + "module, class, function, async-function, method, and async-method AST ownership", + "qualified symbol identities and signatures", + "decorator spelling recovered with ast.unparse and source_span extended across decorator syntax", + "module, class, function, and method docstrings", + "syntax-aware Python line comments", + "leading-trivia and nearest-enclosing-symbol comment attachment, including decorator-region comments and trailing suite comments before lexical dedent", + "shebang, encoding-cookie, RATIOS, and MSDMD-fence module conventions", + "UTF-8 projection spans over decoded source split only at Python newlines (LF, CRLF, CR), including CR-only sources, and raw-source SHA-256", + "projection files written as exact UTF-8 bytes with LF record terminators on every platform", + "parse, tokenization, and malformed MSDMD-fence diagnostics: interrupted, unclosed, name-mismatched or nested-opening (msdmd_fence_mismatched), and unmatched-close (msdmd_fence_unmatched_close) fences, where a mismatch yields one diagnostic because one later close per affected block name, orphaned by that reported mismatch, is suppressed until the next opening of that name (an interrupted fence may also report its later close as msdmd_fence_unmatched_close); stricter than the universal block parser, which matches each block name independently" + ], + "limitations": [ + "imports and dependency edges", + "call graphs and runtime behavior", + "dynamic exports and evaluated annotations", + "Sphinx, Google, and NumPy docstring field parsing", + ".pyi-specific semantics", + "cross-revision rename identity", + "repository discovery and exclusion receipts", + "reconciliation with non-Python or MSDMD block collections", + "raw-file byte offsets: span byte offsets index the UTF-8 re-encoding of the decoded source text and differ from raw file offsets for BOM-prefixed or non-UTF-8 sources" + ], + "attachment": { + "native_docstring": "Attach to the AST owner.", + "leading_trivia": "Attach a contiguous comment group to the immediately following declaration when lexical columns match.", + "nearest_enclosing_symbol": "Attach every other interior comment to the deepest enclosing declaration, whose region begins at its first decorator and extends through trailing indented comments until lexical dedent.", + "module_convention": "Keep shebangs, encoding cookies, RATIOS seals, and MSDMD fences at module scope.", + "module_scope": "Keep otherwise unattached comments at module scope." + }, + "field_mapping": { + "ast_declaration": "symbol", + "ast_signature": "symbol.signature", + "ast_decorator": "symbol.decorators", + "docstring": "metadata:documentation", + "line_comment": "metadata:comment" + }, + "unknown_field_policy": "Unsupported semantic parsing is not fabricated; the partial scope remains declared in this manifest and repository guidance.", + "failure_behavior": "Invalid syntax, incomplete tokenization, or a malformed MSDMD fence (interrupted, unclosed, name-mismatched, nested opening, or unmatched close) emits a diagnostic and causes --check or --write to exit nonzero. Fence validation is stricter than the universal block parser.", + "safety": { + "executes_inspected_code": false, + "imports_inspected_modules": false, + "network_access": false, + "follows_external_references": false + }, + "fixtures": "tests/test_module_projection.py" +} diff --git a/.agents/skills/msdmd/readers.py b/.agents/skills/msdmd/readers.py new file mode 100644 index 00000000..fdb54d9d --- /dev/null +++ b/.agents/skills/msdmd/readers.py @@ -0,0 +1,1009 @@ +# ratios: loc_comments=889:48 imports_exports=22:5 calls_definitions=267:28 +# === MODULE_BUILD === +# id: msdmd_native_reader_registry +# module_name: readers +# module_kind: instrument +# summary: statically extracts supported native metadata conventions into provenance-bearing MSDMD facts +# owner: The Interdependency skill-lib +# public_surface: READER_MANIFESTS, readers_for, reader_for, read_native +# internal_surface: convention-specific pure readers and source-qualified fact helpers +# auth_boundary: none +# storage_boundary: read +# network_boundary: none +# user_data_boundary: read +# admin_only: false +# tests: tests/test_native_collection.py +# rollout: enable through msdmd.collect schema 2 collections +# rollback: retain schema 1 block-only consumers while removing the schema 2 runner claim +# === END MODULE_BUILD === +"""Static native metadata readers used by :mod:`msdmd.collect`. + +Usage guidance: + Call ``read_native(path, bytes, context)`` only with bytes already bounded + by the collector. Readers never import inspected Python, execute scripts, + expand templates, or resolve network references. ``READER_MANIFESTS`` is + emitted in every collection so consumers can negotiate the exact supported + subset instead of treating file recognition as full language support. + +The registry intentionally implements conventions exercised by skill-lib and +the MSDMD worked example: Python syntax/docstrings/imports, RATIOS boundary +lines, JSON documents, TOML manifests, Markdown YAML frontmatter, safe YAML and +JavaScript/TypeScript subsets, GitHub CODEOWNERS, shell declarations, systemd +units, Git ignore rules, Python requirements files, bounded license detection, +SVG document metadata, and root ``llms.txt`` structure. Unmatched inputs stay +visible in the collection discovery ledger as unsupported ``hmmm`` scope. +""" +from __future__ import annotations + +from functools import lru_cache +import datetime as dt +import hashlib +import io +import json +import math +import re +import shlex +import sys +import tokenize +import tomllib +import xml.etree.ElementTree as ET +from collections import Counter +from pathlib import Path +from typing import Any, Callable +from urllib.parse import quote + +from msdmd.parsers.universal import marker_for, parse_ratios, ratios_placement +from msdmd.formats import is_json_schema_uri, parse_json, parse_yaml, parse_xml +from msdmd.grammar_code import read_grammar +from msdmd.native_code import read_python as _read_python, read_typescript as _read_typescript +from msdmd.standards import read_standards + +READER_MANIFESTS: tuple[dict[str, Any], ...] = ({'reader_id': 'python-ast', + 'version': '2.0.0', + 'entry_point': 'msdmd.readers:_read_python', + 'dependencies': ['python-stdlib:ast', 'python-stdlib:tokenize', 'docstring-parser==0.18.0'], + 'convention': 'python.source-metadata', + 'specification': 'https://docs.python.org/3/reference/', + 'supported_versions': ['Python 3 syntax accepted by the running ast parser'], + 'feature_subset': ['qualified symbols and signatures', + 'AST/token comment attachment shared with module_projection', + 'ReST/Google/NumPy docstring fields', + 'lexically scoped imports', + 'literal __all__'], + 'detection': ['.py', '.pyi', '.pyw'], + 'scope_attachment': 'module and qualified symbol', + 'unknown_field_policy': 'preserve source expression or emit dynamic-unresolved diagnostic', + 'execution_policy': 'parse only; never import or evaluate', + 'support': 'implemented-and-tested', + 'known_limitations': ['runtime effects and dynamic imports/exports are not executed']}, + {'reader_id': 'msdmd-ratios', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_ratios', + 'dependencies': ['msdmd.parsers.universal'], + 'convention': 'msdmd.ratios-line', + 'specification': 'The-Interdependency/skill-lib ratios/SKILL.md', + 'supported_versions': ['named ratios line v1'], + 'feature_subset': ['opening/closing placement', 'named ratio/value preservation'], + 'detection': ['supported line-comment source containing ratios:'], + 'scope_attachment': 'file boundary occurrence', + 'unknown_field_policy': 'preserve hmmm as an explicit value', + 'execution_policy': 'text parse only; never run ratio computers', + 'support': 'implemented-and-tested'}, + {'reader_id': 'json-stdlib', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_json', + 'dependencies': ['python-stdlib:json'], + 'convention': 'json.document', + 'specification': 'RFC 8259', + 'supported_versions': ['RFC 8259 JSON'], + 'feature_subset': ['complete typed document tree', 'package.json dependency classes'], + 'detection': ['.json'], + 'scope_attachment': 'document and JSON Pointer', + 'unknown_field_policy': 'preserve in typed document tree', + 'execution_policy': 'parse only; no reference resolution', + 'support': 'implemented-and-tested'}, + {'reader_id': 'toml-stdlib', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_toml', + 'dependencies': ['python-stdlib:tomllib'], + 'convention': 'toml.document', + 'specification': 'TOML 1.0.0 and Python packaging metadata specifications', + 'supported_versions': ['TOML accepted by the running tomllib parser'], + 'feature_subset': ['complete typed document tree', 'pyproject project/build dependency classes'], + 'detection': ['.toml'], + 'scope_attachment': 'document and TOML key path', + 'unknown_field_policy': 'preserve in typed document tree', + 'execution_policy': 'parse only; never invoke build backends', + 'support': 'implemented-and-tested'}, + {'reader_id': 'markdown-frontmatter', + 'version': '2.0.0', + 'entry_point': 'msdmd.readers:_read_markdown', + 'dependencies': ['PyYAML==6.0.3'], + 'convention': 'markdown.yaml-frontmatter', + 'specification': 'https://yaml.org/spec/1.2.2/', + 'supported_versions': ['YAML 1.2 core scalar subset with typed scalar-key preservation'], + 'feature_subset': ['nested frontmatter and typed values', 'safe acyclic aliases', 'multiline scalars'], + 'detection': ['.md', '.mdx'], + 'scope_attachment': 'document', + 'unknown_field_policy': 'preserve typed tree; reject duplicate keys, custom tags, cyclic aliases and merge ' + 'dialects', + 'execution_policy': 'text parse only; never execute MDX', + 'support': 'implemented-and-tested'}, + {'reader_id': 'yaml-core', + 'version': '2.0.0', + 'entry_point': 'msdmd.readers:_read_yaml', + 'dependencies': ['PyYAML==6.0.3'], + 'convention': 'yaml.core', + 'specification': 'https://yaml.org/spec/1.2.2/', + 'supported_versions': ['YAML 1.2 core with typed scalar-key preservation'], + 'feature_subset': ['typed nested mappings/sequences', + 'flow syntax', + 'acyclic aliases', + 'multiline scalars'], + 'known_limitations': ['custom tags, merge keys and cyclic aliases rejected; keys retain lexical strings'], + 'detection': ['.yml', '.yaml'], + 'scope_attachment': 'document and structural path', + 'unknown_field_policy': 'diagnose unsupported syntax; preserve raw source by digest', + 'execution_policy': 'text parse only; no tags, templates, actions, or includes', + 'support': 'implemented-and-tested'}, + {'reader_id': 'typescript-compiler', + 'version': '2.0.0', + 'entry_point': 'msdmd.readers:_read_typescript', + 'dependencies': ['Node.js', 'typescript==5.8.3'], + 'convention': 'typescript.source-metadata', + 'specification': 'https://www.typescriptlang.org/docs/handbook/jsdoc-supported-types.html', + 'supported_versions': ['TypeScript 5.8.3 parser accepting JS/JSX/TS/TSX/MJS/MTS/CTS/CJS'], + 'feature_subset': ['syntax-tree symbol ownership', + 'multiline signatures', + 'imports and reexports', + 'compiler JSDoc tags with unknown tag text preserved'], + 'known_limitations': ['not a TSDoc validator', + 'no type checking, config loading, import resolution or target execution'], + 'detection': ['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts'], + 'scope_attachment': 'compiler AST module/symbol', + 'unknown_field_policy': 'retain raw documentation tags and report parse diagnostics', + 'execution_policy': 'trusted parser worker only; inspected code never executed', + 'support': 'implemented-and-tested'}, + {'reader_id': 'github-codeowners', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_codeowners', + 'dependencies': [], + 'convention': 'github.codeowners', + 'specification': 'https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners', + 'supported_versions': ['GitHub CODEOWNERS line grammar'], + 'feature_subset': ['ordered pattern and owner extraction'], + 'detection': ['CODEOWNERS', '.github/CODEOWNERS', 'docs/CODEOWNERS'], + 'scope_attachment': 'ordered path rule', + 'unknown_field_policy': 'diagnose malformed rules; do not infer operational ownership', + 'execution_policy': 'text parse only', + 'support': 'implemented-and-tested'}, + {'reader_id': 'shell-static', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_shell', + 'dependencies': [], + 'convention': 'shell.source-metadata', + 'specification': 'POSIX shebang convention and ShellCheck directives', + 'supported_versions': ['declaration extraction only'], + 'feature_subset': ['shebang', 'ShellCheck directives'], + 'detection': ['.sh', '.bash', '.zsh', '.fish', 'extensionless shebang scripts'], + 'scope_attachment': 'file', + 'unknown_field_policy': 'preserve directive text', + 'execution_policy': 'text parse only; never source or execute', + 'support': 'implemented-and-tested'}, + {'reader_id': 'systemd-unit', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_systemd', + 'dependencies': [], + 'convention': 'systemd.unit', + 'specification': 'https://www.freedesktop.org/software/systemd/man/latest/systemd.syntax.html', + 'supported_versions': ['section/key/value declaration extraction'], + 'feature_subset': ['ordered sections', 'repeated directives', 'continuation lines'], + 'detection': ['.service', '.socket', '.timer', '.path', '.target'], + 'scope_attachment': 'unit section and directive occurrence', + 'unknown_field_policy': 'preserve key/value and order without interpreting runtime effect', + 'execution_policy': 'text parse only; never invoke systemd', + 'support': 'implemented-and-tested'}, + {'reader_id': 'gitignore-lines', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_gitignore', + 'dependencies': [], + 'convention': 'git.ignore', + 'specification': 'https://git-scm.com/docs/gitignore', + 'supported_versions': ['ordered pattern declarations'], + 'feature_subset': ['pattern order', 'negation', 'anchoring', 'directory-only flag'], + 'detection': ['.gitignore'], + 'scope_attachment': 'repository path rule', + 'unknown_field_policy': 'preserve pattern text without evaluating matches', + 'execution_policy': 'text parse only; never invoke Git matching', + 'support': 'implemented-and-tested'}, + {'reader_id': 'python-requirements', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_requirements', + 'dependencies': [], + 'convention': 'python.requirements-file', + 'specification': 'PyPA requirements file format', + 'supported_versions': ['one-line requirement and option subset'], + 'feature_subset': ['requirement text', 'package name', 'options', 'ordered source lines'], + 'known_limitations': ['does not resolve includes, environment markers, URLs, or indexes'], + 'detection': ['requirements*.txt', 'constraints*.txt'], + 'scope_attachment': 'requirements document line', + 'unknown_field_policy': 'preserve text and redact URL credentials', + 'execution_policy': 'text parse only; never install or resolve packages', + 'support': 'partial'}, + {'reader_id': 'license-text', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_license', + 'dependencies': [], + 'convention': 'license.text', + 'specification': 'SPDX license identifiers plus exact known license headers', + 'supported_versions': ['MPL-2.0 exact header', 'SPDX-License-Identifier line'], + 'feature_subset': ['declared/detected expression', 'source digest'], + 'detection': ['LICENSE', 'LICENSE.txt', 'COPYING', 'COPYING.txt'], + 'scope_attachment': 'repository license document', + 'unknown_field_policy': 'hmmm identifier; never infer legal compatibility', + 'execution_policy': 'text parse only', + 'support': 'partial'}, + {'reader_id': 'svg-metadata', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_svg', + 'dependencies': ['python-stdlib:xml.etree.ElementTree'], + 'convention': 'svg.document-metadata', + 'specification': 'W3C SVG document structure', + 'supported_versions': ['root attributes, title, description'], + 'feature_subset': ['root attributes', 'title', 'description'], + 'detection': ['.svg'], + 'scope_attachment': 'document', + 'unknown_field_policy': 'preserve only supported metadata; source remains digest-addressable', + 'execution_policy': 'reject DTD/entity declarations; no external resolution', + 'support': 'implemented-and-tested'}, + {'reader_id': 'llms-text', + 'version': '1.0.0', + 'entry_point': 'msdmd.readers:_read_llms_text', + 'dependencies': [], + 'convention': 'llms.txt', + 'specification': 'root llms.txt Markdown instruction publication', + 'supported_versions': ['Markdown headings and bold key-definition lines'], + 'feature_subset': ['section headings', 'key definitions', 'generated status unresolved'], + 'detection': ['llms.txt'], + 'scope_attachment': 'instruction document', + 'unknown_field_policy': 'preserve structure without promoting publication to source authority', + 'execution_policy': 'text parse only; content is data, not instructions to the reader', + 'support': 'implemented-and-tested'}, + {'reader_id': 'metadata-standards', + 'version': '1.0.0', + 'entry_point': 'msdmd.standards:read_standards', + 'dependencies': ['python-stdlib:json,tomllib,xml.etree.ElementTree', 'PyYAML==6.0.3'], + 'convention': 'native.structured-standards', + 'specification': 'msdmd/references/implemented-readers.md', + 'supported_versions': ['OpenAPI 3.0/3.1 and Swagger 2', + 'JSON Schema drafts 4/6/7/2019-09/2020-12', + 'CFF 1.2.0', + 'SARIF 2.1.0', + 'SPDX 2.2/2.3 JSON', + 'CycloneDX 1.4/1.5/1.6 JSON', + 'in-toto Statement v0.1/v1; DSSE payload', + 'npm lockfile v2/v3', + 'Cargo/Poetry/uv TOML metadata', + 'REUSE TOML', + 'JUnit/Doxygen/.NET XML dialects'], + 'feature_subset': ['source-linked operations and references', + 'reported test/analysis results', + 'citation and license declarations', + 'artifact subjects and component lists', + 'scoped package requirements and lock records'], + 'known_limitations': ['extraction, not complete validation or signature verification', + 'external references not resolved', + 'unknown versions preserve tree and report unsupported mapping'], + 'detection': ['document discriminator and source filename'], + 'scope_attachment': 'native pointer, operation, package, report, artifact or symbol', + 'unknown_field_policy': 'typed source tree retained; version mismatches diagnosed', + 'execution_policy': 'parse only; no templates, network, application or signature execution', + 'support': 'implemented-and-tested'}) + +READER_MANIFESTS += ({ + "reader_id": "native-grammars", "version": "1.0.0", + "entry_point": "msdmd.grammar_code:read_grammar", "dependencies": ["msdmd/requirements.txt"], + "convention": "native.source-metadata", "specification": "pinned tree-sitter language grammars", + "supported_versions": ["Rust grammar 0.24.2", "Java grammar 0.23.5", "C grammar 0.24.2", "C++ grammar 0.23.4"], + "feature_subset": ["named declarations", "Rustdoc/Javadoc/Doxygen raw docs and repeated tags", "imports", "attributes", "SPDX headers"], + "detection": [".rs", ".java", ".c", ".cc", ".cpp", ".cxx", ".hpp", ".hxx", ".h (ambiguous)"], + "scope_attachment": "lexical symbols and adjacent documentation", + "unknown_field_policy": "preserve raw documentation and syntax; diagnose macro/conditional context", + "execution_policy": "parse only; no compiler, macro expansion or annotation processing", "support": "partial", +},) +_MANIFESTS = {item["reader_id"]: item for item in READER_MANIFESTS} +_PYTHON_SUFFIXES = {".py", ".pyi", ".pyw"} +_MARKDOWN_SUFFIXES = {".md", ".mdx"} +_YAML_SUFFIXES = {".yml", ".yaml"} +_TYPESCRIPT_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts"} +_SHELL_SUFFIXES = {".sh", ".bash", ".zsh", ".fish"} +_SYSTEMD_SUFFIXES = {".service", ".socket", ".timer", ".path", ".target"} +_GENERATED_MARKER = "Generated by tools/build_codex_plugin_skills.py; do not edit." +_SENSITIVE_KEY_RE = re.compile(r"(?:^|[_-])(?:secret|token|password|passwd|api[_-]?key|private[_-]?key|credential|client[_-]?secret|access[_-]?key)(?:$|[_-])", re.IGNORECASE) + + +def _digest(value: bytes | str) -> str: + data = value if isinstance(value, bytes) else value.encode("utf-8") + return hashlib.sha256(data).hexdigest() + + +def _json_safe(value: Any) -> Any: + """Preserve non-JSON native scalar types with explicit tagged values.""" + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else {"$type": "nonfinite-float", "value": repr(value)} + if isinstance(value, (dt.datetime, dt.date, dt.time)): + return {"$type": f"toml-{type(value).__name__}", "value": value.isoformat()} + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return {"$type": f"unsupported:{type(value).__name__}", "value": repr(value)} + + +def _redact_sensitive(value: Any, pointer: str = "") -> tuple[Any, list[str]]: + """Redact values owned by plainly secret-bearing field names.""" + redacted: list[str] = [] + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, item in value.items(): + child_pointer = f"{pointer}/{key}" + if _SENSITIVE_KEY_RE.search(str(key)): + result[str(key)] = {"$redacted": True, "reason": "sensitive-field-name"} + redacted.append(child_pointer) + else: + result[str(key)], child_redacted = _redact_sensitive(item, child_pointer) + redacted.extend(child_redacted) + return result, redacted + if isinstance(value, list): + result_list: list[Any] = [] + for index, item in enumerate(value): + child, child_redacted = _redact_sensitive(item, f"{pointer}/{index}") + result_list.append(child) + redacted.extend(child_redacted) + return result_list, redacted + if isinstance(value, str): + sanitized = re.sub(r"(\b[a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]+@", r"\1[redacted]@", value) + if sanitized != value: + return sanitized, [pointer] + return value, redacted + + +def _redaction_diagnostic(context: dict[str, Any], reader_id: str, pointers: list[str]) -> dict[str, Any] | None: + if not pointers: + return None + return _diagnostic( + context, + reader_id=reader_id, + code="sensitive_fields_redacted", + message=f"withheld {len(pointers)} sensitive field value(s)", + status="redacted", + severity="warning", + ) + + +def _quoted(value: str, *, path: bool = False) -> str: + return quote(value, safe="/" if path else "") + + +def _source(context: dict[str, Any], location: dict[str, Any]) -> dict[str, Any]: + return { + "repository": context["repo"], + "revision": context["revision"], + "file": context["file"], + "content_sha256": context["content_sha256"], + "location": location, + } + + +def _subject_address(context: dict[str, Any], scope: str, identity: str) -> str: + return ( + f"msdmd://{_quoted(context['repo'])}@{_quoted(context['revision'])}/{_quoted(context['file'], path=True)}" + f"#subject/{_quoted(scope)}/{_quoted(identity)}" + ) + + +def _fact( + context: dict[str, Any], + *, + reader_id: str, + kind: str, + scope: str, + identity: str, + location: dict[str, Any], + value: Any, + native_id: str | None = None, + convention: str | None = None, + standing: str = "syntactically-observed", + projection: dict[str, Any] | None = None, +) -> dict[str, Any]: + subject = _subject_address(context, scope, identity) + pointer = location.get("pointer") or f"lines:{location.get('start_line', '?')}-{location.get('end_line', '?')}" + native_component = f"/{_quoted(native_id)}" if native_id is not None else "" + address = f"{subject}/fact/{_quoted(kind)}/{_quoted(str(pointer))}{native_component}" + result = { + "address": address, + "origin": "native", + "kind": kind, + "source": _source(context, location), + "subject": { + "address": subject, + "scope": scope, + "identity": identity, + }, + "convention": { + "namespace": convention or _MANIFESTS[reader_id]["convention"], + "version": context.get("convention_version", "hmmm"), + "dialect": context.get("dialect", "hmmm"), + }, + "native": {"id": native_id, "value": _json_safe(value)}, + "standing": standing, + "extraction": { + "reader_id": reader_id, + "reader_version": _MANIFESTS[reader_id]["version"], + "support": _MANIFESTS[reader_id]["support"], + "configuration_sha256": context["configuration_sha256"], + }, + } + if projection is not None: + result["projection"] = projection + return result + + +def _diagnostic( + context: dict[str, Any], + *, + reader_id: str, + code: str, + message: str, + status: str, + severity: str = "warning", + location: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "code": code, + "severity": severity, + "status": status, + "message": message, + "source": _source(context, location or {"pointer": "document"}), + "reader_id": reader_id, + } + + +def _edge(source: str, target: str, kind: str, *, standing: str = "syntactically-observed") -> dict[str, Any]: + return { + "from": source, + "to": target, + "kind": kind, + "standing": standing, + "target_resolution": "external-or-unresolved", + } + + +def _read_ratios(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "msdmd-ratios" + marker = marker_for(path) + if marker is None: + return [], [], [] + context = dict(context, convention_version="named ratios line v1", dialect=f"line-comment:{marker}") + try: + if path.suffix.lower() in _PYTHON_SUFFIXES: + encoding, _ = tokenize.detect_encoding(io.BytesIO(data).readline) + source = data.decode(encoding).replace("\r\n", "\n").replace("\r", "\n") + comments = [""] * len(source.split("\n")) + for token in tokenize.generate_tokens(io.StringIO(source).readline): + if token.type == tokenize.COMMENT: + comments[token.start[0] - 1] = token.string + text = "\n".join(comments) + else: + source = data.decode("utf-8") + text = context.get("comment_source", source) + except (SyntaxError, UnicodeDecodeError, tokenize.TokenError, IndentationError) as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_ratios_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + diagnostics: list[dict] = [] + occurrence: Counter[str] = Counter() + for line_number, line in enumerate(text.splitlines(), start=1): + for entry in parse_ratios(line, marker): + ratio_id = str(entry["id"]) + occurrence[ratio_id] += 1 + identity = f"{ratio_id}[{occurrence[ratio_id]}]" + facts.append(_fact(context, reader_id=reader_id, kind="composition-ratio", scope="file", identity=identity, location={"start_line": line_number, "end_line": line_number}, value={"ratio": ratio_id, "value": str(entry["value"]), "occurrence": occurrence[ratio_id]}, native_id=ratio_id, convention="msdmd.ratios-line", standing="declared")) + if facts: + opening, closing = ratios_placement(source, marker) + if not opening or not closing: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="misplaced_ratios_boundary", message=f"opening={opening} closing={closing}", status="invalid", severity="error")) + return facts, [], diagnostics + + +def _read_json(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "json-stdlib" + context = dict(context, convention_version="RFC 8259", dialect="json") + try: + value = parse_json(data) + except (UnicodeDecodeError, ValueError) as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="invalid_json", message=str(exc), status="invalid", severity="error")] + value, redacted = _redact_sensitive(value) + if path.name == "package.json": + convention = "npm.package-json" + elif path.name.endswith(".schema.json") or (isinstance(value, dict) and is_json_schema_uri(value.get("$schema"))): + convention = "json.schema" + context["convention_version"] = str(value.get("$schema", "hmmm")) if isinstance(value, dict) else "hmmm" + elif isinstance(value, dict) and "openapi" in value: + convention = "openapi.document" + context["convention_version"] = str(value["openapi"]) + elif context["file"] == "skills.json": + convention = "skill-lib.skills-index" + elif context["file"] == ".codex-plugin/plugin.json": + convention = "codex.plugin-manifest" + elif context["file"] == ".claude-plugin/marketplace.json": + convention = "claude.plugin-marketplace" + else: + convention = "json.document" + facts = [_fact(context, reader_id=reader_id, kind="structured-document", scope="document", identity=context["file"], location={"pointer": ""}, value=value, convention=convention)] + edges: list[dict] = [] + if path.name == "package.json" and isinstance(value, dict): + subject = _subject_address(context, "document", context["file"]) + for field, kind in (("dependencies", "runtime"), ("devDependencies", "development"), ("peerDependencies", "peer"), ("optionalDependencies", "optional")): + dependencies = value.get(field, {}) + if isinstance(dependencies, dict): + for name, constraint in dependencies.items(): + facts.append(_fact(context, reader_id=reader_id, kind="dependency", scope="package", identity=str(value.get("name", context["file"])), location={"pointer": f"/{field}/" + str(name).replace("~", "~0").replace("/", "~1")}, value={"name": name, "constraint": constraint, "class": kind}, native_id=name, convention=convention, projection={"mapping_version": "npm-dependencies@1", "relation": "depends_on", "loss": "none"})) + edges.append(_edge(subject, f"npm-package:{name}", f"depends_on:{kind}")) + diagnostic = _redaction_diagnostic(context, reader_id, redacted) + return facts, edges, [diagnostic] if diagnostic else [] + + +def _read_toml(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "toml-stdlib" + context = dict(context, convention_version="TOML 1.0.0", dialect="toml") + try: + value = tomllib.loads(data.decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="invalid_toml", message=str(exc), status="invalid", severity="error")] + value, redacted = _redact_sensitive(value) + convention = "python.pyproject" if path.name == "pyproject.toml" else "toml.document" + facts = [_fact(context, reader_id=reader_id, kind="structured-document", scope="document", identity=context["file"], location={"pointer": ""}, value=value, convention=convention)] + edges: list[dict] = [] + if path.name == "pyproject.toml": + project = value.get("project", {}) if isinstance(value, dict) else {} + package = str(project.get("name", context["file"])) if isinstance(project, dict) else context["file"] + subject = _subject_address(context, "package", package) + groups: list[tuple[str, Any, str]] = [ + ("/project/dependencies", project.get("dependencies", []) if isinstance(project, dict) else [], "runtime"), + ("/build-system/requires", value.get("build-system", {}).get("requires", []) if isinstance(value.get("build-system", {}), dict) else [], "build"), + ] + optional = project.get("optional-dependencies", {}) if isinstance(project, dict) else {} + if isinstance(optional, dict): + groups.extend((f"/project/optional-dependencies/{group}", deps, f"optional:{group}") for group, deps in optional.items()) + for pointer, dependencies, kind in groups: + if not isinstance(dependencies, list): + continue + for index, dependency in enumerate(dependencies): + if not isinstance(dependency, str): + continue + name = re.split(r"[<>=!~;\[\s]", dependency, maxsplit=1)[0] + facts.append(_fact(context, reader_id=reader_id, kind="dependency", scope="package", identity=package, location={"pointer": f"{pointer}/{index}"}, value={"requirement": dependency, "name": name, "class": kind}, native_id=name, convention=convention, projection={"mapping_version": "pyproject-dependencies@1", "relation": "depends_on", "loss": "marker retained in requirement"})) + edges.append(_edge(subject, f"python-package:{name}", f"depends_on:{kind}")) + diagnostic = _redaction_diagnostic(context, reader_id, redacted) + return facts, edges, [diagnostic] if diagnostic else [] + + +def _read_markdown(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + reader_id = "markdown-frontmatter" + text = data.decode("utf-8-sig").replace("\r\n", "\n") + if not text.startswith("---\n"): + return [], [], [] + match = re.search(r"^---[ \t]*$", text[4:], re.M) + if match is None: + raise ValueError("unterminated YAML frontmatter") + raw = text[4:4 + match.start()] + value = parse_yaml(raw.encode("utf-8")) + value, redacted = _redact_sensitive(value) + context = dict(context, convention_version="YAML 1.2 core", dialect="yaml-frontmatter") + facts = [_fact(context, reader_id=reader_id, kind="frontmatter", scope="document", identity=context["file"], + location={"start_line": 1, "end_line": raw.count("\n") + 2}, + value={"fields": value, "raw_source": "source-reference", "generated": _GENERATED_MARKER in text}, + convention="markdown.yaml-frontmatter", standing="derived" if _GENERATED_MARKER in text else "declared")] + diagnostic = _redaction_diagnostic(context, reader_id, redacted) + return facts, [], [diagnostic] if diagnostic else [] + + +def _read_yaml(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + reader_id = "yaml-core" + value, redacted = _redact_sensitive(parse_yaml(data)) + context = dict(context, convention_version="YAML 1.2 core", dialect="yaml-core-string-keys") + fact = _fact(context, reader_id=reader_id, kind="structured-document", scope="document", identity=context["file"], + location={"pointer": ""}, value=value, convention="yaml.document") + diagnostic = _redaction_diagnostic(context, reader_id, redacted) + return [fact], [], [diagnostic] if diagnostic else [] + + +def _read_codeowners(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "github-codeowners" + context = dict(context, convention_version="GitHub current documented grammar", dialect="github") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + diagnostics: list[dict] = [] + for line_number, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + try: + parts = shlex.split(raw, comments=True, posix=True) + except ValueError as exc: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="invalid_codeowners_rule", message=str(exc), status="invalid", severity="error", location={"start_line": line_number, "end_line": line_number})) + continue + if not parts: + continue + if parts[0].startswith("!") or "[" in parts[0] or "\\#" in raw: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="unsupported_codeowners_pattern", + message="GitHub CODEOWNERS does not support this ignore-pattern feature", status="unsupported")) + facts.append(_fact(context, reader_id=reader_id, kind="review-ownership", scope="path-rule", identity=f"rule:{len(facts)}", location={"start_line": line_number, "end_line": line_number}, value={"pattern": parts[0], "owners": parts[1:], "precedence": line_number, "authority": "review-assignment-only", "provider_source": "active" if context.get("codeowners_source") == context["file"] else "shadowed"}, native_id=parts[0], convention="github.codeowners", projection={"mapping_version": "github-codeowners@1", "canonical_field": "review_owners", "loss": "does not imply operational ownership or permissions"})) + return facts, [], diagnostics + + +def _read_shell(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "shell-static" + context = dict(context, convention_version="source declaration v1", dialect=path.suffix.lstrip(".") or "shebang") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + lines = text.splitlines() + if lines and lines[0].startswith("#!"): + facts.append(_fact(context, reader_id=reader_id, kind="interpreter", scope="file", identity=context["file"], location={"start_line": 1, "end_line": 1}, value={"shebang": lines[0][2:].strip()}, native_id="shebang", convention="posix.shebang")) + for number, line in enumerate(lines, start=1): + match = re.match(r"^\s*#\s*shellcheck\s+(?P.+?)\s*$", line, re.IGNORECASE) + if match: + facts.append(_fact(context, reader_id=reader_id, kind="tool-directive", scope="file", identity=context["file"], location={"start_line": number, "end_line": number}, value={"tool": "shellcheck", "directive": match.group("body")}, native_id=f"shellcheck:{number}", convention="shellcheck.directive")) + return facts, [], [] + + +def _redact_systemd_environment(value: str) -> tuple[str, bool, bool]: + """Redact every assignment in the supported, non-escaped word subset. + + Unknown quoting/escaping withholds the entire value rather than + applying shell rules to systemd syntax or publishing possible secrets. + """ + parts: list[str] = [] + position = 0 + redacted = False + while position < len(value): + start = position + while position < len(value) and value[position].isspace(): + position += 1 + parts.append(value[start:position]) + if position == len(value): + break + start = position + quote_char = value[position] if value[position] in (chr(34), chr(39)) else '' + if quote_char: + end = value.find(quote_char, position + 1) + if end < 0 or (end + 1 < len(value) and not value[end + 1].isspace()): + return '', True, True + word = value[position + 1:end] + position = end + 1 + else: + while position < len(value) and not value[position].isspace(): + position += 1 + word = value[start:position] + if chr(34) in word or chr(39) in word: + return '', True, True + if chr(92) in word: + return '', True, True + name, separator, assigned = word.partition('=') + if not separator or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name): + return '', True, True + if _SENSITIVE_KEY_RE.search(name): + parts.append(quote_char + name + '=' + quote_char) + redacted = True + else: + parts.append(value[start:position]) + return ''.join(parts), redacted, False + + +def _read_systemd(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "systemd-unit" + context = dict(context, convention_version="systemd.syntax current documented grammar", dialect=path.suffix.lstrip(".")) + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + diagnostics: list[dict] = [] + section = "hmmm" + logical = "" + logical_start = 1 + occurrence: dict[tuple[str, str], int] = {} + for number, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if not stripped or stripped.startswith(("#", ";")): + continue + if not logical: + logical_start = number + if stripped.endswith(chr(92)): + logical += stripped[:-1] + ' ' + continue + logical += stripped + line = logical + logical = "" + if line.startswith("[") and line.endswith("]"): + section = line[1:-1] + continue + if "=" not in line: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="invalid_systemd_directive", message="expected key=value", status="invalid", severity="error", location={"start_line": logical_start, "end_line": number})) + continue + key, value = (part.strip() for part in line.split("=", 1)) + published_value = value + if key.lower() == 'environment': + published_value, redacted, unsupported = _redact_systemd_environment(value) + location = {'start_line': logical_start, 'end_line': number} + if redacted: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code='sensitive_fields_redacted', + message='withheld potentially sensitive systemd Environment values', status='redacted', location=location)) + if unsupported: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code='unsupported_systemd_environment_syntax', + message='Environment quoting or escaping is outside the supported subset; value withheld', status='unsupported', location=location)) + identity_key = (section, key) + occurrence[identity_key] = occurrence.get(identity_key, 0) + 1 + identity = f"{section}.{key}[{occurrence[identity_key]}]" + facts.append(_fact(context, reader_id=reader_id, kind="unit-directive", scope="configuration", identity=identity, location={"start_line": logical_start, "end_line": number}, value={"section": section, "key": key, "value": published_value, "occurrence": occurrence[identity_key]}, native_id=key, convention="systemd.unit")) + if logical: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="truncated_systemd_continuation", message="file ends during a continuation", status="invalid", severity="error")) + return facts, [], diagnostics + + +def _read_gitignore(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "gitignore-lines" + context = dict(context, convention_version="gitignore current documented grammar", dialect="gitignore") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + order = 0 + for line_number, raw in enumerate(text.splitlines(), start=1): + # Git ignores unescaped trailing spaces. Count backslash parity so + # an escaped space survives, including before later unescaped spaces. + while raw.endswith(' '): + prefix = raw[:-1] + if (len(prefix) - len(prefix.rstrip('\\'))) % 2: + break + raw = prefix + if not raw or raw.startswith("#"): + continue + order += 1 + negated = raw.startswith("!") and not raw.startswith("\\!") + pattern = raw[1:] if negated else raw + facts.append(_fact(context, reader_id=reader_id, kind="ignore-rule", scope="path-rule", identity=f"rule:{order}", location={"start_line": line_number, "end_line": line_number}, value={"pattern": pattern, "negated": negated, "anchored": pattern.startswith("/"), "directory_only": pattern.endswith("/"), "order": order}, native_id=pattern, convention="git.ignore", standing="declared")) + return facts, [], [] + + +_REQUIREMENT_NAME_RE = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9._-]*)") + + +def _read_requirements(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "python-requirements" + context = dict(context, convention_version="PyPA requirements file subset v1", dialect="requirements") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + edges: list[dict] = [] + diagnostics: list[dict] = [] + subject = _subject_address(context, "requirements-document", context["file"]) + order = 0 + for line_number, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + order += 1 + published = re.sub(r"(?<=://)[^/@\s]+@", "@", stripped) + if published != stripped: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="sensitive_fields_redacted", message="withheld requirement URL credentials", status="redacted", severity="warning", location={"start_line": line_number, "end_line": line_number})) + if published.startswith("-"): + if published.startswith(('-r', '-c')) or re.match(r'^--(?:requirement|constraint)(?:[=\s]|$)', published): + diagnostics.append(_diagnostic(context, reader_id=reader_id, code='unresolved_requirement_include', + message='included requirements/constraints are declared but not resolved', status='dynamic-unresolved', + location={'start_line': line_number, 'end_line': line_number})) + facts.append(_fact(context, reader_id=reader_id, kind="requirements-option", scope="requirements-document", identity=f"option:{order}", location={"start_line": line_number, "end_line": line_number}, value={"text": published, "order": order}, native_id=f"option:{order}", convention="python.requirements-file", standing="declared")) + continue + match = _REQUIREMENT_NAME_RE.match(published) + name = match.group("name") if match else "hmmm" + facts.append(_fact(context, reader_id=reader_id, kind="dependency", scope="requirements-document", identity=f"requirement:{order}", location={"start_line": line_number, "end_line": line_number}, value={"requirement": published, "name": name, "class": "runtime-or-unresolved", "order": order}, native_id=name if name != "hmmm" else None, convention="python.requirements-file", standing="declared", projection={"mapping_version": "requirements-lines@1", "relation": "depends_on", "loss": "resolution and environment markers not evaluated"})) + if name != "hmmm": + edges.append(_edge(subject, f"python-package:{name}", "depends_on:runtime-or-unresolved")) + return facts, edges, diagnostics + + +def _read_license(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "license-text" + context = dict(context, convention_version="detector v1", dialect="license-text") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + spdx_match = re.search(r"SPDX-License-Identifier:\s*(?P[^\r\n]+)", text) + if spdx_match: + expression = spdx_match.group("value").strip() + method = "declared-spdx-line" + elif text.startswith("Mozilla Public License Version 2.0"): + expression = "MPL-2.0" + method = "exact-known-header" + else: + expression = "hmmm" + method = "unrecognized-text" + fact = _fact(context, reader_id=reader_id, kind="license-declaration", scope="document", identity=context["file"], location={"pointer": "document"}, value={"expression": expression, "detection_method": method, "legal_interpretation": "not-performed", "text": "source-reference"}, native_id=expression if expression != "hmmm" else None, convention="license.text", standing="declared" if method == "declared-spdx-line" else "derived") + diagnostics = [] if expression != "hmmm" else [_diagnostic(context, reader_id=reader_id, code="unrecognized_license_text", message="license identifier remains hmmm", status="ambiguous", severity="warning")] + return [fact], [], diagnostics + + +def _read_svg(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = 'svg-metadata' + context = dict(context, convention_version='SVG document subset v1', dialect='xml') + try: + root = parse_xml(data) + except (UnicodeDecodeError, ValueError, ET.ParseError) as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code='invalid_svg_xml', + message=type(exc).__name__ + '; SVG rejected by bounded XML parser', status='invalid', severity='error')] + if root.tag.rsplit('}', 1)[-1] != 'svg': + return [], [], [_diagnostic(context, reader_id=reader_id, code='unexpected_svg_root', + message='expected svg root', status='invalid', severity='error')] + title = None + description = None + for child in root: + child_name = child.tag.rsplit('}', 1)[-1] + if child_name == 'title' and title is None: + title = ''.join(child.itertext()).strip() + elif child_name == 'desc' and description is None: + description = ''.join(child.itertext()).strip() + value = {'attributes': dict(root.attrib), 'title': title, 'description': description} + return [_fact(context, reader_id=reader_id, kind='document-metadata', scope='document', identity=context['file'], + location={'pointer': '/svg'}, value=value, convention='svg.document-metadata')], [], [] + + +_LLMS_HEADING_RE = re.compile(r"^(?P#{1,6})\s+(?P.+?)\s*$") +_LLMS_DEFINITION_RE = re.compile(r"^-\s+\*\*(?P<term>.+?)\*\*\s*=\s*(?P<definition>.+?)\s*$") + + +def _read_llms_text(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: + reader_id = "llms-text" + context = dict(context, convention_version="Markdown structural subset v1", dialect="llms.txt") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return [], [], [_diagnostic(context, reader_id=reader_id, code="undecodable_source", message=str(exc), status="invalid", severity="error")] + facts: list[dict] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + heading = _LLMS_HEADING_RE.match(line) + if heading: + title = heading.group("title") + facts.append(_fact(context, reader_id=reader_id, kind="instruction-section", scope="document-section", identity=f"heading:{line_number}", location={"start_line": line_number, "end_line": line_number}, value={"level": len(heading.group("marks")), "title": title, "generated": "hmmm"}, native_id=title, convention="llms.txt", standing="declared")) + continue + definition = _LLMS_DEFINITION_RE.match(line) + if definition: + term = definition.group("term") + facts.append(_fact(context, reader_id=reader_id, kind="key-definition", scope="instruction-definition", identity=term, location={"start_line": line_number, "end_line": line_number}, value={"term": term, "definition": definition.group("definition"), "generated": "hmmm"}, native_id=term, convention="llms.txt", standing="declared")) + return facts, [], [] + + +@lru_cache(maxsize=1) +def implementation_digest() -> str: + """Identity of all executable reader inputs, independent of inspected files.""" + names = ("readers.py", "native_code.py", "grammar_code.py", "formats.py", "standards.py", "module_projection.py", "typescript-reader.cjs", "requirements.txt", "package.json", "package-lock.json", "module-projection.schema.json", "python-module-reader.json", "parsers/universal.py") + digest = hashlib.sha256() + for name in names: + digest.update(name.encode()) + digest.update((Path(__file__).parent / name).read_bytes()) + return digest.hexdigest() + + +Reader = Callable[[Path, bytes, dict[str, Any]], tuple[list[dict], list[dict], list[dict]]] + + +def readers_for(path: Path, data: bytes | None = None) -> list[tuple[str, Reader]]: + """Return every compatible reader selected for ``path``.""" + suffix = path.suffix.lower() + relative = path.as_posix() + selected: list[tuple[str, Reader]] = [] + if path.name == "CODEOWNERS" and relative in {"CODEOWNERS", ".github/CODEOWNERS", "docs/CODEOWNERS"}: + selected.append(("github-codeowners", _read_codeowners)) + elif suffix in _PYTHON_SUFFIXES: + selected.append(("python-ast", _read_python)) + elif suffix in {".json", ".sarif", ".ipynb"}: + selected.append(("json-stdlib", _read_json)) + elif suffix == ".toml" or path.name in {"Cargo.lock", "poetry.lock", "uv.lock"}: + selected.append(("toml-stdlib", _read_toml)) + elif suffix in _MARKDOWN_SUFFIXES: + selected.append(("markdown-frontmatter", _read_markdown)) + elif suffix in _YAML_SUFFIXES or suffix == ".cff": + selected.append(("yaml-core", _read_yaml)) + elif suffix in _TYPESCRIPT_SUFFIXES: + selected.append(("typescript-compiler", _read_typescript)) + elif suffix in {".rs", ".java", ".c", ".cc", ".cpp", ".cxx", ".hpp", ".hxx", ".h"}: + selected.append(("native-grammars", read_grammar)) + elif suffix in _SHELL_SUFFIXES: + selected.append(("shell-static", _read_shell)) + elif suffix in _SYSTEMD_SUFFIXES: + selected.append(("systemd-unit", _read_systemd)) + elif path.name == ".gitignore": + selected.append(("gitignore-lines", _read_gitignore)) + elif (path.name.startswith("requirements") or path.name.startswith("constraints")) and suffix == ".txt": + selected.append(("python-requirements", _read_requirements)) + elif path.name in {"LICENSE", "LICENSE.txt", "COPYING", "COPYING.txt"} or (path.parent.name == "LICENSES" and suffix == ".txt"): + selected.append(("license-text", _read_license)) + elif suffix == ".svg": + selected.append(("svg-metadata", _read_svg)) + elif path.name == "llms.txt": + selected.append(("llms-text", _read_llms_text)) + elif not suffix and data is not None and data.startswith(b"#!"): + selected.append(("shell-static", _read_shell)) + if suffix in {".json", ".sarif", ".ipynb", ".yaml", ".yml", ".cff", ".toml", ".xml", ".spdx", ".license"} or path.name in {"Cargo.lock", "poetry.lock", "uv.lock"}: + selected.append(("metadata-standards", read_standards)) + if data is not None and b"ratios:" in data and marker_for(path) is not None: + selected.append(("msdmd-ratios", _read_ratios)) + return selected + + +def reader_for(path: Path, data: bytes | None = None) -> tuple[str, Reader] | None: + """Return the primary reader for compatibility; prefer :func:`readers_for`.""" + selected = readers_for(path, data) + return selected[0] if selected else None + + +def read_native(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[str], list[dict], list[dict], list[dict]]: + """Read one immutable source buffer with every applicable native reader.""" + reader_ids: list[str] = [] + facts: list[dict] = [] + edges: list[dict] = [] + diagnostics: list[dict] = [] + for reader_id, reader in readers_for(path, data): + if reader_id == "metadata-standards" and any(d.get("severity") == "error" for d in diagnostics): + continue + reader_ids.append(reader_id) + local_context = dict(context) + try: + if reader_id == "msdmd-ratios" and path.suffix.lower() in _TYPESCRIPT_SUFFIXES | {".rs", ".java", ".c", ".cc", ".cpp", ".cxx", ".hpp", ".hxx", ".h"}: + lines = [""] * len(data.decode("utf-8-sig").split("\n")) + for f in facts: + if f["kind"] == "comment" and f["extraction"]["reader_id"] in {"typescript-compiler", "native-grammars"}: + for offset, line in enumerate(f["native"]["value"]["text"].split("\n")): + index = f["source"]["location"]["start_line"] - 1 + offset + if index < len(lines): + lines[index] = line + local_context["comment_source"] = "\n".join(lines) + local_context["configuration_sha256"] = _digest(json.dumps({"reader_id": reader_id, "reader_version": _MANIFESTS[reader_id]["version"]}, sort_keys=True)) + reader_facts, reader_edges, reader_diagnostics = reader(path, data, local_context) + except (ImportError, FileNotFoundError) as exc: + reader_facts, reader_edges = [], [] + reader_diagnostics = [_diagnostic(local_context, reader_id=reader_id, + code="reader_dependency_unavailable", message=type(exc).__name__ + "; install declared reader dependencies", + status="unsupported")] + except Exception as exc: + # Malformed inputs do not stop independent files. Do not publish exception + # text: parser messages can contain source literals or credentials. + reader_facts, reader_edges = [], [] + reader_diagnostics = [_diagnostic(local_context, reader_id=reader_id, + code="native_reader_error", message=type(exc).__name__ + "; input rejected by " + reader_id, + status="invalid", severity="error")] + for fact in reader_facts: + fact["extraction"]["implementation_sha256"] = implementation_digest() + + facts.extend(reader_facts) + edges.extend(reader_edges) + diagnostics.extend(reader_diagnostics) + # The same complete owning tree is emitted once even when a standard adapter + # adds a more precise root classification than the generic syntax reader. + roots = [f for f in facts if f["extraction"]["reader_id"] == "metadata-standards" and f["source"]["location"] == {"pointer": ""}] + if roots: + facts = [f for f in facts if not (f["kind"] == "structured-document" and f["extraction"]["reader_id"] != "metadata-standards" + and any(f["native"]["value"] == root["native"]["value"] for root in roots))] + return reader_ids, facts, edges, diagnostics +# ratios: loc_comments=889:48 imports_exports=22:5 calls_definitions=267:28 diff --git a/.agents/skills/msdmd/references/implemented-readers.md b/.agents/skills/msdmd/references/implemented-readers.md new file mode 100644 index 00000000..bee2dae5 --- /dev/null +++ b/.agents/skills/msdmd/references/implemented-readers.md @@ -0,0 +1,79 @@ +# Integrated native reader support + +This matrix describes shipped extraction, not certification against entire +standards. `msdmd/readers.py` owns executable selection and machine manifests; +`tests/test_native_collection.py` and `tests/test_native_standards.py` exercise +these readers through the default schema-2 collector. No MSDMD redeclaration is +required. Pin dependencies from `msdmd/requirements.txt` and `package-lock.json`. + +| Owning convention | Implemented extraction | Explicit limit | +|---|---|---| +| Python AST/tokenize, PEP 257 | Qualified symbols, signatures, decorators, structurally attached comments/docstrings, imports, literal `__all__`, SPDX headers | Running Python grammar; no target import, call graph, dynamic exports or rename inference | +| ReST/Sphinx, Google and NumPy docstrings | Structured parameters/returns/raises and descriptive fields with raw text, using docstring-parser 0.18.0 | Parser-supported dialect fields; ambiguous/custom constructs retain source and diagnostics | +| JavaScript/TypeScript/JSDoc | TypeScript 5.8.3 AST, nested declarations, multiline signatures, static imports/exports, compiler-attached repeated documentation tags, actual comments | No target config, module resolution, decorators or runtime execution; full TSDoc validation not claimed | +| Rustdoc / Java Javadoc / C and C++ Doxygen source | Pinned tree-sitter grammar declarations, lexical scope, attached raw docs and repeated tags, source attributes/imports, SPDX | Grammar subsets, not compiler semantics. Macros/conditional compilation unresolved. `.h` is ambiguous, not guessed | +| JSON / TOML | Complete typed document tree; duplicate/nonfinite JSON rejection; TOML dates/times explicitly tagged | Syntax preservation is not domain-schema validation | +| YAML / Markdown frontmatter | Maintained PyYAML syntax parser, explicit YAML 1.2 core resolution, nested/flow structures, block strings, bounded acyclic aliases; typed scalar-key maps tagged for JSON | No custom tags, cycles, merge-key dialect or complex keys; errors remain visible | +| Python / npm / Cargo manifests | Package identity, dependency classes, Cargo conditional dependencies, raw owning trees | No installation, resolution or scripts; Python imports are not calls | +| npm lockfile 2/3; Cargo/Poetry/uv locks; pnpm document | Recorded package metadata; per-package rows for supported npm and TOML lock structures | Version/dialect fields retained; pnpm semantic graph and cross-lock resolution not claimed | +| GitHub CODEOWNERS | Ordered patterns, repeated owners, ownerless exemptions, authoritative-file precedence and shadowed rules | No path matching, team permission lookup or inferred operational ownership | +| SPDX / REUSE | Actual code headers, `.license` and SPDX tag-value lines, REUSE.toml annotations; SPDX 2.2/2.3 JSON document/packages/files/relationships | Tag-value is a line subset, not complete multiline SPDX; no license-compliance conclusion | +| CITATION.cff 1.2.0 | Nested citation metadata, contributors, identifiers and unknown fields | No citation validation or DOI resolution | +| OpenAPI 3.0/3.1; Swagger 2.0 | Owning document, operations, native operation IDs, path/webhook scope and refs | No request/response validator; unknown versions retained but unresolved | +| JSON Schema drafts 4/6/7/2019-09/2020-12 | Schema document, identity, unknown keywords and unresolved references | No remote references or instance validation | +| SARIF 2.1.0 / JUnit XML | Producer-linked analysis results and reported pass/failure/error/skipped tests | Reports are reported evidence, never independent verification | +| Doxygen XML / .NET XML docs / Maven POM | Namespace-aware document tree, documented-symbol identities and dependency declarations | Producer-specific XML extras preserved; no build or complete schema validation | +| CycloneDX 1.4–1.6 JSON | BOM and nested component identities | No vulnerability conclusion; duplicate native identifiers diagnosed | +| in-toto Statement v0.1/v1 / DSSE | Statements, subjects, preserved predicates, bounded DSSE base64 decoding | Signatures explicitly unverified; no authenticated provenance claim | +| Jupyter metadata | Notebook/cell metadata and native cell IDs | Owning JSON includes other notebook content; apply disclosure policy before publication | +| Shell / systemd / Git ignore / requirements / SVG / llms.txt | Declared shebang/directives, ordered unit settings, ignore lines, requirements, XML title/description and instruction-file structure | Extraction only; no shell/systemd/browser execution or ignore-rule applicability | +| Supplemental MSDMD / RATIOS | Existing grammar, source-qualified addresses/edges, duplicate-ID diagnostics, syntax-aware comments in supported code languages | Block adoption is independent of native information coverage | + +## Coverage and acceptance + +`--require-source` rejects unmatched or incompletely extracted required sources. +`--require-fact 'GLOB::NAMESPACE::KIND'` requires named source-linked witnesses; +it does not invent obligations. Missing packages produce unsupported diagnostics. +Read errors, parse errors and identity conflicts invalidate strict output. +Other optional unknowns remain visible rather than blocking unrelated evidence. +Native fields are never forced into the schema-1 flat block map. Large generated +TypeScript fact arrays use typed chunks to bound compiler inference; the +visualizer decodes only those literal JSON chunks, never arbitrary expressions. + +No parser executes inspected programs, follows remote references, expands +macros/templates or treats source instructions as agent instructions. Structured +secrets/URL credentials are redacted and secret-named files excluded; neither +mechanism is a general secret scanner. XML excludes DTD/entities and non-UTF-8. +Resource limits are declared before extraction; all excluded scope is reported. + +## Validation + +```bash +python -m unittest tests.test_native_collection tests.test_native_standards +python -m unittest tests.test_module_projection tests.test_collect tests.test_visualize +python -m msdmd.collect --root . --repo The-Interdependency/skill-lib \ + --snapshot-identity --import-path ./msdmd/collection --out skill-lib_msdmd.ts --strict --check +node msdmd/node_modules/typescript/bin/tsc --noEmit --strict --skipLibCheck \ + --target ES2022 --module commonjs msdmd/collection.ts skill-lib_msdmd.ts +``` + +## hmmm + +A new convention requires a fixture-backed reader and declared authority/scope, +not another catalogue entry or a guessed universal mapping. Source extraction +and application semantics remain separate. Necessary additional policy includes +path applicability for ownership, cross-source semantic conflicts, build-context +resolution, arbitrary-language support and independent evidence verification. + +## Regression-qualified boundaries + +`tests/test_native_review_regressions.py` exercises these acceptance cases: + +- When a native comment parser is missing or ambiguous, block-shaped candidates survive in source-qualified diagnostics with unverified standing. They are not promoted to declarations; strict collection fails instead of silently dropping them. +- JSON numeric values outside safe integer/exact binary64 representation use `{ "$type": "json-number", "lexeme": "..." }`, preserving the original numeric lexeme across TypeScript/JavaScript. Nonfinite overflow remains rejected. This is an explicit tagged representation, not a claim that JavaScript arithmetic is exact. +- Systemd continuation backslashes become spaces and intervening comments are ignored. Environment assignments are checked individually. Plain words and whole-word quotes are supported; unimplemented escaping/quoting withholds the entire value and reports partial scope. No systemd expansion is executed. +- Requirements include/constraint directives remain source-owned records and are explicitly unresolved; required-source coverage fails until their information obligation is satisfied. +- TypeScript local export lists retain public aliases, type-only flags and source-local declaration links. Compiler/build-context resolution remains out of scope. +- Maven XML recognition requires its POM namespace or an unnamespaced project with modelVersion 4.0.0. Other project XML stays generic XML. +- SVG uses the same bounded, UTF-8, DTD/entity-rejecting parser as other XML inputs. +- Explicit YAML `!!float 1` is supported and is a regression control, not an outstanding defect. diff --git a/.agents/skills/msdmd/references/metadata-conventions.md b/.agents/skills/msdmd/references/metadata-conventions.md new file mode 100644 index 00000000..9ebe5935 --- /dev/null +++ b/.agents/skills/msdmd/references/metadata-conventions.md @@ -0,0 +1,212 @@ +# Metadata conventions — discovery and extraction contract + +This reference extends `../SKILL.md`. Use it to inventory actual inputs, select +readers, and design tests. **The rows below specify what to inspect; they do not +claim that skill-lib ships readers for those conventions.** Resolve and record +an authoritative specification or owning implementation before interpreting a +particular dialect. A family name is not an implementation status. + +## Open-world coverage rule + +Any existing metadata convention encountered in the declared scan scope is an +eligible input. Include native and project-local conventions, custom tags, +extensions, older versions still present in inputs, and new formats not named +below. An unrecognized file remains in the discovery ledger; a recognized but +uninterpreted declaration retains its source reference and an unsupported or +opaque status. Do not claim that unknown syntax was fully detected or understood. + +"All conventions" means an open ingestion boundary with explicit accounting, +not a finite catalogue masquerading as universal parser coverage. Add a reader +contract when a new convention appears; do not require changes to its owner merely +to make the source fit MSDMD. Retired input syntax may be inspected as historical +data without being recommended for new authoring. + +## Convention catalogue + +### Shipped implementation and catalogue boundary + +The integrated schema-2 implementation and fixture-backed extraction subsets are +listed in [implemented-readers.md](implemented-readers.md). `readers.py` emits its +machine manifests in every collection. The Python attachment reader is shared +with the per-module projection; it is no longer disconnected from collection. + +The families below remain a discovery catalogue, not a blanket implementation +claim. An unlisted or unsupported format remains visible as `hmmm`, and required +coverage cannot pass without a capable reader and source-linked witnesses. + + +### Language, symbol, and documentation metadata + +| Family | Explicit conventions and sources to inspect | Preservation / interpretation boundary | +|---|---|---| +| Python | PEP 257 docstrings; reStructuredText/Sphinx fields; Google-style and NumPy-style sections; signatures, annotations, type comments, `.pyi`, `Annotated`, decorators, dataclass/field declarations, `__all__`, encoding cookies, inline script metadata. | Separate each docstring dialect and typing version. Preserve expressions without importing modules, evaluating annotations, invoking decorators, or resolving dynamic exports by execution. | +| JavaScript / TypeScript | JSDoc, TSDoc, TypeDoc extensions; `.d.ts`; signatures, generics, interfaces, visibility, import/export declarations, decorators, compiler directives. | JSDoc, TSDoc, and TypeDoc are not interchangeable tag grammars. Keep unknown/custom tags, inline links, overloads, and declared versus inferred types. | +| C / C++ / Objective-C | Doxygen and HeaderDoc comments; declarations, attributes, pragmas, preprocessor conditions, include edges, compiler annotations. | Preserve active/unknown configuration branches. A textual include is not a resolved build dependency under every target. | +| Java / JVM | Javadoc including version-specific comment forms; annotations; `package-info.java`, `module-info.java`, signatures and modifiers; Kotlin KDoc and annotations; Scala Scaladoc and annotations; Groovydoc. | Preserve dialect, target, retention/visibility declarations, generics, and inheritance links. Do not run annotation processors or compile project code during static collection. | +| .NET | C#/VB/F# XML documentation; attributes; nullable/type metadata; assembly and project metadata. | Keep XML structure and `cref`/inheritance references. Disable external entity expansion. Do not infer runtime attribute effects from spelling alone. | +| Rust | rustdoc comments and attributes; `cfg`/`cfg_attr`, `deprecated`, visibility, signatures, traits and bounds; doctest flags. | Preserve feature/target conditions and macro invocations. An example or doctest flag is not a passing execution result. | +| Go | Go doc comments; declaration comments; struct tags; build constraints; `go:generate`, `go:embed`, and other versioned directives. | Preserve directive scope and tag namespaces. Never execute generation directives during extraction. | +| Swift / Apple | Documentation comments, DocC markup, attributes, availability declarations, Swift package and plist metadata. | Keep platform/version availability and symbol attachment; do not load package scripts to collect values. | +| Ruby / PHP | RDoc, YARD and tags; Ruby declaration/mixin syntax; PHPDoc, attributes, signatures and namespace declarations. | Distinguish documentation type claims from language syntax and dynamic metaprogramming. | +| BEAM | Elixir `@moduledoc`, `@doc`, `@typedoc`, `@spec`, `@type`, behaviours and attributes; Erlang EDoc and versioned documentation attributes/specs. | Treat expressions and macros as source data; preserve module/function/type scope without starting a runtime. | +| Lisp / functional | Clojure metadata maps, `^` metadata, docstrings, arglists; Common Lisp documentation forms; Scheme/Racket documentation; Haskell Haddock and pragmas; OCaml odoc. | Do not evaluate forms. Resolve syntax and metadata semantics from the identified language/tool, not a shared punctuation shape. | +| Scientific / statistical | R roxygen2, DESCRIPTION and NAMESPACE; Julia docstrings/macros; MATLAB help comments; Fortran documentation comments and declarations. | Resolve ambiguous extensions explicitly. Preserve language-specific binding and generated-source provenance. | +| Shell / operations | Interpreter shebangs; shell documentation conventions, ShellCheck directives; PowerShell comment-based help, attributes and parameter declarations; Perl POD; Lua LDoc/annotation dialects. | A shebang selects declared interpreter intent, not evidence of executable mode or installed availability. Keep positional requirements and never source the file. | +| Other languages | Dart doc comments/annotations; Zig doc comments; SQL comments and DDL metadata; Ada/SPARK aspects; HDL attributes; Prolog documentation; any additional language present. | Register exact syntax, ownership and tests before interpreting. Unsupported grammar remains visible; no language is silently discarded for lacking an MSDMD comment marker. | +| Ordinary and project-local comments | TODO/FIXME/XXX/HACK/NOTE markers; region markers; generated-file notices; project-defined structured comments; embedded Markdown, JSON, YAML, XML, or TOML. | Record literal markers and local interpretation separately. A TODO is not automatically a defect, assigned issue, or behavioral obligation. Comments inside strings/examples need syntax-aware discrimination. | + +### Repository, packaging, build, and operational metadata + +| Family | Explicit conventions and sources to inspect | Preservation / interpretation boundary | +|---|---|---| +| Python packaging | `pyproject.toml`, inline script metadata, `setup.cfg`, statically recoverable `setup.py`, core METADATA/PKG-INFO, entry points, requirements/constraints and lock formats. | Preserve specification version and declared dynamic fields. Build requirements, runtime requirements, extras, groups, constraints, and resolved packages are different relationships. No build backend execution. | +| JavaScript packaging | `package.json`, npm/yarn/pnpm/bun lockfiles, workspace definitions, exports/imports maps and engine constraints. | Keep peer/optional/dev/runtime dependency classes, environment conditions and resolution scope distinct. Scripts are declarations, not commands to run. | +| Other package ecosystems | Cargo manifests/locks; `go.mod`, `go.sum`, `go.work`; Maven POM, Gradle files/catalogues; NuGet/MSBuild/project files; Composer; Gemfile/gemspec/locks; Mix/rebar; Swift Package/CocoaPods; Dart pub; Julia Project/Manifest; R DESCRIPTION; OS package specifications. | Preserve package/workspace/target scope, constraints and resolutions. Executable configuration stays unresolved unless supplied as a separately authorized, provenance-bearing result. | +| Build systems | CMake, Meson, Make, Bazel BUILD/MODULE/WORKSPACE, Ninja, Ant, SBT, compiler databases and toolchain files. | Static recovery is not complete build evaluation. Preserve conditional targets, generators, configurations, outputs and unknown expressions. | +| Ownership and governance | Provider-specific CODEOWNERS; OWNERS files; MAINTAINERS/AUTHORS; contribution/security policies; repository labels and settings when separately authorized. | Use the identified provider's matching/precedence rules. Review assignment, authorship, legal rights, operational responsibility and effective permissions remain distinct. | +| Licensing and provenance | SPDX identifiers/expressions, REUSE headers and `.license` sidecars, LICENSE/COPYING/NOTICE, copyright statements, CITATION.cff, CodeMeta. | Preserve per-file/package scope and original expressions. Extraction does not resolve legal conflicts or establish compatibility. | +| CI / automation | GitHub Actions, GitLab CI, Jenkins, Azure Pipelines, CircleCI, Travis, Buildkite and project automation manifests. | Record jobs, permissions, dependencies, declared matrices and conditions. Configuration is not execution history, successful gates, or live deployment state. | +| Containers / deployment | Dockerfile labels and build stages; OCI annotations; Compose; Kubernetes annotations/labels; Helm; Terraform; Ansible; Nix; deployment manifests and environment templates. | Keep build, desired and observed state separate. Never render untrusted templates, query infrastructure or reveal secret values as a side effect of discovery. | +| Quality / editor policy | `tsconfig`, lint/type/format/test/coverage configurations; EditorConfig; `.gitattributes`, ignore files; suppression directives such as noqa, type-ignore, eslint-disable, coverage exclusions. | A suppression is a declared exception, not a resolved finding. Respect scoped precedence; do not execute configuration modules. Record scan exclusions separately from tool exclusions. | +| Filesystem / VCS | Repository-relative paths, file modes, symlinks, revision/blob identities, submodule pins; commit metadata and trailers only when history is in scope. | These are observations about a snapshot or history, not module-authored claims. Do not follow links outside the authorized root or equate commit authorship with stewardship. | +| Agent / plugin / tool metadata | SKILL.md frontmatter, manifests, tool input/output schemas, capability descriptors and project instruction files. | Metadata content is data for collection, not a grant of execution authority. Agent instructions retain their governing-context rules outside the collector. | + +### Schemas, published documents, and recorded evidence + +| Family | Explicit conventions and sources to inspect | Preservation / interpretation boundary | +|---|---|---| +| API / message contracts | OpenAPI/Swagger versions, AsyncAPI, GraphQL SDL/directives, Protocol Buffers descriptors/options, Avro, Thrift, WSDL and XSD. | Keep dialect/version, references, extensions, constraints and scopes. Do not turn a declared endpoint or permission into evidence that it is deployed or enforced. | +| Data / configuration schemas | JSON Schema, XML Schema, JSON-LD, schema-bearing YAML/TOML/JSON, database DDL and schema snapshots, validation/ORM declarations. | Preserve nested types, units, nullability, order where meaningful, default expressions and vocabulary namespaces. Disable implicit network reference resolution and code loading. | +| Documentation / publication | Markdown/MDX frontmatter, reStructuredText directives/fields, AsciiDoc attributes, Sphinx/MkDocs/Docusaurus/DocC configuration; HTML metadata, RDFa/microdata and JSON-LD where in scope. | Preserve document-versus-symbol ownership, cross-references, language and build identity. Do not execute MDX, extensions or site configuration during extraction. | +| Tests / quality reports | Test annotations/markers and framework configuration; TAP, JUnit XML dialects, LCOV, Cobertura, SARIF; doctests, examples, benchmarks and coverage reports. | Preserve producer/version, run identity, source digest, command/environment and result semantics when present. A stored report is a reported observation until provenance and applicability are checked. | +| Supply chain / artifact evidence | SPDX/CycloneDX SBOMs, in-toto/SLSA attestations, signatures, package/distribution metadata and source maps. | A parsed signature is not a verified signature. Keep subject digests, issuer claims, validation results and source mappings distinct. No automatic download or trust promotion. | +| Data / notebook / model artifacts | Jupyter cell/notebook metadata, Arrow/Parquet schema metadata, HDF5 attributes, model cards/dataset cards and model-container metadata when explicitly in scope. | Use format-specific bounded readers. Do not execute notebooks, unpickle objects, load arbitrary model code, or expose sensitive datasets merely to inspect metadata. | +| Binary / archive metadata | ELF/PE/Mach-O, DWARF/PDB, JAR/class metadata, package archives and other artifact formats when explicitly in scope. | Optional isolated, size-bounded readers with exact tool identity; no executable loading or unsafe archive extraction. Unsupported binaries remain in the inventory rather than silently disappearing. | +| Unlisted convention | Any additional established or project-local convention found in the authorized scope. | Preserve source and uncertainty, resolve the owner/specification, register a reader and fixtures, then promote only the tested subset to supported. | + +## Reader contract + +Each actual reader must publish a machine-readable manifest containing: + +- `reader_id`, version or implementation digest, entry point, dependencies; +- convention namespace, specification/owning implementation reference and supported + versions, dialects, feature subset and known limitations; +- detection rules and ambiguity handling, plus configuration precedence; +- subject/scope attachment and native precedence rules; +- typed field mappings, relation meanings, raw-source preservation and unknown-field policy; +- invalid-input behavior, resource/disclosure limits and execution/network policy; +- positive, negative and adversarial fixtures with replayable results. + +Mark support as `implemented-and-tested`, `partial`, `specified-only`, or +`unsupported`, qualified by exact version and feature subset. An operational run +also reports `not-applicable`, `ambiguous`, `invalid`, `unreadable`, `excluded`, +`dynamic-unresolved` or other precise diagnostics as needed. These are different +axes: a supported reader can encounter invalid input; a specified reader is not +an available parser. Unmeasured usage maturity is not test evidence. + +Readers consume immutable bytes plus explicit parsing context and return facts, +source references and diagnostics. Do not impose a fictitious callable signature +on helpers that have not been implemented. Keep discovery, extraction, semantic +mapping, policy evaluation and execution evidence separate and composable. + +### Minimum unified native-capable fact shape + +The future unified collection must be able to express: + +| Information | Requirement | +|---|---| +| Source identity | Repository/source authority, exact revision and blob/content digest; dirty-worktree identity when applicable. | +| Location | Relative path and exact span or structural pointer; references to additional source inputs for multi-source derivations. | +| Subject | Qualified subject address and scope: repository, workspace, package, target, file, module, symbol, parameter, document, run, or artifact as applicable. | +| Convention | Namespace, detected/declared version, dialect and any unresolved identification. | +| Native declaration | Native ID/key/tag/path when present; typed value/tree or an access-controlled raw source reference. Preserve repeats and unknown fields. | +| Projection | Optional canonical field/relation, mapping version, input references and declared information loss. | +| Epistemic standing | Declared, syntactically observed, derived, or reported evidence; independently verified status requires a linked verification receipt. | +| Extraction | Reader identity/version, effective configuration digest, feature coverage and diagnostics. | +| Provenance / conflicts | All source witnesses, source-local precedence, disagreement groups and explicit resolution rationale when any. | + +Do not serialize typed unknown values into invented string conventions such as +`"None"` or silently flatten arrays/maps. Distinguish absent fields, explicit null, +empty values, unresolved expressions, redaction and inaccessible sources. Never +invent source locations, native IDs, relationships or verification receipts. + +## Cross-skill consumption + +- **doc-build:** native descriptions, examples, parameter/return docs and links + can satisfy their corresponding documentation obligations. A present comment + does not automatically satisfy every public-surface documentation requirement. +- **cap-build / deps-build:** signatures, exports, API schemas, imports and + manifests provide declared surfaces and scoped dependency relationships. Full + behavioral capability and complete dynamic call graphs need separate evidence. +- **owner-build:** consume native ownership patterns and declarations with their + exact scope and provider semantics; unresolved mappings remain visible. +- **test-build:** native contracts/assertions, test markers and reports can supply + inputs where a tested mapping exists. Never invent `proves` links from names + or treat an example/marker as an executed witness. +- **meta-module-build / risk-boundary-build:** consume existing manifests and + permission/data-effect declarations before requesting missing design intent. + Declared risk controls are not verified enforcement. +- **manifest / llms-build / typed-meta-frontend:** consume native project metadata, + instructions and schemas without creating a second owning copy. Preserve + read-only fields, edit destinations and unresolved values. +- **ratios:** keep shebang position and existing seals intact. Measured ratios + retain their measurement method; native metadata does not imply those ratios + have been recomputed or checked. + +## Acceptance matrix + +These are required executable fixture cases for reader implementation, **not +results already obtained by editing the skill**. + +| Case | Required result | +|---|---| +| Native-only repository, zero MSDMD blocks | Exact supported metadata extraction; no fabricated declarations or false missing-information findings. | +| Truly absent required information | A scoped missing result after eligible sources and capable readers have been evaluated. | +| Native plus supplemental MSDMD | Complementary information composes without forced redeclaration; all provenance survives. | +| Conflicting descriptions or types | Preserve both, identify their distinct authority/scope, and expose unresolved disagreement. | +| Same symbol name in two packages / overloads | Qualified identities remain distinct; no accidental merging. | +| Unknown/custom tags and nested extensions | Preserve raw/typed source and namespaces; mark semantic mapping unknown rather than dropping it. | +| Unsupported language, dialect or version | Visible unsupported/ambiguous scope; neither absent nor complete. | +| Invalid, truncated or undecodable source | Parse/read diagnostic with source identity; no fabricated recovery success. | +| Dynamic metadata or executable configuration | Expression and unresolved status retained; application code is not executed. | +| Ownership precedence and unmatched paths | Correct provider-specific matching; unrelated author/reviewer/permission concepts are not collapsed. | +| Conditional imports / optional dependencies / target features | Conditions, dependency class and resolution uncertainty survive. | +| Comments inside strings, examples and generated outputs | No false declaration attachment or self-generated evidence loops. | +| Shebang / encoding / newline / source positions | Original boundaries and byte/source references preserved correctly. | +| Excluded, symlinked, missing and binary files | Full scope accounting; no traversal escape or silent denominator reduction. | +| Embedded commands, YAML tags, XML entities, remote references | No implicit code execution, unsafe object construction or network access. | +| Secret-bearing fields and private reports | Correct access controls/redaction with visible withheld scope; no secret leakage. | +| Stale test report / wrong commit / unverified signature | Keep reported evidence separate; reject unsupported verification promotion. | +| Deterministic replay and source change | Stable semantic output for identical pinned inputs; stale outputs invalidated when owning inputs change. | +| Legacy consumer cannot represent a native fact | Explicit schema incompatibility or labeled loss; no silent flattening. | +| Claimed reader support without passing fixtures | Reject the support claim; retain specified-only/partial/unsupported standing. | + +## Primary references + +Resolve the exact relevant version at reader implementation time. The following +sources were consulted for this skill revision on 2026-09-15; they establish +specific convention families, not implementation coverage of the catalogue. + +- Python docstrings: https://peps.python.org/pep-0257/ +- Google/NumPy docstring handling: https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html +- Python project metadata, including version-sensitive dynamic rules: https://packaging.python.org/en/latest/specifications/pyproject-toml/ +- JSDoc: https://jsdoc.app/ +- TSDoc tag kinds: https://tsdoc.org/pages/spec/tag_kinds/ +- TypeScript JSDoc support: https://www.typescriptlang.org/docs/handbook/jsdoc-supported-types.html +- Doxygen comments: https://www.doxygen.nl/manual/docblocks.html +- Rust documentation tests: https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html +- Go doc comments: https://go.dev/doc/comment +- JDK 25 Javadoc: https://docs.oracle.com/en/java/javase/25/docs/specs/javadoc/doc-comment-spec.html +- C# XML documentation: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/xmldoc/ +- npm v11 package metadata: https://docs.npmjs.com/cli/v11/configuring-npm/package-json/ +- GitHub CODEOWNERS: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners +- OpenAPI specifications: https://spec.openapis.org/oas/latest.html +- JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-core +- SPDX specifications: https://spdx.dev/use/specifications/ +- SARIF 2.1.0: https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html + +## hmmm + +Each implemented reader still needs its own exact-version authority, supported +feature list and executable receipts. This catalogue deliberately leaves new +conventions admissible. An open door is not a claim that everyone has arrived. diff --git a/.agents/skills/msdmd/requirements.txt b/.agents/skills/msdmd/requirements.txt new file mode 100644 index 00000000..2beb831e --- /dev/null +++ b/.agents/skills/msdmd/requirements.txt @@ -0,0 +1,9 @@ +# Native reader runtime. Install before the full native-standards gate: +# python -m pip install -r msdmd/requirements.txt +PyYAML==6.0.3 +docstring-parser==0.18.0 +tree-sitter==0.26.0 +tree-sitter-rust==0.24.2 +tree-sitter-java==0.23.5 +tree-sitter-c==0.24.2 +tree-sitter-cpp==0.23.4 diff --git a/.agents/skills/msdmd/standards.py b/.agents/skills/msdmd/standards.py new file mode 100644 index 00000000..d021cbf7 --- /dev/null +++ b/.agents/skills/msdmd/standards.py @@ -0,0 +1,284 @@ +# ratios: loc_comments=254:13 imports_exports=8:2 calls_definitions=146:5 +"""Semantic projections for native metadata documents, preserving owning trees. + +Usage: registry invokes read_standards(path, bytes, context). The JSON/YAML/TOML +syntax readers retain unknown fields; this adapter adds source-linked operations, +schemas, dependency classes, ownership, reported tests and supply-chain subjects. +These are extraction subsets, not complete schema validators or attestation +verifiers. References are retained and diagnosed; no network fetch is performed. +""" +from __future__ import annotations + +import base64 +import re +import tomllib +from pathlib import Path +from typing import Any + +from msdmd.formats import is_json_schema_uri, parse_json, parse_yaml, parse_xml + + +def pointer(*parts: Any) -> str: + return ''.join('/' + str(p).replace('~', '~0').replace('/', '~1') for p in parts) + + +def read_standards(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list, list, list]: + from msdmd.readers import _fact, _diagnostic, _edge, _redact_sensitive + rid = 'metadata-standards' + facts: list = [] + edges: list = [] + diagnostics: list = [] + + def emit(convention: str, version: str, kind: str, loc: str, value: Any, + *, scope: str = 'document', identity: str | None = None, + native_id: str | None = None, standing: str = 'declared') -> dict: + value, redactions = _redact_sensitive(value) + if redactions: + diagnostics.append(_diagnostic(context, reader_id=rid, code='sensitive_fields_redacted', + message='sensitive values withheld from metadata projection', status='redacted')) + fact = _fact(dict(context, convention_version=version, dialect=convention), reader_id=rid, + convention=convention, kind=kind, scope=scope, identity=identity or context['file'], + location={'pointer': loc}, value=value, native_id=native_id, standing=standing) + facts.append(fact) + return fact + + def unsupported(code: str, message: str, *, status: str = 'unsupported') -> None: + diagnostics.append(_diagnostic(context, reader_id=rid, code=code, message=message, status=status)) + + suffix = path.suffix.lower() + if suffix == '.xml': + root = parse_xml(data) + local = lambda node: node.tag.rsplit('}', 1)[-1] + xml_redacted = False + + def scrub_xml(node: Any) -> None: + # Redact in the XML name domain, before names become generic tag/text keys. + # Scrub the parsed tree once so every later semantic projection is safe too. + nonlocal xml_redacted + if _redact_sensitive({local(node): ''})[1]: + node.text = '[redacted]' + node.attrib.clear() + node[:] = [] + xml_redacted = True + return + for name in list(node.attrib): + if _redact_sensitive({name.rsplit('}', 1)[-1]: ''})[1]: + node.set(name, '[redacted]') + xml_redacted = True + for child in node: + scrub_xml(child) + + scrub_xml(root) + if xml_redacted: + diagnostics.append(_diagnostic(context, reader_id=rid, code='sensitive_fields_redacted', + message='sensitive XML element or attribute content withheld before projection', status='redacted')) + def element_value(node: Any) -> dict: + return {'tag': node.tag, 'attributes': dict(node.attrib), 'text': node.text, + 'tail': node.tail, 'children': [element_value(child) for child in node]} + root_name = local(root) + conventions = {'testsuites': 'junit.xml', 'testsuite': 'junit.xml', + 'doxygen': 'doxygen.xml', 'doxygenindex': 'doxygen.xml', 'doc': 'dotnet.xml-doc', + 'project': 'maven.pom'} + convention = conventions.get(root_name, 'xml.document') + maven_namespace = 'http://maven.apache.org/POM/4.0.0' + maven_identity = root.tag == '{' + maven_namespace + '}project' or (root.tag == 'project' and + any(child.tag == 'modelVersion' and (child.text or '').strip() == '4.0.0' for child in root)) + if convention == 'maven.pom' and not maven_identity: + convention = 'xml.document' + emit(convention, root.get('version', 'unspecified-producer-dialect'), 'structured-document', '', element_value(root), + standing='reported-evidence' if convention == 'junit.xml' else 'declared') + def walk(node: Any, loc: str) -> None: + name = local(node) + if name == 'testcase' and convention == 'junit.xml': + children = [local(c) for c in node] + result = 'error' if 'error' in children else 'failure' if 'failure' in children else 'skipped' if 'skipped' in children else 'reported-pass' + emit(convention, 'unspecified-producer-dialect', 'test-result', loc, + {'attributes': dict(node.attrib), 'result': result, 'children': [element_value(c) for c in node]}, + scope='testcase', identity=loc, native_id=node.get('name'), standing='reported-evidence') + elif convention == 'doxygen.xml' and name in {'compounddef', 'memberdef'}: + emit(convention, root.get('version', 'hmmm'), 'documented-symbol', loc, element_value(node), + scope='symbol', identity=node.get('id', loc), native_id=node.get('id')) + elif convention == 'dotnet.xml-doc' and name == 'member': + emit(convention, 'xml-doc-member-v1', 'documented-symbol', loc, element_value(node), + scope='symbol', identity=node.get('name', loc), native_id=node.get('name')) + elif convention == 'maven.pom' and name == 'dependency': + fields = {local(c): c.text for c in node} + emit(convention, 'POM 4.0.0', 'dependency', loc, fields, scope='package', identity=fields.get('artifactId') or loc) + for index, child in enumerate(node): + walk(child, loc + pointer('children', index)) + walk(root, '') + return facts, edges, diagnostics + + if suffix in {'.spdx', '.license'}: + for index, line in enumerate(data.decode('utf-8-sig').splitlines()): + match = re.match(r'\s*(SPDX-[A-Za-z]+|SPDXVersion|PackageName|PackageVersion|Relationship|LicenseID|LicenseConcluded|LicenseDeclared):\s*(.*)', line) + if match: + fact = emit('spdx.tag-value', 'tag-value extraction', 'license-declaration', pointer('lines', index), + {'tag': match[1], 'value': match[2]}, native_id=match[1]) + fact['source']['location'] = {'start_line': index + 1, 'end_line': index + 1} + return facts, edges, diagnostics + + if suffix in {'.yaml', '.yml', '.cff'}: + value = parse_yaml(data) + elif suffix == '.toml' or path.name in {'Cargo.lock', 'poetry.lock', 'uv.lock'}: + value = tomllib.loads(data.decode('utf-8')) + else: + value = parse_json(data) + if not isinstance(value, dict): + return facts, edges, diagnostics + + # A structured-document fact is a lossless data view, not standard validation. + if 'openapi' in value or 'swagger' in value: + version = str(value.get('openapi', value.get('swagger'))) + emit('openapi.document', version, 'api-description', '', value) + if not (version == '2.0' or re.fullmatch(r'3\.[01]\.\d+', version)): + unsupported('unsupported_openapi_version', 'preserved document; operations require Swagger 2 or OpenAPI 3.0/3.1') + else: + for section in ('paths', 'webhooks'): + paths = value.get(section, {}) + if not isinstance(paths, dict): + raise ValueError('OpenAPI paths/webhooks must be a mapping') + for route, item in paths.items(): + if not isinstance(item, dict): + raise ValueError('OpenAPI path item must be a mapping') + for method, operation in item.items(): + if method not in {'get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'}: + continue + if not isinstance(operation, dict): + raise ValueError('OpenAPI operation must be a mapping') + emit('openapi.document', version, 'api-operation', pointer(section, route, method), operation, + scope='operation', identity=f'{section}:{method.upper()} {route}', native_id=operation.get('operationId')) + elif is_json_schema_uri(value.get('$schema')): + version = str(value['$schema']) + emit('json.schema', version, 'schema-description', '', value, scope='schema', native_id=value.get('$id', value.get('id'))) + if not re.fullmatch(r'https?://json-schema\.org/(?:draft/(?:2020-12|2019-09)|draft-0[467])/schema#?', version): + unsupported('unsupported_json_schema_draft', 'preserved schema; semantic extraction tested for drafts 4/6/7/2019-09/2020-12') + if 'cff-version' in value: + version = str(value['cff-version']) + emit('citation.cff', version, 'citation', '', value, scope='work', native_id=value.get('doi')) + if version != '1.2.0': + unsupported('unsupported_cff_version', 'citation fields preserved; tested CFF version is 1.2.0') + if 'runs' in value and ('sarif' in str(value.get('$schema', '')).lower() or value.get('version') == '2.1.0'): + version = str(value.get('version', 'hmmm')) + emit('sarif', version, 'analysis-report', '', value, scope='report', standing='reported-evidence') + if version != '2.1.0': + unsupported('unsupported_sarif_version', 'result mapping requires SARIF 2.1.0') + else: + for i, run in enumerate(value['runs']): + if not isinstance(run, dict): + raise ValueError('SARIF run must be a mapping') + for j, result in enumerate(run.get('results', [])): + emit('sarif', version, 'analysis-result', pointer('runs', i, 'results', j), + {'result': result, 'producer': run.get('tool'), 'invocations': run.get('invocations')}, + scope='analysis-result', identity=f'run:{i}/result:{j}', standing='reported-evidence') + if 'spdxVersion' in value: + version = str(value['spdxVersion']) + emit('spdx.json', version, 'sbom', '', value, scope='artifact', native_id=value.get('SPDXID'), standing='reported-evidence') + if version not in {'SPDX-2.2', 'SPDX-2.3'}: + unsupported('unsupported_spdx_version', 'package mapping tested for SPDX 2.2/2.3 JSON') + else: + for section in ('packages', 'files', 'relationships'): + for i, item in enumerate(value.get(section, [])): + emit('spdx.json', version, 'artifact-' + section.rstrip('s'), pointer(section, i), item, + scope='artifact', identity=item.get('SPDXID', f'{section}:{i}'), native_id=item.get('SPDXID'), standing='reported-evidence') + if value.get('bomFormat') == 'CycloneDX': + version = str(value.get('specVersion', 'hmmm')) + emit('cyclonedx.json', version, 'sbom', '', value, scope='artifact', native_id=value.get('serialNumber'), standing='reported-evidence') + if version not in {'1.4', '1.5', '1.6'}: + unsupported('unsupported_cyclonedx_version', 'component mapping tested for CycloneDX 1.4/1.5/1.6') + else: + def components(items: list, loc: str) -> None: + for index, item in enumerate(items): + where = loc + pointer(index) + emit('cyclonedx.json', version, 'artifact-component', where, item, scope='artifact', + identity=item.get('bom-ref', where), native_id=item.get('bom-ref'), standing='reported-evidence') + components(item.get('components', []), where + '/components') + components(value.get('components', []), '/components') + statement = value + statement_pointer = '' + if value.get('payloadType') == 'application/vnd.in-toto+json' and isinstance(value.get('payload'), str): + statement = parse_json(base64.b64decode(value['payload'], validate=True)) + statement_pointer = '/payload' + emit('dsse.envelope', 'v1', 'signed-envelope', '', value, scope='attestation', standing='reported-evidence') + unsupported('unverified_signature', 'envelope parsed; signature verification was not requested or performed', status='unverified') + if isinstance(statement, dict) and str(statement.get('_type', '')).startswith('https://in-toto.io/Statement/'): + version = str(statement['_type']) + fact = emit('in-toto.statement', version, 'attestation', statement_pointer, statement, scope='attestation', standing='reported-evidence') + if statement_pointer: + fact['projection'] = {'mapping_version': 'dsse-base64-json@1', 'loss': 'none; signature not verified'} + if version not in {'https://in-toto.io/Statement/v1', 'https://in-toto.io/Statement/v0.1'}: + unsupported('unsupported_statement_version', 'subject mapping requires in-toto Statement v1 or v0.1') + else: + for i, subject in enumerate(statement.get('subject', [])): + emit('in-toto.statement', version, 'attested-subject', statement_pointer + pointer('subject', i), subject, + scope='artifact', identity=subject.get('name', str(i)), native_id=subject.get('name'), standing='reported-evidence') + if path.name in {'package-lock.json', 'npm-shrinkwrap.json'}: + version = str(value.get('lockfileVersion', 'hmmm')) + emit('npm.lockfile', version, 'lockfile', '', value, scope='package') + if version not in {'2', '3'}: + unsupported('unsupported_npm_lock_version', 'resolved-package mapping requires npm lockfileVersion 2 or 3') + else: + for name, item in value.get('packages', {}).items(): + emit('npm.lockfile', version, 'locked-package', pointer('packages', name), item, + scope='package', identity=name or '.', native_id=name) + if path.name in {'Cargo.toml', 'Cargo.lock', 'poetry.lock', 'uv.lock', 'pnpm-lock.yaml'}: + convention = {'Cargo.toml': 'cargo.manifest', 'Cargo.lock': 'cargo.lockfile', + 'poetry.lock': 'poetry.lockfile', 'uv.lock': 'uv.lockfile', 'pnpm-lock.yaml': 'pnpm.lockfile'}[path.name] + version = str(value.get('version', value.get('lockfileVersion', 'source-owned-manifest'))) + emit(convention, version, 'package-metadata', '', value, scope='package') + if path.name == 'Cargo.toml': + for group in ('dependencies', 'dev-dependencies', 'build-dependencies'): + for name, requirement in value.get(group, {}).items(): + emit(convention, version, 'dependency', pointer(group, name), {'name': name, 'requirement': requirement, 'class': group}, + scope='package', identity=name, native_id=name) + for target, config in value.get('target', {}).items(): + for group in ('dependencies', 'dev-dependencies', 'build-dependencies'): + for name, requirement in config.get(group, {}).items(): + emit(convention, version, 'dependency', pointer('target', target, group, name), + {'name': name, 'requirement': requirement, 'class': group, 'condition': target}, + scope='package', identity=name, native_id=name) + elif isinstance(value.get('package'), list): + for i, package in enumerate(value['package']): + emit(convention, version, 'locked-package', pointer('package', i), package, + scope='package', identity=f"{package.get('name', '?')}@{package.get('version', '?')}", native_id=package.get('name')) + if path.name == 'REUSE.toml': + emit('reuse.toml', str(value.get('version', 'hmmm')), 'license-metadata', '', value) + for i, annotation in enumerate(value.get('annotations', [])): + emit('reuse.toml', str(value.get('version', 'hmmm')), 'license-rule', pointer('annotations', i), annotation, scope='path-rule', identity=str(i)) + if path.suffix == '.ipynb': + version = str(value.get('nbformat', 'hmmm')) + emit('jupyter.notebook', version, 'notebook-metadata', '/metadata', value.get('metadata', {})) + for i, cell in enumerate(value.get('cells', [])): + emit('jupyter.notebook', version, 'cell-metadata', pointer('cells', i, 'metadata'), cell.get('metadata', {}), scope='cell', identity=cell.get('id', str(i)), native_id=cell.get('id')) + # Reference objects are retained, never downloaded or treated as resolved. + def refs(item: Any, loc: str) -> None: + if isinstance(item, dict): + for key, child in item.items(): + child_loc = loc + pointer(key) + if key in {'$ref', '$dynamicRef'} and isinstance(child, str): + emit('metadata.reference', 'URI-reference', 'reference', child_loc, + {'uri': child, 'resolution': 'not-resolved', 'network_access': False}, standing='declared') + else: + refs(child, child_loc) + elif isinstance(item, list): + for index, child in enumerate(item): + refs(child, loc + pointer(index)) + refs(value, '') + # Native IDs have convention-owned uniqueness rules; collector pointers do + # not excuse duplicate IDs. Retain all witnesses while rejecting validity. + unique_kinds = {'openapi.document': {'api-operation'}, + 'spdx.json': {'sbom', 'artifact-package', 'artifact-file'}, + 'cyclonedx.json': {'artifact-component'}} + identities: dict[tuple[str, str], list[dict]] = {} + for fact in facts: + namespace = fact['convention']['namespace'] + native_id = fact['native']['id'] + if native_id is not None and fact['kind'] in unique_kinds.get(namespace, set()): + identities.setdefault((namespace, str(native_id)), []).append(fact) + for (namespace, _), witnesses in identities.items(): + if len(witnesses) > 1: + diagnostics.append(_diagnostic(context, reader_id=rid, + code='duplicate_native_identifier', message='duplicate native ID in ' + namespace, + status='invalid', severity='error')) + return facts, edges, diagnostics +# ratios: loc_comments=254:13 imports_exports=8:2 calls_definitions=146:5 diff --git a/.agents/skills/msdmd/typescript-reader.cjs b/.agents/skills/msdmd/typescript-reader.cjs new file mode 100644 index 00000000..b6a53a27 --- /dev/null +++ b/.agents/skills/msdmd/typescript-reader.cjs @@ -0,0 +1,120 @@ +// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm +/** Syntax-only worker. Usage: node typescript-reader.cjs < input.json + * Input: {path, text}; output: version, declarations, imports, exports, docs, + * lexically owned comments and diagnostics. No target code/config is executed. + */ +'use strict'; +const fs = require('node:fs'); +const ts = require('typescript'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +if (typeof input.path !== 'string' || typeof input.text !== 'string') throw new Error('Invalid reader input'); +const sf = ts.createSourceFile(input.path, input.text, ts.ScriptTarget.Latest, true); +const result = {version: ts.version, declarations: [], imports: [], exports: [], docs: [], comments: [], diagnostics: []}; +const span = n => ({start_line: sf.getLineAndCharacterOfPosition(n.getStart(sf)).line + 1, + end_line: sf.getLineAndCharacterOfPosition(n.getEnd()).line + 1}); +const text = n => n ? n.getText(sf) : null; +const counts = new Map(); +const commentRanges = new Map(); +const declarationRanges = []; +const qualify = (parent, name) => parent ? `${parent}.${name}` : name; +function visit(node, owner = '') { + if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) { + if (node.moduleSpecifier && ts.isStringLiteralLike(node.moduleSpecifier)) { + result.imports.push({module: node.moduleSpecifier.text, kind: ts.isImportDeclaration(node) ? 'import' : 'reexport', + declaration: text(node), owner, ...span(node)}); + } else if (ts.isExportDeclaration(node) && node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const item of node.exportClause.elements) { + result.exports.push({kind: 'local-export', local_name: (item.propertyName || item.name).text, + exported_name: item.name.text, type_only: !!node.isTypeOnly || !!item.isTypeOnly, + declaration: text(node), owner, ...span(item)}); + } + } + } + if (ts.isExportAssignment(node)) { + let expression = node.expression; + while (ts.isParenthesizedExpression(expression)) expression = expression.expression; + result.exports.push({kind: 'export-assignment', + local_name: ts.isIdentifier(expression) ? expression.text : null, + exported_name: node.isExportEquals ? 'export=' : 'default', type_only: false, + expression: text(node.expression), declaration: text(node), owner, ...span(node)}); + } + if (ts.isCallExpression(node) && (node.expression.kind === ts.SyntaxKind.ImportKeyword || + (ts.isIdentifier(node.expression) && node.expression.text === 'require'))) { + result.diagnostics.push({code: 'dynamic_module_loading', status: 'dynamic-unresolved', ...span(node)}); + } + let localOwner = owner; + const defaultModifier = !!node.modifiers?.some(m => m.kind === ts.SyntaxKind.DefaultKeyword); + let name = node.name && (ts.isIdentifier(node.name) || ts.isStringLiteralLike(node.name)) ? node.name.text : null; + const anonymousDefault = !name && defaultModifier && (ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node)); + if (anonymousDefault) name = '$default'; // Collector identity, explicitly not a source name. + const kind = ts.SyntaxKind[node.kind]; + const isDecl = name && (ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) || + ts.isTypeAliasDeclaration(node) || ts.isMethodDeclaration(node) || ts.isMethodSignature(node) || + ts.isPropertyDeclaration(node) || ts.isPropertySignature(node) || ts.isVariableDeclaration(node) || + ts.isEnumDeclaration(node) || ts.isModuleDeclaration(node)); + if (isDecl) { + localOwner = qualify(owner, name); + const ordinal = counts.get(localOwner) || 0; counts.set(localOwner, ordinal + 1); + const identity = `${localOwner}#${ordinal}`; + let carrier = node; + if (ts.isVariableDeclaration(node)) carrier = node.parent.parent; + const exported = !!carrier.modifiers?.some(m => m.kind === ts.SyntaxKind.ExportKeyword); + const declaration = {name: anonymousDefault ? null : name, qualified_name: localOwner, identity, kind, exported, + anonymous: anonymousDefault, ...span(node), + parameters: node.parameters?.map(p => ({name: text(p.name), type: text(p.type), optional: !!p.questionToken, + rest: !!p.dotDotDotToken, default: text(p.initializer)})) || [], + returns: text(node.type), decorators: (ts.canHaveDecorators(node) ? ts.getDecorators(node) : [])?.map(text) || []}; + result.declarations.push(declaration); + declarationRanges.push({start: node.getStart(sf), end: node.getEnd(), identity, owner: localOwner}); + if (defaultModifier) { + result.exports.push({kind: 'default-declaration', local_name: anonymousDefault ? null : name, + exported_name: 'default', type_only: false, declaration: text(node), owner, + declaration_identities: [identity], ...span(node)}); + } + for (const doc of carrier.jsDoc || []) { + result.docs.push({identity, owner: localOwner, ...span(doc), text: text(doc), + description: typeof doc.comment === 'string' ? doc.comment : (doc.comment || []).map(c => c.text || '').join(''), + tags: (doc.tags || []).map(t => ({tag: t.tagName.text, text: text(t), name: text(t.name), + type: text(t.typeExpression), comment: typeof t.comment === 'string' ? t.comment : null}))}); + } + } + ts.forEachChild(node, child => visit(child, localOwner)); +} +visit(sf); +// Token traversal includes trivia before closing braces (even empty bodies). +// Compiler token boundaries avoid treating strings/templates/regexes as comments. +function collectComments(node) { + for (const range of [...(ts.getLeadingCommentRanges(input.text, node.getFullStart()) || []), + ...(ts.getTrailingCommentRanges(input.text, node.getEnd()) || [])]) { + commentRanges.set(range.pos, range); + } + for (const child of node.getChildren(sf)) collectComments(child); +} +collectComments(sf); +for (const range of [...commentRanges.values()].sort((a, b) => a.pos - b.pos)) { + const owners = declarationRanges.filter(d => d.start <= range.pos && range.end <= d.end); + owners.sort((a, b) => (a.end - a.start) - (b.end - b.start)); + const owner = owners[0]; + result.comments.push({text: input.text.slice(range.pos, range.end), + start_line: sf.getLineAndCharacterOfPosition(range.pos).line + 1, + end_line: sf.getLineAndCharacterOfPosition(range.end).line + 1, + identity: owner?.identity || null, owner: owner?.owner || null, + attachment: owner ? 'lexically-enclosing-symbol' : 'module-trivia'}); +} +for (const binding of result.exports) { + const qualified = binding.local_name === null ? null : qualify(binding.owner, binding.local_name); + const declarations = binding.declaration_identities + ? result.declarations.filter(item => binding.declaration_identities.includes(item.identity)) + : result.declarations.filter(item => qualified !== null && item.qualified_name === qualified); + binding.declaration_identities = declarations.map(item => item.identity); + for (const declaration of declarations) { + declaration.exported = true; + declaration.export_names = [...new Set([...(declaration.export_names || []), binding.exported_name])]; + } +} +for (const d of sf.parseDiagnostics) result.diagnostics.push({code: `typescript_${d.code}`, status: 'invalid', + message: ts.flattenDiagnosticMessageText(d.messageText, '\n'), + start_line: sf.getLineAndCharacterOfPosition(d.start || 0).line + 1, + end_line: sf.getLineAndCharacterOfPosition((d.start || 0) + (d.length || 0)).line + 1}); +process.stdout.write(JSON.stringify(result)); +// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm diff --git a/.agents/skills/msdmd/visualize.py b/.agents/skills/msdmd/visualize.py index 7b3665f1..98b80920 100644 --- a/.agents/skills/msdmd/visualize.py +++ b/.agents/skills/msdmd/visualize.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=167:13 imports_exports=5:3 calls_definitions=70:8 +# ratios: loc_comments=231:15 imports_exports=6:3 calls_definitions=104:9 """Render an msdmd collection as a small Mermaid relationship graph. The input may be raw JSON, the generated TypeScript shape emitted by @@ -11,13 +11,16 @@ from __future__ import annotations import argparse +import hashlib import json import re from pathlib import Path _SAFE_NODE_RE = re.compile(r"[^A-Za-z0-9_]") _IDENT_RE = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*") -_CALL_MARKER = "defineMsdmdCollection(" +_COLLECTION_CALL_RE = re.compile( + r"export\s+default\s+(?:defineMsdmdCollectionV2|defineMsdmdCollection)\s*\(" +) def _strip_comments(text: str) -> str: @@ -57,10 +60,10 @@ def _strip_comments(text: str) -> str: def _extract_payload(text: str, path: Path) -> str: """Return the argument of ``defineMsdmdCollection(...)`` in ``text``.""" - start = text.find(_CALL_MARKER) - if start < 0: + match = _COLLECTION_CALL_RE.search(text) + if match is None: raise ValueError(f"{path} is not JSON or a defineMsdmdCollection TypeScript collection point") - i = start + len(_CALL_MARKER) + i = match.end() depth, j, quote = 1, i, "" while j < len(text): ch = text[j] @@ -132,13 +135,57 @@ def _object_literal_to_json(text: str) -> str: return "".join(out) +def _expand_fact_chunks(text: str) -> str: + """Decode only the generator's literal JSON chunks; never evaluate JS.""" + token = "mergeMsdmdFactChunks(" + decoder = json.JSONDecoder() + out: list[str] = [] + i = 0 + while i < len(text): + if text[i] in "\"'": + quote, start = text[i], i + i += 1 + while i < len(text): + if text[i] == "\\": + i += 2 + continue + if text[i] == quote: + i += 1 + break + i += 1 + out.append(text[start:i]) + elif text.startswith(token, i): + i += len(token) + facts: list[dict] = [] + while True: + while i < len(text) and text[i].isspace(): + i += 1 + chunk, i = decoder.raw_decode(text, i) + if not isinstance(chunk, list) or not all(isinstance(v, dict) for v in chunk): + raise ValueError("fact chunks must be literal JSON object arrays") + facts.extend(chunk) + while i < len(text) and text[i].isspace(): + i += 1 + if i < len(text) and text[i] == ")": + i += 1 + break + if i >= len(text) or text[i] != ",": + raise ValueError("unterminated fact chunks") + i += 1 + out.append(json.dumps(facts, ensure_ascii=False)) + else: + out.append(text[i]) + i += 1 + return "".join(out) + + def load_collection(path: Path) -> dict: """Load a collection from JSON, generated, or hand-authored TypeScript.""" text = _strip_comments(path.read_text(encoding="utf-8")).strip() if text.startswith("{"): return json.loads(text) - payload = _extract_payload(text, path) + payload = _expand_fact_chunks(_extract_payload(text, path)) try: return json.loads(payload) except json.JSONDecodeError: @@ -146,8 +193,9 @@ def load_collection(path: Path) -> dict: def _node_id(value: str) -> str: - normalized = _SAFE_NODE_RE.sub("_", value).strip("_") - return normalized or "hmmm" + """Return a collision-resistant Mermaid id for a qualified address.""" + normalized = _SAFE_NODE_RE.sub("_", value).strip("_")[:32] or "hmmm" + return f"{normalized}_{hashlib.sha256(value.encode('utf-8')).hexdigest()[:12]}" def _label(value: str) -> str: @@ -155,28 +203,42 @@ def _label(value: str) -> str: def render_mermaid(collection: dict) -> str: - """Render ``collection`` as Mermaid flowchart text.""" + """Render schema-1 or schema-2 ``collection`` as Mermaid flowchart text.""" lines = ["flowchart TD"] repo = collection.get("repo", "repo") - lines.append(f' repo["{_label(str(repo))}"]') + repo_node = _node_id(f"repository:{repo}") + lines.append(f' {repo_node}["{_label(str(repo))}"]') - emitted_nodes = {"repo"} + emitted_nodes = {repo_node} + labels: dict[str, str] = {} for declaration in collection.get("declarations", []): - node = _node_id(str(declaration["id"])) + address = str(declaration.get("address", declaration["id"])) + node = _node_id(address) label = f'{declaration["id"]}\\n{declaration["block"]}\\n{declaration["file"]}' + labels[address] = label if node not in emitted_nodes: lines.append(f' {node}["{_label(label)}"]') - lines.append(f" repo --> {node}") + lines.append(f" {repo_node} --> {node}") emitted_nodes.add(node) + for fact in collection.get("facts", []): + subject = fact.get("subject", {}) + address = str(subject.get("address", fact.get("address", "hmmm"))) + labels.setdefault( + address, + f'{subject.get("identity", "hmmm")}\\n{subject.get("scope", fact.get("kind", "native"))}\\n{fact.get("source", {}).get("file", "hmmm")}', + ) + for edge in collection.get("edges", []): - source = _node_id(str(edge["from"])) - target = _node_id(str(edge["to"])) + source_address = str(edge["from"]) + target_address = str(edge["to"]) + source = _node_id(source_address) + target = _node_id(target_address) if source not in emitted_nodes: - lines.append(f' {source}["{_label(str(edge["from"]))}"]') + lines.append(f' {source}["{_label(labels.get(source_address, source_address))}"]') emitted_nodes.add(source) if target not in emitted_nodes: - lines.append(f' {target}["{_label(str(edge["to"]))}"]') + lines.append(f' {target}["{_label(labels.get(target_address, target_address))}"]') emitted_nodes.add(target) lines.append(f' {source} -- "{_label(str(edge["kind"]))}" --> {target}') @@ -185,7 +247,15 @@ def render_mermaid(collection: dict) -> str: missing = ", ".join(gap.get("missing", [])) label = f'{gap.get("file", "hmmm")}\\nmissing: {missing or "hmmm"}' lines.append(f' {node}[["{_label(label)}"]]') - lines.append(f" repo -. gap .-> {node}") + lines.append(f" {repo_node} -. gap .-> {node}") + + for index, diagnostic in enumerate(collection.get("diagnostics", []), start=1): + if diagnostic.get("severity") != "error": + continue + node = f"diagnostic_{index}" + label = f'{diagnostic.get("code", "hmmm")}\\n{diagnostic.get("status", "hmmm")}' + lines.append(f' {node}{{"{_label(label)}"}}') + lines.append(f" {repo_node} -. diagnostic .-> {node}") return "\n".join(lines) + "\n" @@ -206,4 +276,4 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) -# ratios: loc_comments=167:13 imports_exports=5:3 calls_definitions=70:8 +# ratios: loc_comments=231:15 imports_exports=6:3 calls_definitions=104:9 diff --git a/.agents/skills/test-build/SKILL.md b/.agents/skills/test-build/SKILL.md index 1276ba44..bac18326 100644 --- a/.agents/skills/test-build/SKILL.md +++ b/.agents/skills/test-build/SKILL.md @@ -1,21 +1,35 @@ --- name: test-build -description: Self-declaring contract tests built on msdmd. Source modules own behavior obligations in `# === CONTRACTS ===` blocks; test modules own executable evidence in `# === CHECKS ===` blocks. Load this when adding tests that ride the msdmd convention, when refactoring a module with CONTRACTS/CHECKS declarations, or when authoring a contract/check audit or executor. +description: Native-first contract evidence built on msdmd. Source owners declare behavior obligations and test owners declare accountable witnesses through supported native conventions or supplemental CONTRACTS and CHECKS blocks. Preserve explicit claim linkage and no-exec audit boundaries. Load this when adding tests that use msdmd, refactoring CONTRACTS/CHECKS declarations, or authoring a contract/check audit or executor. --- # test-build — Contract tests on msdmd `test-build` is an application of [msdmd](../msdmd/SKILL.md). The -foundational skill defines the comment-block convention, the universal -parser, and the visible-gap requirement; this skill applies the -convention to behavior contracts and their executable witnesses. +foundation defines native-first ingestion, supplemental block syntax, reader +support and information coverage; this skill applies those contracts to +behavior obligations and accountable executable witnesses. -Read `msdmd/SKILL.md` first if you haven't — the block syntax, -parser contract, and visibility rules below are inherited from there -and not redefined. +Read `msdmd/SKILL.md` first. For the ratified source/test ownership split, +see [`doctrine/msdmd-checks.md`](../doctrine/msdmd-checks.md). -For the ratified doctrine behind this split, see -[`doctrine/msdmd-checks.md`](../doctrine/msdmd-checks.md). +## Native-first coverage + +Source-owned native contracts/assertions and test-owned witness declarations +may supply evidence-graph inputs through explicit, tested mappings. Keep each +obligation and witness attributable to its actual owner. A native witness needs +no CHECKS copy when the supported mapping provides the required linkage, +execution target and safety information. Test names, markers and examples alone +do not establish a `proves` relationship or a passing execution result. + +Native fields retain their own schemas; the block schemas below constrain +supplemental entries. They do not require native sources to manufacture MSDMD +IDs or strings. Preserve qualified source identities and unresolved mappings. + +Implementation status: the examples and bundled RepoLOTO audit/check path use +CONTRACTS/CHECKS blocks. Native contract/witness readers and a unified native +evidence reconciler are not shipped by this skill revision. A block-only audit +must disclose that scope, and cannot call required native scope complete. ## The split @@ -26,13 +40,13 @@ audit reconciles the witness list against the obligation list. ``` Source modules own promises. Test modules own evidence. Neither owns -the other's declarations. +the other's declarations. Native ingestion preserves this ownership split. ## Source block: CONTRACTS -Every module that promises behavior declares those obligations in a -`CONTRACTS` block. A contract says what must remain true; it does not -name the test topology. +Source owners declare promised behavior through supported native contracts or +supplemental `CONTRACTS` entries. A contract says what must remain true; it does +not name the test topology. Supplement unexpressed obligations as follows: ```python # === CONTRACTS === @@ -50,7 +64,7 @@ name the test topology. ### CONTRACTS field schema -Required: +Required for a supplemental CONTRACTS entry: | Field | Meaning | |---|---| @@ -68,13 +82,13 @@ Optional: | `deprecated` | If present, the runner skips and reports the entry as deprecated. | `call:` is not a CONTRACTS field in skill-lib. The call belongs to the -CHECKS entry that owns the executable evidence. +CHECKS entry or native witness that owns the executable evidence. ## Test block: CHECKS -A test module declares the checks it contributes in a `CHECKS` block. -A check is an evidentiary procedure: an executable claim to prove one -or more named contracts. +Use `CHECKS` for witness information not adequately expressed by a supported +native source. A check is an evidentiary procedure: an executable claim to prove +one or more named contracts. ```python # === CHECKS === @@ -98,7 +112,7 @@ or more named contracts. ### CHECKS field schema -Required: +Required for a supplemental CHECKS entry: | Field | Meaning | |---|---| @@ -121,28 +135,27 @@ as a defect, not diligence. ## The contract for check functions -A check function: +A check function in the block executor contract: - Is resolvable at the path declared in `call:`. - Takes no required arguments. The executor does not inject fixtures or context; the check is self-contained or pulls from the language's standard environment (env vars, a known service URL, etc.). - Returns `None` on pass. -- Raises `AssertionError` on behavior violation. The runner reports - this as `FAIL`. -- Lets unexpected exceptions escape. The runner reports these as - `ERROR` (infrastructure/harness failure) rather than `FAIL` - (contract violation). -- Cleans up any persistent state it creates. Isolation is the check's - responsibility unless the runner explicitly provides a fixture. +- Raises `AssertionError` on behavior violation. The runner reports `FAIL`. +- Lets unexpected exceptions escape. The runner reports `ERROR` + (infrastructure/harness failure), not `FAIL` (contract violation). +- Cleans up persistent state. Isolation belongs to the check unless the runner + explicitly supplies a fixture. + +A native framework adapter must separately declare its actual calling and +fixture conventions rather than pretending every native test has this shape. ## Authoring an audit -Audit is the cheapest runner mode: reconcile declarations without -executing checks. Resolve `self::fn` against the declaring file's -**parsed** function definitions — never by importing it or reading -loaded callables, since import executes module top level and an audit -that executes is not an audit: +Audit reconciles declarations without executing checks. Resolve `self::fn` +against the declaring file's **parsed** definitions — never by importing the +module, since importing executes its top level: ```python import ast @@ -164,12 +177,10 @@ def resolve_self_call(spec: str, defined: set[str]) -> str: return name ``` -(The bundled `tests/test_repo_loto.py` reads `globals()` instead — it -can, because its audit runs *as* that module, so its own `def`s are -already in scope. A central audit walking other test files has no such -shortcut and must parse, as above.) +The bundled `tests/test_repo_loto.py` may read `globals()` because its audit runs +as that module; a central audit of other test files has no such shortcut. -An audit MUST report, at minimum: +For an explicitly block-only audit, the existing diagnostics are: ```text GAP <contract> has no CHECKS entry claiming to prove it @@ -178,34 +189,37 @@ GAP <check> call does not resolve: <reason> GAP executable check <fn> has no resolving CHECKS declaration ``` -Exit nonzero on any gap. A reconciler that has only ever said -"closed" is itself unverified; negative-test it by planting an orphan -contract, a phantom `proves` target, and an unresolvable call, then -observing the GAP. +A native-capable audit applies the same semantic obligations to supported native +witnesses as well. Missing CHECKS syntax alone is not missing evidence. Required +unsupported mappings are unresolved coverage and fail a strict audit; they must +not be reported as proof that no witness exists. + +Exit nonzero on genuine evidence gaps. Negative-test the audit with an orphan +contract, a phantom `proves` target and an unresolvable call. A reconciler that +has only ever reported closed is unverified. ## Authoring an executor -A full executor runs after audit or as part of the same command. It -should: +A full executor should: -1. Parse source `CONTRACTS` and test `CHECKS` using the msdmd parser. -2. Reconcile the graph before execution. +1. Extract supported native obligations/witnesses and parse supplemental + source `CONTRACTS` and test `CHECKS` using the msdmd parser. An existing + block-only executor must disclose its narrower scope. +2. Reconcile the evidence graph before execution; refuse required unresolved scope. 3. Refuse execution when consumed `requires` fields are unmet. 4. Apply consumed `timeout` fields to the actual spawned work. -5. Report per-check `PASS`, `FAIL`, and `ERROR` without aborting the - remaining checks on a single harness error. -6. Surface source contracts with no proving checks, checks proving - unknown contracts, and executable checks with no declaration. +5. Report per-check `PASS`, `FAIL` and `ERROR` without concealing harness failures. +6. Surface contracts without accountable witnesses, orphan witness claims and + tests lacking required linkage, without requiring duplicate native declarations. -The visibility-of-gaps requirement is mandatory per msdmd. Drop it and -the runner stops being a msdmd application. +The visibility-of-gaps requirement is mandatory per msdmd. Report information +and evidence coverage, supported extraction and block adoption separately. ## Semantics of "proves" `proves:` means claims-to-prove. Audit verifies linkage and call resolution. A passing check demonstrates the declared witness ran -successfully. It does not prove the check is sensitive to every -possible breakage of the contract. +successfully. It does not prove sensitivity to every possible breakage. Status vocabulary: @@ -217,34 +231,36 @@ Status vocabulary: Do not claim one rung above the evidence. +## Validation + +Native-only witness fixtures need explicit source-linked obligations, executable +targets and safety metadata, with no invented `proves` edges or CHECKS copies. +Pair these future reader tests with orphan, unsupported-reader, unsafe-import +and wrong-revision-report cases. Existing block audit fixtures remain required. + ## Anti-patterns -- **Contracts in test files instead of source files.** The contract - belongs to the module that promises the behavior; the test file owns - the check. -- **`call:` in CONTRACTS.** Source modules should not know test - topology. Put executable targets in CHECKS. -- **Executable tests with no CHECKS entry.** They may still run through - ad hoc tooling, but they are invisible to the msdmd evidence graph. -- **CHECKS proving unknown CONTRACTS.** This is an orphan witness; fix - the target id or declare the source contract. -- **Implementation-shaped ids.** `chat_create_returns_200` tells you - little; `chat_create_owner_isolation` tells you what's protected. -- **Importing during audit.** Python imports execute module top level. - Use no-exec resolution such as `self::fn`, or make import execution - an explicit non-audit mode. -- **Catching unexpected exceptions in the check to "make it pass".** - Let the exception escape so the runner can mark `ERROR` honestly. +- **Contracts in test files instead of source files.** The promise belongs to + its source owner; the test file owns the check. +- **`call:` in CONTRACTS.** Put executable targets with their accountable witnesses. +- **Executable tests with no CHECKS entry.** Do not label them untracked until + eligible native witness mappings have been inspected; missing required mapping + is unresolved coverage, not permission to invent evidence. +- **CHECKS proving unknown CONTRACTS.** Repair the orphan target or source obligation. +- **Implementation-shaped ids.** `chat_create_returns_200` says less about the + protected capability than `chat_create_owner_isolation`. +- **Importing during audit.** Use no-exec resolution or a separately authorized execution mode. +- **Catching unexpected exceptions to make a check pass.** Preserve `ERROR` honestly. ## Versioning -The `CONTRACTS` block name remains stable for source-owned -obligations. `CHECKS` is the paired test-owned evidence block. -Field additions are non-breaking only when they are additive and -consumed by a runner. Field renames or removals are breaking; bump the -major version and note the migration in the lib README. +The `CONTRACTS` block name remains stable for source-owned obligations. +`CHECKS` remains the paired test-owned evidence block. Field additions are +non-breaking only when additive and consumed by a runner; renames or removals +require a major version and migration note. Native schema migration is separate. hmmm -- The block type for harness/infrastructure tests that prove no product contract remains unnamed is still unsettled. -- Mutation-level verification is defined but not yet generalized across skills. -- Slow/flaky/quarantined states should enter only when a runner consumes them rather than as decorative labels. +- native contract/witness readers and qualified evidence identities remain implementation work +- the block type for harness tests that prove no product contract remains unsettled +- mutation-level verification is not yet generalized across skills +- slow/flaky/quarantined states need actual consumers before becoming schema fields From 9a0222eb1a8bf75b4869ee0343c63586bbc268ec Mon Sep 17 00:00:00 2001 From: Erin Spencer <erin.eps.hovel@gmail.com> Date: Tue, 6 Oct 2026 02:50:28 -0700 Subject: [PATCH 2/2] chore(skills): re-sync vendored skill-lib skills to 38c6433 Re-propagated with skill-lib tools/propagate_skills.py from The-Interdependency/skill-lib@38c64332b840b2bbe1c07e53aeee8996644548e9 (main after #118). Pin updated in .agents/skills/README.md; the vendored msdmd helper exports MSDMD_COLLECTION_HELPER_VERSION = 1. --- .agents/skills/README.md | 2 +- .agents/skills/msdmd/SKILL.md | 29 +- .agents/skills/msdmd/collect.py | 525 +++++++++++++++++- .agents/skills/msdmd/collection.ts | 14 +- .agents/skills/msdmd/native_code.py | 15 +- .agents/skills/msdmd/readers.py | 140 ++++- .../msdmd/references/implemented-readers.md | 20 + .agents/skills/msdmd/standards.py | 22 +- 8 files changed, 712 insertions(+), 55 deletions(-) diff --git a/.agents/skills/README.md b/.agents/skills/README.md index ba32555b..d3a83971 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -3,7 +3,7 @@ This directory contains repo-local copies of canonical skills from `The-Interdependency/skill-lib`. -Source commit: `9867ab33877f2b1f50f8a501cf379aa99360bd52` +Source commit: `38c64332b840b2bbe1c07e53aeee8996644548e9` Repo-local copies are not the source of truth. Edit `skill-lib` first, then propagate from the canonical source. diff --git a/.agents/skills/msdmd/SKILL.md b/.agents/skills/msdmd/SKILL.md index fa8be158..c6300e84 100644 --- a/.agents/skills/msdmd/SKILL.md +++ b/.agents/skills/msdmd/SKILL.md @@ -36,7 +36,10 @@ python -m pip install -r msdmd/requirements.txt npm ci --ignore-scripts --prefix msdmd ``` -Missing dependencies become unsupported-reader diagnostics, never empty success. +A missing runtime is an error diagnostic and marks the reader run +`runtime-unavailable`; it never yields empty success. The CLI exits 3 without +writing unless `--allow-missing-reader-runtimes` is given, which still warns. +Other TypeScript worker failures are `typescript_reader_failed` errors. The universal MSDMD block parsers themselves remain dependency-free. ## Doctrine @@ -344,6 +347,25 @@ python -m msdmd.collect --root . --repo The-Interdependency/skill-lib \ Source revision, worktree state and byte digests remain separate. A snapshot is not producer authentication. Excluded output paths are configured even before their first write, preventing generation from changing its own source identity. +The collector never reads its own output: the `--out` path, its hidden +`.<name>.*` siblings, any `.*_msdmd.ts.*` candidate and a shell-redirected +stdout file are skipped and unrecorded, so differently named temporaries render +identical bytes. Only the stable `<repo>_msdmd.ts` name is recorded. Git-ignored +files are never read in a Git checkout; if git cannot list files there (for +example, git is not on PATH), or the root is ignored by an enclosing repository, +the CLI prints an ERROR and exits 5 without writing (`--check` reports 5, not +drift) instead of reading ignored files. A git submodule is never read: it +is an `excluded` ledger entry (`entry_kind: submodule`, `reason: git-submodule`) +carrying the pinned `commit`, which snapshot identities include. +`--print-generator-identity` prints a digest of every collector file that can +change output (TypeScript worker and lock file included), the Python minor, +reader package, Node and TypeScript versions, and digests of the reader modules +that actually resolve on `sys.path`; add `--json` for the parts. A schema-2 +artifact needs a helper exporting `MSDMD_COLLECTION_HELPER_VERSION` at least the +collector's; an older helper stops the CLI with exit 4 before writing. When +`--out` is outside the root, the helper is located from the root. Propagate the +skill or use `--legacy-blocks-only`. Exit 3, 4 and 5 problems are reported +together, with precedence 5, then 3, then 4. ### Required information and disclosure @@ -364,7 +386,10 @@ Parse errors and identity conflicts always invalidate strict collection. Native facts and blocks remain separate; no universal precedence overwrites disagreements. Directory-descriptor discovery does not follow symlinks. It accounts for ignored -subtrees, unsupported inputs, byte limits and read errors. This safety path +subtrees, unsupported inputs, byte limits and read errors. Bytes no reader can +consume are hashed but not retained; retained bytes have an aggregate bound +(`--max-total-bytes`, default 256 MiB) whose overflow is an error and a CLI +warning. This safety path requires POSIX no-follow directory-descriptor support. Secret-named files are excluded; sensitive structured fields and URL credentials are redacted. This is not a complete secret detector: public release still requires audience review. diff --git a/.agents/skills/msdmd/collect.py b/.agents/skills/msdmd/collect.py index 390afb6c..32992db6 100644 --- a/.agents/skills/msdmd/collect.py +++ b/.agents/skills/msdmd/collect.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=706:54 imports_exports=17:5 calls_definitions=213:21 +# ratios: loc_comments=1064:134 imports_exports=21:9 calls_definitions=374:33 # === DOCS === # id: msdmd_foundational_contract # source: msdmd/SKILL.md @@ -26,7 +26,7 @@ # network_boundary: none # user_data_boundary: read # admin_only: false -# tests: tests/test_collect.py, tests/test_native_collection.py +# tests: tests/test_collect.py, tests/test_native_collection.py, tests/test_msdmd_consumer_review.py, tests/test_msdmd_review_followup.py # rollout: schema 2 is the default CLI output; --legacy-blocks-only keeps explicit schema 1 compatibility # rollback: use --legacy-blocks-only while preserving schema incompatibility visibility # === END MODULE_BUILD === @@ -36,6 +36,15 @@ python -m msdmd.collect --root . --repo owner/repo --out repo_msdmd.ts python -m msdmd.collect --root . --repo owner/repo --strict python -m msdmd.collect --root . --repo owner/repo --legacy-blocks-only + python -m msdmd.collect --print-generator-identity + +Exit status: 1 drift under --check, 2 strict diagnostics, 3 missing native +reader runtime (unless --allow-missing-reader-runtimes), 4 schema helper +older than the rendered output (MSDMD_COLLECTION_HELPER_VERSION), 5 git +could not list visible files or the root is git-ignored by an enclosing +repository. Exit 3, 4 and 5 problems are all reported before exiting, with +precedence 5, then 3, then 4. Nothing is written or compared (--check never +reports them as drift) on exit 3, 4 or 5. The default schema-2 path statically reads supported native conventions and supplemental MSDMD blocks. It never imports inspected code, executes scripts, @@ -51,8 +60,10 @@ import hashlib import json import os +import re import stat import subprocess +import sys import io import tokenize from collections import Counter, defaultdict @@ -61,12 +72,28 @@ from urllib.parse import quote from msdmd.parsers.universal import marker_for, parse_text -from msdmd.readers import READER_MANIFESTS, _redact_sensitive, read_native +from msdmd.readers import READER_MANIFESTS, _redact_sensitive, read_native, readers_for SCHEMA_ID = "the-interdependency.msdmd-collection" SCHEMA_VERSION = "2.0.0" LEGACY_SCHEMA_VERSION = "1.0.0" DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 +# Aggregate bound on source bytes retained for readers during one scan. +DEFAULT_MAX_TOTAL_BYTES = 256 * 1024 * 1024 +# Native reader runtimes whose absence makes extraction incomplete. +RUNTIME_UNAVAILABLE_CODES = frozenset({ + "missing_docstring_parser", + "typescript_reader_unavailable", + "reader_dependency_unavailable", +}) +# Git could not establish which files are visible, or the root is ignored by +# an enclosing repository: the CLI exits 5 without writing. +VISIBILITY_FAILURE_CODES = frozenset({"git_visibility_unavailable", "root_git_ignored"}) +# tempfile.mkstemp(prefix=".msdmd-") names written by this CLI. +_COLLECTOR_TEMP_RE = re.compile(r"\.msdmd-[a-z0-9_]{8}") +# Hidden sibling temporaries/candidates of any collection artifact, such as +# ".repo_msdmd.ts.<random>.candidate" written by fresh-making executors. +_ARTIFACT_SIBLING_RE = re.compile(r"\..+_msdmd\.ts\..+") DEFAULT_BLOCK_NAMES = ( "DOCS", @@ -204,10 +231,99 @@ def _raw_block_edges(declaration: dict[str, Any]) -> list[dict[str, Any]]: return edges -def _git_identity(root: Path, explicit_revision: str | None) -> tuple[str, bool | str, str | None]: +def _is_own_output(relative: str, configured: frozenset[str] | set[str]) -> bool: + """Return whether ``relative`` is this collection's output or a temporary of it. + + Configured outputs, their hidden ``.<name>.*`` siblings, any hidden + ``.*_msdmd.ts.*`` sibling, and this CLI's ``.msdmd-*`` atomic-write + temporaries are never inputs, so differently named temporaries cannot + change collection bytes. + """ + if relative in configured: + return True + parent, _, name = relative.rpartition("/") + if _COLLECTOR_TEMP_RE.fullmatch(name) or _ARTIFACT_SIBLING_RE.fullmatch(name): + return True + for output in configured: + output_parent, _, output_name = output.rpartition("/") + if parent == output_parent and name.startswith("." + output_name + "."): + return True + return False + + +def _git_marker(root: Path) -> Path | None: + """Return the nearest ``.git`` entry at or above ``root``, if any.""" + for candidate in (root, *root.parents): + if (candidate / ".git").exists(): + return candidate / ".git" + return None + + +def _git_visibility(root: Path) -> tuple[set[str] | None, dict[str, str], list[dict[str, Any]]]: + """Return git-visible files, pinned submodules and visibility problems. + + Visible files are tracked plus untracked, not-ignored files under ``root``. + Git-ignored files are machine-local and may hold credentials; they are not + part of a commit-bound or portable snapshot identity. When a ``.git`` entry + exists but git cannot list files, discovery fails closed (an empty visible + set plus an error) instead of falling back to reading ignored files. A root + that an enclosing repository ignores is diagnosed rather than reported as a + complete, empty snapshot. Outside any git checkout, ``None`` means a plain + directory scan. + """ + def run(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + + def problem(code: str, message: str) -> dict[str, Any]: + return {"code": code, "severity": "error", "status": "unreadable", "message": message, + "source": {"file": "."}, "reader_id": None} + + marker = _git_marker(root) + try: + inside = run("rev-parse", "--is-inside-work-tree") + except OSError: + inside = None + if inside is None or inside.returncode != 0 or inside.stdout.strip() != b"true": + if marker is None: + return None, {}, [] + return set(), {}, [problem("git_visibility_unavailable", + f"{marker} exists but git could not list visible files; no files were read rather than risk reading git-ignored files")] + problems: list[dict[str, Any]] = [] + prefix = run("rev-parse", "--show-prefix") + if prefix.returncode == 0 and prefix.stdout.strip(): + ignored = run("check-ignore", "-q", "--", ".") + if ignored.returncode == 0: + problems.append(problem("root_git_ignored", + "the collection root is git-ignored by its enclosing repository; untracked files under it are not read " + "and the snapshot is incomplete. Collect from a non-ignored root or initialize a repository there")) + elif ignored.returncode not in (0, 1): + return set(), {}, [problem("git_visibility_unavailable", + "git check-ignore failed for the collection root; no files were read")] + listed = run("ls-files", "-z", "--cached", "--others", "--exclude-standard") + staged = run("ls-files", "-z", "--stage") + if listed.returncode != 0 or staged.returncode != 0: + return set(), {}, [problem("git_visibility_unavailable", + "git ls-files failed inside a git checkout; no files were read rather than risk reading git-ignored files")] + submodules: dict[str, str] = {} + for record in staged.stdout.split(b"\0"): + meta, _, path = record.partition(b"\t") + fields = meta.split() + if len(fields) == 3 and fields[0] == b"160000": + submodules[os.fsdecode(path)] = fields[1].decode("ascii") + visible = {os.fsdecode(item) for item in listed.stdout.split(b"\0") if item} - set(submodules) + return visible, submodules, problems + + +def _git_identity( + root: Path, + explicit_revision: str | None, + ignored_changes: Iterable[str] = (), + configured_outputs: frozenset[str] | set[str] = frozenset(), +) -> tuple[str, bool | str, str | None]: revision = explicit_revision dirty: bool | str = "hmmm" git_head: str | None = None + ignored_changes = set(ignored_changes) try: git_head = subprocess.run( ["git", "-C", str(root), "rev-parse", "HEAD"], @@ -217,30 +333,94 @@ def _git_identity(root: Path, explicit_revision: str | None) -> tuple[str, bool ).stdout.strip() if revision is None: revision = git_head - status_output = subprocess.run( - ["git", "-C", str(root), "status", "--porcelain", "--untracked-files=all"], + prefix = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-prefix"], check=True, capture_output=True, text=True, + ).stdout.strip() + status_output = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain=v1", "-z", "--untracked-files=all", "--", "."], + check=True, + capture_output=True, ).stdout + records = [os.fsdecode(item) for item in status_output.split(b"\0")] input_changes = [] - for line in status_output.splitlines(): - changed_path = line[3:].split(" -> ")[-1].strip('"') if len(line) > 3 else "" - if not Path(changed_path).name.endswith("_msdmd.ts"): - input_changes.append(line) + index = 0 + while index < len(records): + record = records[index] + index += 1 + if len(record) < 4: + continue + changed_paths = [record[3:]] + if {"R", "C"} & set(record[:2]): + # Either porcelain column may carry the rename/copy; its source path follows. + if index < len(records): + changed_paths.append(records[index]) + index += 1 + for changed_path in changed_paths: + if prefix and changed_path.startswith(prefix): + changed_path = changed_path[len(prefix):] + if (Path(changed_path).name.endswith("_msdmd.ts") or changed_path in ignored_changes + or _is_own_output(changed_path, configured_outputs)): + continue + input_changes.append(changed_path) dirty = bool(input_changes) except (OSError, subprocess.CalledProcessError): pass return revision or "hmmm", dirty, git_head -def _discover(root: Path, max_file_bytes: int, generated_outputs: Iterable[str] = (), *, stable_exclusions: bool = False) -> tuple[list, list, list]: - """Read bounded files through directory descriptors; never follow symlinks.""" +def _discover( + root: Path, + max_file_bytes: int, + generated_outputs: Iterable[str] = (), + *, + stable_exclusions: bool = False, + recorded_outputs: Iterable[str] | None = None, + visible_files: set[str] | None = None, + excluded_inodes: Iterable[tuple[int, int]] = (), + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, + own_outputs: list[str] | None = None, + submodules: dict[str, str] | None = None, + visibility_problems: Iterable[dict[str, Any]] = (), +) -> tuple[list, list, list]: + """Read bounded files through directory descriptors; never follow symlinks. + + Own outputs and temporaries are skipped without a ledger entry because + their names vary per run. Git-ignored files are skipped the same way when + ``visible_files`` is known. Only bytes that a reader or block marker can + consume are retained, and the retained total is bounded. + """ if max_file_bytes < 1: raise ValueError("max_file_bytes must be positive") + if max_total_bytes < 1: + raise ValueError("max_total_bytes must be positive") readable: list = [] ledger: list = [] - diagnostics: list = [] + diagnostics: list = list(visibility_problems) + for item in diagnostics: + # A git visibility failure or ignored root leaves the snapshot incomplete. + ledger.append({"file": ".", "entry_kind": "subtree", "status": "excluded", "reason": item["code"].replace("_", "-"), + "reader_ids": [], "content_sha256": "hmmm"}) + submodules = dict(submodules or {}) + for path, commit in sorted(submodules.items()): + # Submodule contents belong to another repository; record the pin, never read it. + ledger.append({"file": path, "entry_kind": "submodule", "status": "excluded", "reason": "git-submodule", + "reader_ids": [], "content_sha256": "hmmm", "commit": commit}) + configured = frozenset(generated_outputs) + excluded_inodes = set(excluded_inodes) + own_outputs = own_outputs if own_outputs is not None else [] + visible_dirs: set[str] | None = None + if visible_files is not None: + visible_dirs = set() + for visible in visible_files: + parent = visible.rpartition("/")[0] + while parent and parent not in visible_dirs: + visible_dirs.add(parent) + parent = parent.rpartition("/")[0] + retained_total = [0] + over_budget: list[str] = [] if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY"): raise OSError("safe native discovery requires no-follow directory descriptor support") @@ -254,8 +434,7 @@ def excluded(relative: str, reason: str, *, entry_kind: str = "file", digest: st if stable_exclusions: for name in sorted(_DEFAULT_SKIP): excluded("**/" + name + "/**", "configured-subtree", entry_kind="subtree") - generated_outputs = set(generated_outputs) - for output in sorted(generated_outputs): + for output in sorted(configured if recorded_outputs is None else set(recorded_outputs)): excluded(output, "generated-collection-output", entry_kind="output") def failed(relative: str, exc: Exception, entry_kind: str = "file") -> None: @@ -271,17 +450,24 @@ def visit(directory_fd: int, parent: str = "") -> None: return for name in names: relative = f"{parent}/{name}" if parent else name + if relative in submodules: + continue try: - if relative in generated_outputs: + if _is_own_output(relative, configured): + own_outputs.append(relative) continue mode = os.stat(name, dir_fd=directory_fd, follow_symlinks=False).st_mode if stat.S_ISLNK(mode): + if visible_files is not None and relative not in visible_files: + continue excluded(relative, "symlink-not-followed", digest=_sha256(os.readlink(name, dir_fd=directory_fd))) continue if stat.S_ISDIR(mode): if name in _DEFAULT_SKIP: if not stable_exclusions: excluded(relative, "configured-subtree", entry_kind="subtree") + elif visible_dirs is not None and relative not in visible_dirs: + continue else: child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory_fd) try: @@ -289,6 +475,8 @@ def visit(directory_fd: int, parent: str = "") -> None: finally: os.close(child) continue + if visible_files is not None and relative not in visible_files: + continue if not stat.S_ISREG(mode): excluded(relative, "not-regular-file") continue @@ -301,6 +489,9 @@ def visit(directory_fd: int, parent: str = "") -> None: fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd) with os.fdopen(fd, "rb") as handle: info = os.fstat(handle.fileno()) + if (info.st_dev, info.st_ino) in excluded_inodes: + own_outputs.append(relative) + continue if not stat.S_ISREG(info.st_mode): excluded(relative, "not-regular-file") continue @@ -311,7 +502,18 @@ def visit(directory_fd: int, parent: str = "") -> None: if len(data) > max_file_bytes: excluded(relative, "size-limit", size=len(data)) continue - readable.append({"file": relative, "data": data, "content_sha256": _sha256(data), "size": len(data)}) + digest = _sha256(data) + relative_path = Path(relative) + if not readers_for(relative_path, data) and marker_for(relative_path) is None: + # No reader or block grammar consumes these bytes; keep identity only. + readable.append({"file": relative, "data": b"", "content_sha256": digest, "size": len(data)}) + continue + if retained_total[0] + len(data) > max_total_bytes: + excluded(relative, "aggregate-size-limit", digest=digest, size=len(data)) + over_budget.append(relative) + continue + retained_total[0] += len(data) + readable.append({"file": relative, "data": data, "content_sha256": digest, "size": len(data)}) except OSError as exc: failed(relative, exc) try: @@ -323,6 +525,10 @@ def visit(directory_fd: int, parent: str = "") -> None: visit(root_fd) finally: os.close(root_fd) + if over_budget: + diagnostics.append({"code": "aggregate_size_limit", "severity": "error", "status": "unreadable", + "message": f"{len(over_budget)} file(s) not read after the {max_total_bytes}-byte aggregate scan budget", + "source": {"file": over_budget[0], "files": over_budget}, "reader_id": None}) return readable, ledger, diagnostics @@ -425,14 +631,25 @@ def collect( generated_outputs: Iterable[str] = (), required_sources: Iterable[str] = (), required_facts: Iterable[dict[str, str]] = (), + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, + excluded_inodes: Iterable[tuple[int, int]] = (), ) -> dict[str, Any]: """Collect native facts and supplemental blocks under ``root`` as schema 2.""" root = root.resolve() block_names = tuple(block_names) expected_blocks = tuple(expected_blocks) - revision, dirty, git_head = _git_identity(root, source_commit) - outputs = set(generated_outputs) | {repo.rsplit("/", 1)[-1] + "_msdmd.ts"} - readable, discovery, diagnostics = _discover(root, max_file_bytes, outputs, stable_exclusions=snapshot_identity) + canonical_output = repo.rsplit("/", 1)[-1] + "_msdmd.ts" + outputs = frozenset(generated_outputs) | {canonical_output} + own_outputs: list[str] = [] + visible_files, submodules, visibility_problems = _git_visibility(root) + readable, discovery, diagnostics = _discover( + root, max_file_bytes, outputs, stable_exclusions=snapshot_identity, + # Only the stable artifact name is recorded; per-run output names are not. + recorded_outputs={canonical_output}, visible_files=visible_files, + excluded_inodes=excluded_inodes, max_total_bytes=max_total_bytes, own_outputs=own_outputs, + submodules=submodules, visibility_problems=visibility_problems, + ) + revision, dirty, git_head = _git_identity(root, source_commit, own_outputs, outputs) required_sources = tuple(required_sources) required_facts = tuple(required_facts) for requirement in required_facts: @@ -443,6 +660,9 @@ def collect( if source_commit is not None: raise ValueError("snapshot identity and an explicit commit cannot both be selected") source_bytes = "\n".join(item["file"] + "\0" + item["content_sha256"] for item in sorted(readable, key=lambda x: x["file"])) + if submodules: + # Pinned submodule commits are part of the superproject snapshot. + source_bytes += "".join(f"\n{path}\0gitlink:{commit}" for path, commit in sorted(submodules.items())) revision = "snapshot:" + _sha256(source_bytes) dirty, git_head = "hmmm", None if source_commit is not None and git_head is not None and source_commit != git_head: @@ -607,6 +827,12 @@ def collect( diagnostics.append({"code": "required_fact_unresolved", "severity": "error", "status": "unsupported", "message": "required native fact is not supported by complete source extraction", "source": dict(requirement), "reader_id": None}) + runtime_unavailable_readers: set[str] = set() + for diagnostic in diagnostics: + if diagnostic.get("code") in RUNTIME_UNAVAILABLE_CODES: + # Missing runtimes make output incomplete, never an accepted subset. + diagnostic["severity"] = "error" + runtime_unavailable_readers.add(str(diagnostic.get("reader_id"))) reader_runs = [] for manifest in READER_MANIFESTS: reader_id = manifest["reader_id"] @@ -615,7 +841,7 @@ def collect( "reader_id": reader_id, "reader_version": manifest["version"], "support": manifest["support"], - "status": "applied" if matched else "not-applicable", + "status": "runtime-unavailable" if reader_id in runtime_unavailable_readers else "applied" if matched else "not-applicable", "files_matched": matched, "facts_emitted": reader_fact_counts[reader_id], }) @@ -637,7 +863,7 @@ def collect( "git_head": git_head or "hmmm", "dirty_worktree": dirty, "snapshot_sha256": snapshot_sha256, - "snapshot_complete": not any(item["status"] == "unreadable" or (item["status"] == "excluded" and item.get("reason") not in {"configured-subtree", "generated-collection-output"}) for item in discovery), + "snapshot_complete": not any(item["status"] == "unreadable" or (item["status"] == "excluded" and item.get("reason") not in {"configured-subtree", "generated-collection-output", "git-submodule"}) for item in discovery), "revision_kind": "content-snapshot" if snapshot_identity else "git-or-declared", "scope": "configured in-scope files; excluded subtrees are reported without traversing", "exclusion_accounting": "configured-patterns" if snapshot_identity else "observed-paths", @@ -721,6 +947,183 @@ def collection_errors(collection: dict[str, Any]) -> list[dict[str, Any]]: return [item for item in collection.get("diagnostics", []) if item.get("severity") == "error"] +def runtime_unavailable(collection: dict[str, Any]) -> list[dict[str, Any]]: + """Return diagnostics showing a native reader runtime was not installed.""" + return [item for item in collection.get("diagnostics", []) if item.get("code") in RUNTIME_UNAVAILABLE_CODES] + + +_NODE_PROBE = ("let t='absent';try{t=require('typescript/package.json').version}catch(e){}" + "process.stdout.write(JSON.stringify({node:process.version,typescript:t}))") + + +# Import names that cannot be derived from a distribution name. +_DISTRIBUTION_MODULES = {"pyyaml": ("yaml",)} + + +def _resolved_reader_modules(distributions: list[str], metadata: Any) -> dict[str, str]: + """Digest the files each reader distribution's import names resolve to. + + Resolution uses ``importlib.util.find_spec`` on the live ``sys.path`` + without importing the module, so the digest reflects what the readers + would actually import, including a shadowing module earlier on the path. + """ + import importlib.util + + try: + provided = metadata.packages_distributions() + except Exception: # pragma: no cover - metadata backends vary + provided = {} + by_distribution: dict[str, set[str]] = defaultdict(set) + for module, owners in provided.items(): + for owner in owners: + by_distribution[re.sub(r"[-_.]+", "-", owner).lower()].add(module) + resolved: dict[str, str] = {} + for distribution in distributions: + key = re.sub(r"[-_.]+", "-", distribution).lower() + names = set(_DISTRIBUTION_MODULES.get(key, ())) | {key.replace("-", "_")} + names |= {name for name in by_distribution.get(key, ()) if name.isidentifier() and not name.startswith("_")} + for name in sorted(names): + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + spec = None + if spec is None or not spec.origin or not os.path.isfile(spec.origin): + resolved[name] = "absent" + continue + if spec.submodule_search_locations: + roots = [Path(location) for location in spec.submodule_search_locations] + files = sorted(path for location in roots for path in location.rglob("*") + if path.is_file() and "__pycache__" not in path.parts) + else: + roots = [Path(spec.origin).parent] + files = [Path(spec.origin)] + digest = hashlib.sha256() + for path in files: + owner = next((location for location in roots if path.is_relative_to(location)), path.parent) + digest.update(path.relative_to(owner).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + resolved[name] = "sha256:" + digest.hexdigest() + return resolved + + +def generator_identity_components(base: Path | None = None) -> dict[str, Any]: + """Return every collector input that can change output bytes. + + ``source_sha256`` covers the Python implementation, the trusted TypeScript + worker, its npm manifest and lock file, reader schema assets and the pinned + Python requirements (docs and installed trees excluded). The runtime part + records the Python minor version, the installed version of each pinned + reader package, a digest of the module files each of those packages + actually resolves to on ``sys.path`` (so a shadowing module on PYTHONPATH + changes the identity even when metadata versions do not), the Node version + and the TypeScript version that the worker resolves from ``base``; a + missing runtime is recorded as ``absent``. + """ + from importlib import metadata + + base = (base or Path(__file__).parent).resolve() + files = sorted( + path for path in base.rglob("*") + if path.is_file() + and not {"node_modules", "__pycache__", "references"} & set(path.relative_to(base).parts) + and (path.suffix in {".py", ".cjs", ".json"} or path.name == "requirements.txt") + ) + digest = hashlib.sha256() + for path in files: + digest.update(path.relative_to(base).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + packages: dict[str, str] = {} + requirements = base / "requirements.txt" + if requirements.is_file(): + for line in requirements.read_text(encoding="utf-8").splitlines(): + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", line.split("#", 1)[0]) + if match: + try: + packages[match.group(1)] = metadata.version(match.group(1)) + except metadata.PackageNotFoundError: + packages[match.group(1)] = "absent" + modules = _resolved_reader_modules(list(packages), metadata) + node = {"node": "absent", "typescript": "absent"} + env = {key: value for key, value in os.environ.items() if key not in {"NODE_OPTIONS", "NODE_PATH"}} + try: + probe = subprocess.run(["node", "-e", _NODE_PROBE], cwd=base, env=env, capture_output=True, text=True, check=False) + if probe.returncode == 0: + node = {key: str(value) for key, value in json.loads(probe.stdout).items()} + except (OSError, ValueError): + pass + return { + "source_sha256": digest.hexdigest(), + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "python_packages": packages, + "python_modules": modules, + "node": node["node"], + "typescript": node["typescript"], + } + + +def generator_identity(base: Path | None = None) -> str: + """Return ``sha256:<hex>`` over :func:`generator_identity_components`.""" + components = generator_identity_components(base) + return "sha256:" + _sha256(json.dumps(components, sort_keys=True, separators=(",", ":"))) + + +# Schema-2 helper shape revision the renderer requires; see +# MSDMD_COLLECTION_HELPER_VERSION in collection.ts. +REQUIRED_HELPER_VERSION = 1 +_HELPER_VERSION_RE = re.compile(r"^export const MSDMD_COLLECTION_HELPER_VERSION\s*(?::\s*number\s*)?=\s*(\d+)\s*;", re.MULTILINE) + + +def helper_incompatibility(out: Path, import_path: str, rendered: str, root: Path | None = None) -> tuple[str | None, bool]: + """Check that the target's schema helper can type-check ``rendered``. + + Schema-2 output requires the helper's exported + ``MSDMD_COLLECTION_HELPER_VERSION`` to be at least + ``REQUIRED_HELPER_VERSION``; helpers without it (schema 1, or schema 2 + from before the constant) are refused. Schema-1 output only needs + ``defineMsdmdCollection``. ``out`` is resolved to an absolute path first. + When ``out`` lies outside ``root`` (for example a CI scratch file under + /tmp), the helper is located from ``root``, where the in-tree artifact and + its configured or default helper live, so the version check still runs. + Returns ``(problem, found)``. Bare module specifiers are not resolved. + """ + if not import_path.startswith((".", "/")): + return None, False + first = rendered.split("\n", 1)[0] + match = re.match(r"import \{ ([^}]+) \} from ", first) + if not match: + return None, False + names = [name.strip() for name in match.group(1).split(",") if name.strip()] + out = out.resolve() + anchor = out.parent + if root is not None and not out.is_relative_to(root.resolve()): + anchor = root.resolve() + base = (anchor / import_path) if not import_path.startswith("/") else Path(import_path) + candidates = [base.with_name(base.name + ".ts"), base.with_name(base.name + ".d.ts"), base / "index.ts"] + if base.suffix == ".ts": + candidates.insert(0, base) + remedy = "propagate the current msdmd skill to the target or regenerate with --legacy-blocks-only" + for candidate in candidates: + if candidate.is_file(): + text = candidate.read_text(encoding="utf-8", errors="replace") + if "defineMsdmdCollectionV2" in names: + version = _HELPER_VERSION_RE.search(text) + found_version = int(version.group(1)) if version else 0 + if found_version < REQUIRED_HELPER_VERSION: + return (f"schema helper {candidate} has MSDMD_COLLECTION_HELPER_VERSION " + f"{found_version if version else 'missing'}; schema-2 output requires " + f"{REQUIRED_HELPER_VERSION} or newer (defineMsdmdCollectionV2 with current types); {remedy}"), True + return None, True + missing = [name for name in names if not re.search(r"\bexport\b[^;]*\b" + re.escape(name) + r"\b", text)] + if missing: + return f"schema helper {candidate} does not export {', '.join(missing)}; {remedy}", True + return None, True + return None, False + + def render_typescript(collection: dict[str, Any], *, import_path: str) -> str: """Render a schema-1 or schema-2 collection as a TypeScript module.""" helper = "defineMsdmdCollectionV2" if collection.get("schema_version") == SCHEMA_VERSION else "defineMsdmdCollection" @@ -743,16 +1146,32 @@ def render_typescript(collection: dict[str, Any], *, import_path: str) -> str: return f'import {{ {imports} }} from "{import_path}";\n\nexport default {helper}({payload});\n' +def _positive_int(value: str) -> int: + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return number + + +def _skill_dir_hint() -> str: + """Return where this skill actually lives, relative to the working directory when possible.""" + here = Path(__file__).resolve().parent + try: + return os.path.relpath(here) + except ValueError: + return str(here) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=Path("."), help="repository root to scan") - parser.add_argument("--repo", required=True, help="repository slug for the collection") + parser.add_argument("--repo", help="repository slug for the collection (required unless --print-generator-identity)") parser.add_argument("--out", type=Path, help="output .ts path; stdout when omitted") parser.add_argument("--block", action="append", dest="blocks", help="supplemental block name to collect; may be repeated") parser.add_argument("--expected-block", action="append", default=[], help="block expected on every supported line-comment file for adoption reporting") parser.add_argument("--import-path", default="./.agents/skills/msdmd/collection", help="TypeScript import path for the schema helper") parser.add_argument("--source-commit", help="source commit SHA to record; current git HEAD is detected when omitted") - parser.add_argument("--max-file-bytes", type=int, default=DEFAULT_MAX_FILE_BYTES, help="per-file read boundary") + parser.add_argument("--max-file-bytes", type=_positive_int, default=DEFAULT_MAX_FILE_BYTES, help="per-file read boundary") parser.add_argument("--legacy-blocks-only", action="store_true", help="emit explicit schema-1 block-only compatibility output") parser.add_argument("--snapshot-identity", action="store_true", help="bind generated output to source bytes rather than a circular output commit") parser.add_argument("--require-source", action="append", default=[], help="glob that must have matched, fully extracted inputs; repeat as needed") @@ -760,7 +1179,18 @@ def main() -> int: parser.add_argument("--json", action="store_true", help="emit raw collection JSON instead of TypeScript") parser.add_argument("--check", action="store_true", help="compare --out without writing; nonzero on drift") parser.add_argument("--strict", action="store_true", help="exit nonzero after rendering if invalid identity or syntax diagnostics exist") + parser.add_argument("--max-total-bytes", type=_positive_int, default=DEFAULT_MAX_TOTAL_BYTES, help="aggregate bound on source bytes retained for readers") + parser.add_argument("--allow-missing-reader-runtimes", action="store_true", help="write incomplete output when native reader runtimes are absent (still warns)") + parser.add_argument("--print-generator-identity", action="store_true", help="print the collector identity (sources plus Python, reader package, Node and TypeScript versions) and exit; add --json for components") args = parser.parse_args() + if args.print_generator_identity: + if args.json: + print(json.dumps(generator_identity_components(), indent=2, sort_keys=True)) + else: + print(generator_identity()) + return 0 + if not args.repo: + parser.error("--repo is required") required_facts = [] for value in args.require_fact: @@ -779,6 +1209,15 @@ def main() -> int: source_commit=args.source_commit, ) else: + excluded_inodes = [] + if not args.out: + # A shell redirect into the scanned tree is the output, not an input. + try: + info = os.fstat(sys.stdout.fileno()) + except (OSError, ValueError, AttributeError): + info = None + if info is not None and stat.S_ISREG(info.st_mode): + excluded_inodes.append((info.st_dev, info.st_ino)) collection = collect( args.root, args.repo, @@ -790,8 +1229,44 @@ def main() -> int: generated_outputs=[args.out.resolve().relative_to(args.root.resolve()).as_posix()] if args.out and args.out.resolve().is_relative_to(args.root.resolve()) else [], required_sources=args.require_source, required_facts=required_facts, + max_total_bytes=args.max_total_bytes, + excluded_inodes=excluded_inodes, ) rendered = json.dumps(collection, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" if args.json else render_typescript(collection, import_path=args.import_path) + # Report every blocking problem from one run before choosing an exit status. + exit_status = 0 + visibility = (_git_visibility(args.root.resolve())[2] if args.legacy_blocks_only + else [item for item in collection.get("diagnostics", []) if item.get("code") in VISIBILITY_FAILURE_CODES]) + for item in visibility: + print(f"msdmd: ERROR: {item['code']}: {item['message']}", file=sys.stderr) + if visibility: + # A collection that could not see the source tree is never written or compared. + exit_status = 5 + missing_runtimes = [] if args.legacy_blocks_only else runtime_unavailable(collection) + if missing_runtimes: + readers = sorted({str(item.get("reader_id")) for item in missing_runtimes}) + files = {str(item.get("source", {}).get("file")) for item in missing_runtimes} + skill = _skill_dir_hint() + print(f"msdmd: ERROR: native reader runtime unavailable ({', '.join(readers)}) for {len(files)} file(s); " + f"the collection is incomplete. Install with: python -m pip install -r {os.path.join(skill, 'requirements.txt')} " + f"&& npm ci --ignore-scripts --prefix {skill} (Node required). To write incomplete output anyway, " + "pass --allow-missing-reader-runtimes.", file=sys.stderr) + if args.allow_missing_reader_runtimes: + print("msdmd: WARNING: --allow-missing-reader-runtimes set; writing incomplete output.", file=sys.stderr) + else: + exit_status = exit_status or 3 + if args.out and not args.json: + problem, found = helper_incompatibility(args.out, args.import_path, rendered, args.root) + if problem: + print("msdmd: ERROR: " + problem, file=sys.stderr) + exit_status = exit_status or 4 + elif not found and args.import_path.startswith("."): + print(f"msdmd: WARNING: schema helper {args.import_path} not found", file=sys.stderr) + if exit_status: + return exit_status + budget = [item for item in collection.get("diagnostics", []) if item.get("code") == "aggregate_size_limit"] + if budget: + print("msdmd: WARNING: " + budget[0]["message"] + "; raise --max-total-bytes to read them.", file=sys.stderr) if args.check: if not args.out: parser.error("--check requires --out") @@ -817,4 +1292,4 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) -# ratios: loc_comments=706:54 imports_exports=17:5 calls_definitions=213:21 +# ratios: loc_comments=1064:134 imports_exports=21:9 calls_definitions=374:33 diff --git a/.agents/skills/msdmd/collection.ts b/.agents/skills/msdmd/collection.ts index 9d13533b..251e1f9f 100644 --- a/.agents/skills/msdmd/collection.ts +++ b/.agents/skills/msdmd/collection.ts @@ -10,6 +10,14 @@ * native facts and must never receive a silent lossy projection. */ +/** + * Schema-2 shape revision of this helper. The collector refuses to write a + * schema-2 artifact against a helper whose revision is older than it renders + * (exit 4), so a stale vendored helper cannot fail tsc after regeneration. + * Revision 1 adds `runtime-unavailable` reader runs and submodule ledger entries. + */ +export const MSDMD_COLLECTION_HELPER_VERSION = 1; + export type JsonPrimitive = string | number | boolean | null; export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; @@ -146,7 +154,7 @@ export interface MsdmdReaderRun { reader_id: string; reader_version: string; support: MsdmdReaderManifest["support"]; - status: "applied" | "not-applicable"; + status: "applied" | "not-applicable" | "runtime-unavailable"; files_matched: number; facts_emitted: number; } @@ -156,9 +164,11 @@ export interface MsdmdDiscoveryEntry { status: "supported" | "partial" | "unsupported" | "invalid" | "excluded" | "unreadable"; reader_ids: string[]; content_sha256: string; - entry_kind?: "file" | "subtree" | "output"; + entry_kind?: "file" | "subtree" | "output" | "submodule"; size?: number; reason?: string; + /** Pinned gitlink commit for an excluded git submodule. */ + commit?: string; } export interface MsdmdEdgeV2 { diff --git a/.agents/skills/msdmd/native_code.py b/.agents/skills/msdmd/native_code.py index ba276be4..5c78b6e0 100644 --- a/.agents/skills/msdmd/native_code.py +++ b/.agents/skills/msdmd/native_code.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=195:14 imports_exports=15:3 calls_definitions=83:3 +# ratios: loc_comments=200:16 imports_exports=15:3 calls_definitions=84:3 """Syntax-aware code readers for the unified collection. Usage: registry calls read_python/read_typescript with bounded bytes and context. @@ -167,8 +167,15 @@ def read_typescript(path: Path, data: bytes, context: dict[str, Any]) -> tuple[l result = subprocess.run(['node', str(helper)], input=json.dumps({'path': str(path), 'text': data.decode('utf-8-sig')}), capture_output=True, text=True, cwd=helper.parent, env=env, check=False) if result.returncode: - return [], [], [_diagnostic(context, reader_id=rid, code='typescript_reader_unavailable', - message='trusted TypeScript worker failed; install Node and run npm ci --prefix msdmd', status='unsupported')] + if "Cannot find module 'typescript'" in result.stderr: + return [], [], [_diagnostic(context, reader_id=rid, code='typescript_reader_unavailable', + message='TypeScript compiler package not installed for the trusted worker; run npm ci --prefix <msdmd skill dir>', + status='unsupported')] + # Any other worker failure is a reader error, not a missing runtime. Worker + # stderr is not published because it can quote inspected source text. + return [], [], [_diagnostic(context, reader_id=rid, code='typescript_reader_failed', + message=f'trusted TypeScript worker exited with status {result.returncode}; input not extracted', + status='invalid', severity='error')] parsed = json.loads(result.stdout) context = dict(context, convention_version=parsed['version'], dialect='typescript-compiler-jsdoc') facts: list = [] @@ -217,4 +224,4 @@ def read_typescript(path: Path, data: bytes, context: dict[str, Any]) -> tuple[l for target in targets or [fallback]: edges.append(_edge(fact['subject']['address'], target, 'exports:' + item['exported_name'])) return facts, edges, diagnostics -# ratios: loc_comments=195:14 imports_exports=15:3 calls_definitions=83:3 +# ratios: loc_comments=200:16 imports_exports=15:3 calls_definitions=84:3 diff --git a/.agents/skills/msdmd/readers.py b/.agents/skills/msdmd/readers.py index fdb54d9d..4224634b 100644 --- a/.agents/skills/msdmd/readers.py +++ b/.agents/skills/msdmd/readers.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=889:48 imports_exports=22:5 calls_definitions=267:28 +# ratios: loc_comments=964:66 imports_exports=22:5 calls_definitions=300:32 # === MODULE_BUILD === # id: msdmd_native_reader_registry # module_name: readers @@ -319,7 +319,31 @@ _SHELL_SUFFIXES = {".sh", ".bash", ".zsh", ".fish"} _SYSTEMD_SUFFIXES = {".service", ".socket", ".timer", ".path", ".target"} _GENERATED_MARKER = "Generated by tools/build_codex_plugin_skills.py; do not edit." -_SENSITIVE_KEY_RE = re.compile(r"(?:^|[_-])(?:secret|token|password|passwd|api[_-]?key|private[_-]?key|credential|client[_-]?secret|access[_-]?key)(?:$|[_-])", re.IGNORECASE) +_SENSITIVE_KEY_RE = re.compile(r"(?:^|[_-])(?:secret|token|password|passwd|authorization|api[_-]?key|private[_-]?key|credential|client[_-]?secret|access[_-]?key)(?:$|[_-])", re.IGNORECASE) +_CAMEL_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") +_URL_USERINFO_RE = re.compile(r"(\b[a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]+@") +# camel/Pascal names are secret-bearing only when the sensitive word is the +# final token, so modifiers such as tokenUrl, passwordHash, apiKeyPrefix, +# accessKeyId, tokenCount, tokenLimit, tokenType, secretName and +# privateKeyPath describe a secret without carrying one and are kept. +_SENSITIVE_FINAL_TOKENS = frozenset({"secret", "token", "password", "passwd", "authorization", "credential", "credentials"}) +_SENSITIVE_FINAL_PAIRS = frozenset({("api", "key"), ("private", "key"), ("access", "key"), ("secret", "key")}) + + +def _is_sensitive_key(name: Any) -> bool: + """Match secret-bearing names in snake, kebab, camel and Pascal case.""" + text = str(name) + if _SENSITIVE_KEY_RE.search(text): + return True + tokens = [token.lower() for token in re.split(r"[^A-Za-z0-9]+", _CAMEL_BOUNDARY_RE.sub("_", text)) if token] + if len(tokens) < 2: + return False + return tokens[-1] in _SENSITIVE_FINAL_TOKENS or (tokens[-2], tokens[-1]) in _SENSITIVE_FINAL_PAIRS + + +def _sanitize_url_credentials(value: str) -> str: + """Withhold URL userinfo such as ``scheme://user:password@host``.""" + return _URL_USERINFO_RE.sub(r"\1[redacted]@", value) def _digest(value: bytes | str) -> str: @@ -349,7 +373,7 @@ def _redact_sensitive(value: Any, pointer: str = "") -> tuple[Any, list[str]]: result: dict[str, Any] = {} for key, item in value.items(): child_pointer = f"{pointer}/{key}" - if _SENSITIVE_KEY_RE.search(str(key)): + if _is_sensitive_key(key): result[str(key)] = {"$redacted": True, "reason": "sensitive-field-name"} redacted.append(child_pointer) else: @@ -364,7 +388,7 @@ def _redact_sensitive(value: Any, pointer: str = "") -> tuple[Any, list[str]]: redacted.extend(child_redacted) return result_list, redacted if isinstance(value, str): - sanitized = re.sub(r"(\b[a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]+@", r"\1[redacted]@", value) + sanitized = _sanitize_url_credentials(value) if sanitized != value: return sanitized, [pointer] return value, redacted @@ -700,14 +724,23 @@ def _redact_systemd_environment(value: str) -> tuple[str, bool, bool]: name, separator, assigned = word.partition('=') if not separator or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name): return '<redacted>', True, True - if _SENSITIVE_KEY_RE.search(name): + if _is_sensitive_key(name): parts.append(quote_char + name + '=<redacted>' + quote_char) redacted = True else: - parts.append(value[start:position]) + sanitized = _sanitize_url_credentials(assigned) + if sanitized != assigned: + parts.append(quote_char + name + '=' + sanitized + quote_char) + redacted = True + else: + parts.append(value[start:position]) return ''.join(parts), redacted, False +# systemd.exec credential directives whose value carries literal secret data. +_SYSTEMD_CREDENTIAL_DATA_KEYS = {'setcredential', 'setcredentialencrypted'} + + def _read_systemd(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: reader_id = "systemd-unit" context = dict(context, convention_version="systemd.syntax current documented grammar", dialect=path.suffix.lstrip(".")) @@ -741,15 +774,26 @@ def _read_systemd(path: Path, data: bytes, context: dict[str, Any]) -> tuple[lis continue key, value = (part.strip() for part in line.split("=", 1)) published_value = value + location = {'start_line': logical_start, 'end_line': number} if key.lower() == 'environment': published_value, redacted, unsupported = _redact_systemd_environment(value) - location = {'start_line': logical_start, 'end_line': number} if redacted: diagnostics.append(_diagnostic(context, reader_id=reader_id, code='sensitive_fields_redacted', message='withheld potentially sensitive systemd Environment values', status='redacted', location=location)) if unsupported: diagnostics.append(_diagnostic(context, reader_id=reader_id, code='unsupported_systemd_environment_syntax', message='Environment quoting or escaping is outside the supported subset; value withheld', status='unsupported', location=location)) + elif key.lower() in _SYSTEMD_CREDENTIAL_DATA_KEYS: + # ID:DATA -- keep the credential identifier, never its data. + credential_id, separator, _ = value.partition(':') + published_value = credential_id + ':<redacted>' if separator else '<redacted>' + diagnostics.append(_diagnostic(context, reader_id=reader_id, code='sensitive_fields_redacted', + message='withheld systemd credential data', status='redacted', location=location)) + else: + published_value = _sanitize_url_credentials(value) + if published_value != value: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code='sensitive_fields_redacted', + message='withheld systemd URL credentials', status='redacted', location=location)) identity_key = (section, key) occurrence[identity_key] = occurrence.get(identity_key, 0) + 1 identity = f"{section}.{key}[{occurrence[identity_key]}]" @@ -786,6 +830,51 @@ def _read_gitignore(path: Path, data: bytes, context: dict[str, Any]) -> tuple[l _REQUIREMENT_NAME_RE = re.compile(r"^(?P<name>[A-Za-z0-9][A-Za-z0-9._-]*)") +# Direct URL, VCS and local-path forms carry no leading distribution name. +_REQUIREMENT_DIRECT_RE = re.compile( + r"^(?:[A-Za-z][A-Za-z0-9+.-]*://|(?:git|hg|svn|bzr)\+|file:|\.{1,2}(?:/|\\|$)|/|\\|~|[A-Za-z]:[\\/])", re.IGNORECASE) +_REQUIREMENT_ARCHIVE_RE = re.compile(r"\.(?:whl|zip|tar\.gz|tar\.bz2|tar\.xz|tgz|tar)(?:[#?]|$)", re.IGNORECASE) +# PEP 508 direct reference with an explicit name: ``name[extras] @ url``. +_REQUIREMENT_NAMED_REFERENCE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\s*(?:\[[^\]]*\])?\s*@") + + +def _is_unnamed_direct_requirement(line: str) -> bool: + if _REQUIREMENT_NAMED_REFERENCE_RE.match(line): + return False + first = line.split(maxsplit=1)[0] if line.split() else "" + return bool(_REQUIREMENT_DIRECT_RE.match(first) or _REQUIREMENT_ARCHIVE_RE.search(first)) + + +def _requirement_logical_lines(text: str) -> tuple[list[tuple[int, int, str]], int | None]: + """Join pip backslash continuations into ``(start, end, line)`` records. + + As in pip, a line whose stripped text starts with ``#`` never continues, + so a commented line ending in a backslash cannot swallow the next + requirement. Returns the records and the line of a dangling trailing + continuation at end of file, if any. + """ + records: list[tuple[int, int, str]] = [] + pending: str | None = None + start = 0 + line_number = 0 + for line_number, raw in enumerate(text.splitlines(), start=1): + if pending is None: + start = line_number + pending = "" + if raw.endswith("\\") and not raw.lstrip().startswith("#"): + pending += raw[:-1] + continue + if raw.lstrip().startswith("#") and pending: + # A comment line ends the continuation and contributes nothing. + records.append((start, line_number, pending)) + else: + records.append((start, line_number, pending + raw)) + pending = None + dangling = None + if pending is not None: + records.append((start, line_number, pending)) + dangling = line_number + return records, dangling def _read_requirements(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[dict], list[dict], list[dict]]: @@ -800,24 +889,37 @@ def _read_requirements(path: Path, data: bytes, context: dict[str, Any]) -> tupl diagnostics: list[dict] = [] subject = _subject_address(context, "requirements-document", context["file"]) order = 0 - for line_number, raw in enumerate(text.splitlines(), start=1): + records, dangling = _requirement_logical_lines(text) + if dangling is not None: + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="dangling_requirement_continuation", + message="file ends with a backslash continuation; the final requirement may be incomplete", + status="ambiguous", severity="warning", location={"start_line": dangling, "end_line": dangling})) + for start_line, line_number, raw in records: stripped = raw.strip() if not stripped or stripped.startswith("#"): continue order += 1 + location = {"start_line": start_line, "end_line": line_number} published = re.sub(r"(?<=://)[^/@\s]+@", "<redacted>@", stripped) if published != stripped: - diagnostics.append(_diagnostic(context, reader_id=reader_id, code="sensitive_fields_redacted", message="withheld requirement URL credentials", status="redacted", severity="warning", location={"start_line": line_number, "end_line": line_number})) + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="sensitive_fields_redacted", message="withheld requirement URL credentials", status="redacted", severity="warning", location=location)) if published.startswith("-"): if published.startswith(('-r', '-c')) or re.match(r'^--(?:requirement|constraint)(?:[=\s]|$)', published): diagnostics.append(_diagnostic(context, reader_id=reader_id, code='unresolved_requirement_include', message='included requirements/constraints are declared but not resolved', status='dynamic-unresolved', - location={'start_line': line_number, 'end_line': line_number})) - facts.append(_fact(context, reader_id=reader_id, kind="requirements-option", scope="requirements-document", identity=f"option:{order}", location={"start_line": line_number, "end_line": line_number}, value={"text": published, "order": order}, native_id=f"option:{order}", convention="python.requirements-file", standing="declared")) + location=location)) + facts.append(_fact(context, reader_id=reader_id, kind="requirements-option", scope="requirements-document", identity=f"option:{order}", location=location, value={"text": published, "order": order}, native_id=f"option:{order}", convention="python.requirements-file", standing="declared")) continue - match = _REQUIREMENT_NAME_RE.match(published) - name = match.group("name") if match else "hmmm" - facts.append(_fact(context, reader_id=reader_id, kind="dependency", scope="requirements-document", identity=f"requirement:{order}", location={"start_line": line_number, "end_line": line_number}, value={"requirement": published, "name": name, "class": "runtime-or-unresolved", "order": order}, native_id=name if name != "hmmm" else None, convention="python.requirements-file", standing="declared", projection={"mapping_version": "requirements-lines@1", "relation": "depends_on", "loss": "resolution and environment markers not evaluated"})) + if _is_unnamed_direct_requirement(published): + # A URL/VCS/path requirement names no distribution; never invent one. + name = "hmmm" + diagnostics.append(_diagnostic(context, reader_id=reader_id, code="unresolved_direct_requirement_name", + message="direct URL, VCS or path requirement declares no distribution name; name not inferred", + status="dynamic-unresolved", location=location)) + else: + match = _REQUIREMENT_NAME_RE.match(published) + name = match.group("name") if match else "hmmm" + facts.append(_fact(context, reader_id=reader_id, kind="dependency", scope="requirements-document", identity=f"requirement:{order}", location=location, value={"requirement": published, "name": name, "class": "runtime-or-unresolved", "order": order}, native_id=name if name != "hmmm" else None, convention="python.requirements-file", standing="declared", projection={"mapping_version": "requirements-lines@1", "relation": "depends_on", "loss": "resolution and environment markers not evaluated"})) if name != "hmmm": edges.append(_edge(subject, f"python-package:{name}", "depends_on:runtime-or-unresolved")) return facts, edges, diagnostics @@ -864,9 +966,10 @@ def _read_svg(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[di title = ''.join(child.itertext()).strip() elif child_name == 'desc' and description is None: description = ''.join(child.itertext()).strip() - value = {'attributes': dict(root.attrib), 'title': title, 'description': description} + value, redactions = _redact_sensitive({'attributes': dict(root.attrib), 'title': title, 'description': description}) + diagnostic = _redaction_diagnostic(context, reader_id, redactions) return [_fact(context, reader_id=reader_id, kind='document-metadata', scope='document', identity=context['file'], - location={'pointer': '/svg'}, value=value, convention='svg.document-metadata')], [], [] + location={'pointer': '/svg'}, value=value, convention='svg.document-metadata')], [], [diagnostic] if diagnostic else [] _LLMS_HEADING_RE = re.compile(r"^(?P<marks>#{1,6})\s+(?P<title>.+?)\s*$") @@ -1003,7 +1106,8 @@ def read_native(path: Path, data: bytes, context: dict[str, Any]) -> tuple[list[ # adds a more precise root classification than the generic syntax reader. roots = [f for f in facts if f["extraction"]["reader_id"] == "metadata-standards" and f["source"]["location"] == {"pointer": ""}] if roots: + superseding = any((root.get("projection") or {}).get("supersedes") == "structured-document" for root in roots) facts = [f for f in facts if not (f["kind"] == "structured-document" and f["extraction"]["reader_id"] != "metadata-standards" - and any(f["native"]["value"] == root["native"]["value"] for root in roots))] + and (superseding or any(f["native"]["value"] == root["native"]["value"] for root in roots)))] return reader_ids, facts, edges, diagnostics -# ratios: loc_comments=889:48 imports_exports=22:5 calls_definitions=267:28 +# ratios: loc_comments=964:66 imports_exports=22:5 calls_definitions=300:32 diff --git a/.agents/skills/msdmd/references/implemented-readers.md b/.agents/skills/msdmd/references/implemented-readers.md index bee2dae5..c8a7f656 100644 --- a/.agents/skills/msdmd/references/implemented-readers.md +++ b/.agents/skills/msdmd/references/implemented-readers.md @@ -77,3 +77,23 @@ resolution, arbitrary-language support and independent evidence verification. - Maven XML recognition requires its POM namespace or an unnamespaced project with modelVersion 4.0.0. Other project XML stays generic XML. - SVG uses the same bounded, UTF-8, DTD/entity-rejecting parser as other XML inputs. - Explicit YAML `!!float 1` is supported and is a regression control, not an outstanding defect. + +`tests/test_msdmd_consumer_review.py` closes consumer-review findings: + +- Candidate and verifier renders written beside the artifact under different temporary names are byte-identical; neither name nor a redirected stdout file enters discovery or worktree state. +- Git-ignored files are not read. Retained reader bytes have an aggregate budget. +- camelCase/PascalCase secret keys, systemd URL credentials and `SetCredential` data, SVG metadata and raw DSSE payloads are redacted. +- Direct URL, VCS and path requirements keep an unresolved name and no package edge; backslash continuations form one requirement. +- Missing reader runtimes are errors; a schema-1 target helper is refused; the generator identity covers the TypeScript worker and lock file. + +`tests/test_msdmd_review_followup.py` closes the follow-up review: + +- A comment line ending in a backslash never continues; a trailing continuation at end of file is diagnosed; Windows paths are unnamed requirements. +- A git listing failure, broken `.git` marker or root ignored by an enclosing repository is an error with an incomplete snapshot; submodules are excluded ledger entries with their pinned commit. +- Schema-2 output requires `MSDMD_COLLECTION_HELPER_VERSION`; runtime and helper problems are reported together; only a missing `typescript` package is a missing TypeScript runtime. +- The generator identity includes Python minor, reader package, Node and TypeScript versions. +- A DSSE payload that is not an in-toto Statement is diagnosed and kept as a decoded `signed-payload` fact. +- camelCase names are secret-bearing only when the final word is sensitive; `authorization` is sensitive. +- ratios semantic graphs do not resolve ambiguous targets; worktree renames in either porcelain column are parsed. +- Git visibility failures (no git on PATH, ignored root) stop the CLI with exit 5 without writing; `--check` reports 5, not drift. +- With `--out` outside the repository, the schema helper is located from the root; reader module files that actually resolve are part of the generator identity. diff --git a/.agents/skills/msdmd/standards.py b/.agents/skills/msdmd/standards.py index d021cbf7..821d5537 100644 --- a/.agents/skills/msdmd/standards.py +++ b/.agents/skills/msdmd/standards.py @@ -1,4 +1,4 @@ -# ratios: loc_comments=254:13 imports_exports=8:2 calls_definitions=146:5 +# ratios: loc_comments=265:18 imports_exports=8:2 calls_definitions=149:5 """Semantic projections for native metadata documents, preserving owning trees. Usage: registry invokes read_standards(path, bytes, context). The JSON/YAML/TOML @@ -196,10 +196,14 @@ def components(items: list, loc: str) -> None: components(value.get('components', []), '/components') statement = value statement_pointer = '' + envelope_fact = None if value.get('payloadType') == 'application/vnd.in-toto+json' and isinstance(value.get('payload'), str): statement = parse_json(base64.b64decode(value['payload'], validate=True)) statement_pointer = '/payload' - emit('dsse.envelope', 'v1', 'signed-envelope', '', value, scope='attestation', standing='reported-evidence') + # The decoded statement is published (redacted) at /payload; never also + # publish the raw base64 payload, which would reverse that redaction. + envelope = dict(value, payload={'$withheld': True, 'reason': 'decoded-statement-projected', 'pointer': '/payload'}) + envelope_fact = emit('dsse.envelope', 'v1', 'signed-envelope', '', envelope, scope='attestation', standing='reported-evidence') unsupported('unverified_signature', 'envelope parsed; signature verification was not requested or performed', status='unverified') if isinstance(statement, dict) and str(statement.get('_type', '')).startswith('https://in-toto.io/Statement/'): version = str(statement['_type']) @@ -212,6 +216,18 @@ def components(items: list, loc: str) -> None: for i, subject in enumerate(statement.get('subject', [])): emit('in-toto.statement', version, 'attested-subject', statement_pointer + pointer('subject', i), subject, scope='artifact', identity=subject.get('name', str(i)), native_id=subject.get('name'), standing='reported-evidence') + elif envelope_fact is not None: + # Declared in-toto payload that is not a Statement: keep the decoded + # (redacted) payload as its own fact rather than claiming a statement. + unsupported('dsse_payload_not_in_toto_statement', + 'payloadType declares in-toto JSON but the decoded payload has no in-toto Statement _type; payload kept unprojected', + status='ambiguous') + fact = emit('dsse.envelope', 'v1', 'signed-payload', '/payload', statement, scope='attestation', standing='reported-evidence') + fact['projection'] = {'mapping_version': 'dsse-base64-json@1', 'loss': 'decoded payload not interpreted; signature not verified'} + if envelope_fact is not None: + # Supersede the generic syntax view only once a decoded replacement exists. + envelope_fact['projection'] = {'mapping_version': 'dsse-envelope@1', 'supersedes': 'structured-document', + 'loss': 'raw base64 payload withheld; decoded payload published at /payload'} if path.name in {'package-lock.json', 'npm-shrinkwrap.json'}: version = str(value.get('lockfileVersion', 'hmmm')) emit('npm.lockfile', version, 'lockfile', '', value, scope='package') @@ -281,4 +297,4 @@ def refs(item: Any, loc: str) -> None: code='duplicate_native_identifier', message='duplicate native ID in ' + namespace, status='invalid', severity='error')) return facts, edges, diagnostics -# ratios: loc_comments=254:13 imports_exports=8:2 calls_definitions=146:5 +# ratios: loc_comments=265:18 imports_exports=8:2 calls_definitions=149:5