From 619bb2e362cbea728a604f2e9e435e7083f7ea7b Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 15:05:28 -0400 Subject: [PATCH 01/19] Classify documentation and publish readiness from one engine result Adds evidence-bounded three-fact classification (role, user-owned never-edit, repository status) driven by a fail-closed docs:policy block in the committed map; readiness v3 with honest percentage absence; deterministic unresolved cards with stateless single-use DOCS-R- resolution preview/apply; treatment write-eligibility policy; Init persistence of approved never-edit choices; corrected public evidence documents; and all adapters regenerated from canonical sources. Verified locally: 1,142 tests across core/lifecycle/trajectory, 54 PR 3 contracts, blind whole-PR review findings fixed and re-verified. Two expected failures remain assigned to the beginner-interface phase. --- ARCHITECTURE.md | 13 +- BENCHMARK.md | 12 +- COMMANDS.md | 8 +- EVALUATION.md | 14 +- PRODUCT.md | 107 +- README.md | 4 +- ROADMAP.md | 12 +- .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../docs/scripts/_docs_checker/manifest.py | 92 +- .../docs/scripts/_docs_checker/measurement.py | 1 + .../docs/scripts/_docs_checker/readiness.py | 229 +++ .../docs/scripts/_docs_checker/resolution.py | 426 +++++ .../skills/docs/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ adapters/claude/skills/docs/scripts/check.py | 58 +- .../skills/docs/scripts/init_closeout.py | 11 + .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../copilot/scripts/_docs_checker/manifest.py | 92 +- .../scripts/_docs_checker/measurement.py | 1 + .../scripts/_docs_checker/readiness.py | 229 +++ .../scripts/_docs_checker/resolution.py | 426 +++++ .../copilot/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ adapters/copilot/scripts/check.py | 58 +- adapters/copilot/scripts/init_closeout.py | 11 + .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../cursor/scripts/_docs_checker/manifest.py | 92 +- .../scripts/_docs_checker/measurement.py | 1 + .../cursor/scripts/_docs_checker/readiness.py | 229 +++ .../scripts/_docs_checker/resolution.py | 426 +++++ adapters/cursor/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ adapters/cursor/scripts/check.py | 58 +- adapters/cursor/scripts/init_closeout.py | 11 + .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../grok/scripts/_docs_checker/manifest.py | 92 +- .../grok/scripts/_docs_checker/measurement.py | 1 + .../grok/scripts/_docs_checker/readiness.py | 229 +++ .../grok/scripts/_docs_checker/resolution.py | 426 +++++ adapters/grok/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ adapters/grok/scripts/check.py | 58 +- adapters/grok/scripts/init_closeout.py | 11 + .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../docs/scripts/_docs_checker/manifest.py | 92 +- .../docs/scripts/_docs_checker/measurement.py | 1 + .../docs/scripts/_docs_checker/readiness.py | 229 +++ .../docs/scripts/_docs_checker/resolution.py | 426 +++++ .../skills/docs/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ adapters/plugin/skills/docs/scripts/check.py | 58 +- .../skills/docs/scripts/init_closeout.py | 11 + docs/STATE.md | 20 +- .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- .../docs/scripts/_docs_checker/manifest.py | 92 +- .../docs/scripts/_docs_checker/measurement.py | 1 + .../docs/scripts/_docs_checker/readiness.py | 229 +++ .../docs/scripts/_docs_checker/resolution.py | 426 +++++ .../skills/docs/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ .../skills/docs/scripts/check.py | 58 +- .../skills/docs/scripts/init_closeout.py | 11 + .../scripts/_docs_checker/doctor_baseline.py | 26 +- .../_docs_checker/document_classification.py | 113 ++ .../_docs_checker/documentation_policy.py | 599 +++++++ .../_docs_checker/documentation_result.py | 117 ++ .../scripts/_docs_checker/init_adoption.py | 216 ++- .../scripts/_docs_checker/init_closeout.py | 25 +- skills/docs/scripts/_docs_checker/manifest.py | 92 +- .../docs/scripts/_docs_checker/measurement.py | 1 + .../docs/scripts/_docs_checker/readiness.py | 229 +++ .../docs/scripts/_docs_checker/resolution.py | 426 +++++ skills/docs/scripts/_docs_checker/scan.py | 85 +- .../scripts/_docs_checker/treatment_policy.py | 208 +++ skills/docs/scripts/check.py | 58 +- skills/docs/scripts/init_closeout.py | 11 + tests/test_adapters.py | 6 + tests/test_docs_checker_architecture.py | 54 +- tests/test_doctor_map_journey_parity.py | 90 +- tests/test_documentation_manifest.py | 27 +- tests/test_documentation_result_contract.py | 1390 +++++++++++++++++ tests/test_init_adoption_cli.py | 10 +- tests/test_resolution_treatment_contract.py | 859 ++++++++++ tools/build_adapters.py | 6 + 114 files changed, 17587 insertions(+), 487 deletions(-) create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/readiness.py create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/resolution.py create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py create mode 100644 adapters/copilot/scripts/_docs_checker/document_classification.py create mode 100644 adapters/copilot/scripts/_docs_checker/documentation_policy.py create mode 100644 adapters/copilot/scripts/_docs_checker/documentation_result.py create mode 100644 adapters/copilot/scripts/_docs_checker/readiness.py create mode 100644 adapters/copilot/scripts/_docs_checker/resolution.py create mode 100644 adapters/copilot/scripts/_docs_checker/treatment_policy.py create mode 100644 adapters/cursor/scripts/_docs_checker/document_classification.py create mode 100644 adapters/cursor/scripts/_docs_checker/documentation_policy.py create mode 100644 adapters/cursor/scripts/_docs_checker/documentation_result.py create mode 100644 adapters/cursor/scripts/_docs_checker/readiness.py create mode 100644 adapters/cursor/scripts/_docs_checker/resolution.py create mode 100644 adapters/cursor/scripts/_docs_checker/treatment_policy.py create mode 100644 adapters/grok/scripts/_docs_checker/document_classification.py create mode 100644 adapters/grok/scripts/_docs_checker/documentation_policy.py create mode 100644 adapters/grok/scripts/_docs_checker/documentation_result.py create mode 100644 adapters/grok/scripts/_docs_checker/readiness.py create mode 100644 adapters/grok/scripts/_docs_checker/resolution.py create mode 100644 adapters/grok/scripts/_docs_checker/treatment_policy.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py create mode 100644 skills/docs/scripts/_docs_checker/document_classification.py create mode 100644 skills/docs/scripts/_docs_checker/documentation_policy.py create mode 100644 skills/docs/scripts/_docs_checker/documentation_result.py create mode 100644 skills/docs/scripts/_docs_checker/readiness.py create mode 100644 skills/docs/scripts/_docs_checker/resolution.py create mode 100644 skills/docs/scripts/_docs_checker/treatment_policy.py create mode 100644 tests/test_documentation_result_contract.py create mode 100644 tests/test_resolution_treatment_contract.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 56f75cd..7872be4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -11,7 +11,12 @@ The canonical product lives in `skills/docs/`. Everything under `adapters/` and - `_docs_checker/scan.py` measures maintained Markdown, MarkdownX, links, anchors, titles, and reachability without executing repository code. - `_docs_checker/paths.py` confines every path to the repository and rejects unsafe reparse boundaries. - `_docs_checker/selection.py` is the one place that decides which documentation a read command measures: the boundary, the map route, and — with the manifest — the entry route. The repository is the default; only an explicit human request narrows it, and that selection is recorded as partial. It reads directory listings and the repository inventory the manifest needs, never local operational state. -- `_docs_checker/manifest.py` builds the additive, immutable documentation manifest: repository-complete accounting, the recorded measured boundary with its map and provider entry routes, classification facts, exclusions, limits, and a deterministic identity over normalized routes and inspected-body digests. It reuses `health.normalized_content_digest` rather than defining a second content identity. Every read command route through `check.py` builds exactly one and publishes its identity, selected routes, and coverage completeness. `evidence_receipt.py` still takes its own explicit map and scope from the pinned corpus manifest and does not build one. +- `_docs_checker/manifest.py` builds the additive, immutable documentation manifest: repository-complete accounting, the recorded measured boundary with its map and provider entry routes, exclusions, limits, and a deterministic identity over normalized routes and inspected-body digests. It reuses `health.normalized_content_digest` rather than defining a second content identity. Every read command route through `check.py` builds exactly one. `evidence_receipt.py` still takes its own explicit map and scope from the pinned corpus manifest and does not build one. +- `_docs_checker/documentation_policy.py` parses the selected map's current working-tree `docs:policy` block, applies exact-over-directory precedence, preserves hand-authored order during edits, and reports committed, uncommitted, or non-Git provenance. +- `_docs_checker/document_classification.py` assigns each accounted document one evidence-bounded role plus independent `never_edit` and repository-status facts. Selected map/current-truth routes and declared generator evidence can decide a role; filenames, folders, and ordinary links cannot. +- `_docs_checker/readiness.py` calculates the unweighted readiness-v3 fraction from per-document reachability, link/anchor, title, and confinement facts. Trust and stored rubric-v2 state stay separate. +- `_docs_checker/documentation_result.py` assembles the one ordered `documentation` ledger, deterministic groups, unresolved cards, coverage, policy provenance, and write facts consumed by Map, Check, Doctor, Audit, and Init. +- `_docs_checker/resolution.py` previews and exactly applies one selected-map policy edit through a single-use short code and compare-before-write drift checks. `_docs_checker/treatment_policy.py` owns write and purpose-backed link eligibility without changing classification or readiness. - `_docs_checker/memory.py` reads the optional `.diataxis/` control plane. - `_docs_checker/lifecycle.py` decides whether a requested state change is authorized; `lifecycle_io.py` performs the transactional write. - `init_adoption.py` and the closeout modules bind a preview to the exact evidence that must still be true at apply time. @@ -22,12 +27,12 @@ The `.diataxis/` directory stores routes, stable finding identities, verificatio The adapter generator packages the same checker with thin host-specific entrypoints. Codex and Claude receive an umbrella plus focused command skills; static and web adapters receive only the command contract they need. Generated files are parity-checked and must never be edited by hand. -The structural percentage, Trust state, semantic findings, unavailable evidence, and byte telemetry remain separate evidence classes. An unsupported or ambiguous documentation surface is unmeasured rather than guessed. +Readiness v3, transitional rubric-v2 health, Trust state, semantic findings, unavailable evidence, and byte telemetry remain separate evidence classes. Readiness never compares itself with rubric-v2 state. An unsupported or ambiguous documentation surface is unmeasured or `unresolved` rather than guessed. -## The known structural defect +## Repaired read-model boundary and remaining integration Boundary selection used to be spread across several routes that could disagree: `check.py` defaulted an omitted `--scope` to `docs`, the stateless Doctor baseline ran its own discovery, an initialized Doctor used its recorded state, and the prompt contract asked the host model to supply the map and current-truth paths. Those routes now share `selection.py` and one manifest per invocation. -What remains: the public result envelope still carries no document inventory, so a displayed map tree is assembled by the model rather than by the engine, and every role outside the measured surface is still `unresolved` rather than classified. The reproduced evidence is in [current state](docs/STATE.md); the target is in the [product constitution](PRODUCT.md). +The public result envelope now carries the complete documentation ledger, classification, grouping, policy provenance, and readiness facts. What remains is integration at the interface boundary: the installed Map contract does not render that ledger, legacy rubric-v2 health remains beside readiness v3, and `context`, `write`, and `update` still lack matching deterministic engine entrypoints. The reproduced evidence is in [current state](docs/STATE.md); the target is in the [product constitution](PRODUCT.md). For module diagrams and design rationale, see the [Architecture wiki page](https://github.com/Statusnone420/Skills/wiki/Architecture). For measured limits and current proof, see [BENCHMARK.md](BENCHMARK.md). diff --git a/BENCHMARK.md b/BENCHMARK.md index a51b190..867ad5b 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,12 +6,12 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,087 tests — 457 core, 451 lifecycle, and 179 trajectory. The focused PR 2 correction suites pass locally; the complete current Windows matrix is delegated to PR CI and is not claimed here before it runs. The prior 1,059-test snapshot passed with 3 lifecycle cases skipped as POSIX-only and 6 strict expected failures naming the later phase that owns each unmet contract. | -| Windows lifecycle feedback | Five local shards passed the prior 438-test lifecycle snapshot in 182.1 s. The current 451-test lifecycle partition is delegated to PR CI. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | -| Generated adapter prompts | 3,557–34,971 UTF-8 bytes | +| Windows deterministic groups | Current partition: 1,142 tests — 511 core, 452 lifecycle, and 179 trajectory. The complete Stage 4B local matrix passed; the 54 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted CI remains unclaimed before a later authorized push. | +| Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | +| Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | -| Repository documentation check | The default and explicit `--scope .` routes both measure boundary `.`: 89%; 151/151 links valid; all 41 non-generated maintained pages reachable; remaining 20 unreachable and 7 duplicate-title findings are generated marketplace copies. | -| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,087-test snapshot has not run on Linux yet; fresh CI remains required before release. | +| Repository documentation check | Exit 0 with repository-complete accounting: 93 documentation paths — 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stayed unread. Readiness v3 is 2 of 2 eligible documents; that 100% eligible fraction sits beside 59 nonignored unresolved documents and is not whole-repository health. Transitional rubric v2 remains separately visible at 89% with 27 findings until the beginner-interface phase removes it. | +| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,142-test snapshot has not run on Linux yet; fresh CI remains required before release. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | | Installed-package treatment canary | Pass: exact stateless preview → one-file apply → candidate and installed verification clean; zero control writes, no `.diataxis/`, unchanged real Git index, and only the intended disposable worktree file changed | @@ -56,7 +56,7 @@ This proves routing and safety contracts for those fixtures. It does not prove a - A 2026-07-17 Claude/Cline canary showed that a tracked Mintlify/MDX corpus could be misread as empty. The repair made MarkdownX inert text, protected unsupported provider surfaces, and reports unmeasured evidence instead of a misleading zero. - A 2026-07-27 no-state dogfood run exposed an uninitialized-lifecycle conflict: Doctor produced a valid read-only scan, then closeout treated absent `.diataxis/` state as corruption and recovery offered no action. The 0.1.8 candidate now distinguishes a genuinely stateless repository from partial or malformed controls, and the installed-package retry passed. -- A 2026-07-27 measurement reproduced a command boundary divergence on this repository at commit `da02ba2`: two default command routes disagreed about which documentation exists, with no repository change between runs. The current PR 2 candidate repairs that divergence locally: the default checker, explicit repository boundary, first-contact Doctor, and Init routes consume the same repository selection and manifest identity. Hosted Linux and installed-host proof remain pending. The three-route comparison is tabulated in [docs/STATE.md](docs/STATE.md), against the contract in [PRODUCT.md](PRODUCT.md). +- A 2026-07-27 measurement reproduced a command boundary divergence on this repository at commit `da02ba2`: two default command routes disagreed about which documentation exists, with no repository change between runs. The current 0.1.8 candidate repairs that divergence locally: the default checker, explicit repository boundary, first-contact Doctor, and Init routes consume the same repository selection, documentation result, and readiness result. Hosted Linux and installed-host proof remain pending. The three-route comparison is tabulated in [docs/STATE.md](docs/STATE.md), against the contract in [PRODUCT.md](PRODUCT.md). - A 2026-07-27 adversarial audit reproduced the same dead end for non-transactional corruption: missing, malformed, merge-conflicted, or foreign-only `.diataxis` evidence failed closed but recovery returned `action: none`. The candidate now returns one zero-write operator action, never applies it automatically, and preserves exact-approved transaction recovery unchanged. Failures remain in this ledger after repair so the safety story cannot be rewritten as uninterrupted success. diff --git a/COMMANDS.md b/COMMANDS.md index c5ba379..99acfe5 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -18,7 +18,7 @@ Unknown or missing commands return help without side effects. ## Init: one-time adoption -Init's first response is a read-only preview built by the deterministic engine. It first measures the same repository reality as Map and Doctor, then reports any narrower setup choice separately. The model presents that verified result rather than improvising one. The engine builds a repository-wide inventory but does not yet publish its full path list in the result, so a displayed map *tree* is still assembled by the model from the engine's summary ([gap 2](PRODUCT.md#what-the-current-candidate-does-not-do-yet)). The preview covers the measured repository, the proposed setup map, the `.diataxis/` memory files, and a complete disposition manifest. `--scope ` is the only public component override. Nothing is written until you approve the exact preview. +Init's first response is a read-only preview built by the deterministic engine. It first measures the same repository reality as Map and Doctor, then reports any narrower setup choice separately. The engine result now publishes the complete documentation path ledger, classification, policy provenance, deterministic groups, and readiness facts; the installed Map presentation does not render that ledger yet. The preview covers the measured repository, the proposed setup map, the `.diataxis/` memory files, a complete disposition manifest, and the initial user-owned `never-edit` choice. Repeated `--never-edit ` values provide explicit entries instead of the default `.github/` suggestion; host integrations can also pass an explicit empty choice to decline the suggestion. `--scope ` is the only public component override. Nothing is written until the exact preview is approved, and approved Init writes the policy into the selected map. ## Doctor: ongoing diagnosis @@ -29,7 +29,7 @@ Doctor reports every finding in the evidenced scope, groups them into treatments ## Check: the score -`check` reports the structural percentage separately from Trust coverage (are declared current-truth routes verified?) and hash freshness. A route counts as declared current truth when your documentation map marks it: append `` (or ``) to the end of the map line that links your current-state document. The engine reads that marker; it never infers a current-truth route from prose, and a map that declares none reports Trust as `unverified` rather than guessing. Byte measurements are telemetry with provenance — never a score input, a health failure, or a reason to delete anything. +`check` now returns readiness v3 as an unweighted ready/eligible fraction with every per-document failed fact. Trust coverage (are declared current-truth routes verified?), hash freshness, and byte telemetry stay separate. The envelope temporarily also carries rubric-v2 structural health for compatibility; the two percentages are different contracts and are never compared. A route counts as declared current truth when your documentation map marks it: append `` (or ``) to the end of the map line that links your current-state document. The engine reads that marker; it never infers a current-truth route from prose, and a map that declares none reports Trust as `unverified` rather than guessing. ## Boundaries @@ -41,9 +41,9 @@ These never move: - Approved structural changes carry exact IDs, fingerprints, a recovery boundary, and one complete disposition for every removed file or unique section. - Failed verification becomes a rollback or an explicit conflict — never a silent success. -### Not settled yet +### Ignored documentation -When a repository has no `docs/README.md`, the map and check fallback probes read root files by name — including `PLAN.md`. A private, Git-ignored file with one of those names can therefore be selected as documentation evidence and read into a host transcript. Until the discovery privacy rule lands, keep private working notes out of the repository root or under a name the probe does not list. +Git-ignored or explicitly excluded documentation is represented by repository-relative path, name, and count, but its body is not read and it receives no classification card. Nonignored root documents are inventoried regardless of filename; a conventional name may help select an entry but cannot decide whether a document exists or assign its role. ## More detail diff --git a/EVALUATION.md b/EVALUATION.md index c79be70..a4c4451 100644 --- a/EVALUATION.md +++ b/EVALUATION.md @@ -9,13 +9,19 @@ Diátaxis Docs separates deterministic product proof from model judgment. Tests 3. **Sanitized trajectory gate:** a receipt records visible outcomes, tool actions, diagnostics, and unavailable evidence without raw traces, private paths, credentials, or hidden reasoning. 4. **Capped live canary:** a release candidate may be tested against mapped, missing-map, and hostile fixtures. Infrastructure failure remains a failure; it is never converted into product evidence. -### Known gap in these gates +### Closed cross-command gap and remaining coverage -No gate compares the documentation boundary that one command selected against the boundary another command selected on the same repository. Existing tests supply that boundary as an *input* to both runs rather than comparing it as an *output*, so the suite passes while the two commands disagree. A green checker run over `docs/` is therefore not evidence that Map and Doctor measured the same repository — see [current state](docs/STATE.md). Cross-command parity gates are required before the 0.1.8 candidate can be called complete. +Journey parity now compares the engine-selected boundary, manifest identity, documentation ledger, and readiness result across Map, Check, Doctor, Audit, and Init on the same snapshot. Two strict expected-failure contracts remain assigned to the beginner-interface phase: the installed Map contract must render the engine inventory, and the presented result must carry only readiness v3 rather than also exposing rubric-v2 health. `context`, `write`, and `update` still need matching deterministic entrypoints before the 0.1.8 candidate can be called complete. -## Structural-health rubric +## Readiness v3 -The checker emits rubric v2 as a versioned `health` object with raw counts, earned weight, available weight, a percentage, and a 20-cell text meter. Its exact weights are a versioned, testable local operationalization, not an externally validated scientific or universal constant. This is a reproducible structural baseline according to `$docs`; it is not a universal Diátaxis score and does not prove factual accuracy. +The checker emits one versioned `readiness` object for the engine-classified maintained, nonignored set. It names `ready_count`, `eligible_count`, the derived unweighted percentage, completeness and selected-map validity, every eligible document's four facts, and every excluded document's reason. + +A document is ready only when reachability from the selected map, repository-internal links and anchors, deterministic title identity, and path confinement all pass. There is no partial credit or category weight. Incomplete accounting, a missing selected map, or an empty eligible set yields no percentage. Trust and freshness remain separate; a stored rubric-v2 baseline is reported as a different contract and is neither compared with readiness v3 nor rewritten during a read. + +## Transitional rubric-v2 compatibility + +The envelope temporarily still emits rubric v2 as a versioned `health` object with raw counts, earned weight, available weight, a percentage, and a 20-cell text meter so existing lifecycle state remains readable. Its exact weights are a versioned, testable local operationalization, not an externally validated scientific or universal constant. It is not a universal Diátaxis score or the readiness-v3 contract, no new classification or treatment code consumes it, and the installed presentation must remove the second percentage before the candidate is complete. | Category | Weight | Evidence | | --- | ---: | --- | diff --git a/PRODUCT.md b/PRODUCT.md index 3360954..b977712 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -15,11 +15,11 @@ Everything below exists to make that one sentence true and checkable. | Owner | Decides | | --- | --- | -| **Engine** | Documentation inventory, classification, protected surfaces, navigation graph, health denominator, evidence identity, path confinement, resource limits, findings and proposed target paths, exact write set, approval receipts, drift detection, and the applied/verified/blocked outcome | +| **Engine** | Documentation inventory, evidence-bounded classification, navigation graph, readiness facts and denominator, treatment eligibility, evidence identity, path confinement, resource limits, findings and proposed target paths, exact write set, approval receipts, drift detection, and the applied/verified/blocked outcome | | **Model** | Plain-English presentation, bounded interpretation of engine-selected evidence, why a finding matters, and questions about genuinely ambiguous intent | -| **Human** | The desired outcome, project policy and exceptions, which treatment to run, exact approval for every repository change, and all commit, push, and release decisions | +| **Human** | The desired outcome, role and `never-edit` policy and exceptions, which treatment to run, exact approval for every repository change, and all commit, push, and release decisions | -A model may explain the evidence differently. **A model may not choose a different documentation reality.** The same repository snapshot produces the same inventory, classifications, graph, score inputs, and treatment evidence regardless of which host or model runs the command. +A model may explain the evidence differently. **A model may not choose a different documentation reality.** The same repository snapshot produces the same inventory, classifications, graph, readiness facts, and treatment evidence regardless of which host or model runs the command. ## Required runtime @@ -42,7 +42,7 @@ One word — *scope* — currently hides four different decisions. Under this co | Boundary | Meaning | Example | | --- | --- | --- | | **Repository coverage** | Every documentation path Map accounted for | Entire repository | -| **Health set** | The maintained documents inside the percentage | 8 maintained − 2 protected = 6 documents | +| **Readiness set** | Maintained, nonignored documents inside the factual fraction | 8 maintained documents | | **Files read closely** | File bodies opened for semantic interpretation | Map, current state, and selected evidence | | **Approved changes** | The exact files the human permitted Doctor to modify | `docs/README.md` only | @@ -89,7 +89,7 @@ The command set shipped in the current candidate is larger than this and is docu ## Classification is three separate facts -A document's purpose, its protection, and its Git state are independent. Collapsing them into one exclusive label loses information — a protected page still has a purpose, and an uncommitted page is not thereby private. So every accounted document carries **three separate fields**. +A document's purpose, its user-owned write eligibility, and its Git state are independent. Collapsing them into one exclusive label loses information — a `never-edit` page still has a purpose and readiness facts, and an uncommitted page is not thereby private. So every accounted document carries **three separate fields**. **1. Role — exactly one, describing what the document is for** @@ -103,9 +103,9 @@ A document's purpose, its protection, and its Git state are independent. Collaps *Reference* material is a kind of `preserved`. It is not a separate role. -**2. Protection — an independent yes/no flag** +**2. `never-edit` — an independent yes/no fact** -Protection overlays any role. A protected document stays fully visible in the inventory, is **not** a treatment target, and does **not** enter the health denominator. Agent instructions, community policy, legal text, and security policy are typical protected material. +`never-edit` overlays any role. A document on the user-owned list stays fully visible and, when maintained, stays inside readiness; Doctor simply will not propose a content edit to it. There is no filename-based default ban. Init may suggest `.github/`, but the human may decline or later change that policy through an exact approved map-policy edit. **3. Repository status — an independently reported fact** @@ -119,7 +119,7 @@ These three buckets are **mutually exclusive**, so the totals always reconcile a "Uncommitted" is not one of these buckets, because it is not one condition: it may mean a modified `tracked` file or a new `untracked` file. Either way it is **provenance, not privacy** — nonignored documentation participates normally whether or not Git has committed it. -**Protect** and **ignore/exclude** are actions or modifiers, never roles. Persisting either requires a separately approved write. +**`never-edit`** and **ignore/exclude** are modifiers, never roles. Persisting a role or `never-edit` decision requires a separately approved policy write. Conventional names and locations are classification *evidence*, never an existence gate. Sufficiency differs by kind: @@ -128,50 +128,45 @@ Conventional names and locations are classification *evidence*, never an existen Letting an unrecognized name remove a document from the inventory is forbidden outright. A document whose evidence does not settle its role takes the role `unresolved` — never a guess, never silently dropped. -Which documents are **protected** is an engine-owned deterministic rule, not a per-run judgement, because protection changes the health denominator. The rule reads the same way on every host and every run, and a human may extend it for their project only through a separately approved write. The exact default protected set is settled with the classification implementation; it may not vary by command, model, or invocation. +Write eligibility is deterministic from the document's role, repository status, path confinement, and the user-owned `never-edit` policy. Conventional filenames may produce a warning but never a refusal. The selected map and README entry points remain eligible when otherwise maintained unless the human adds them to `never-edit`. ### Unresolved asks the human plainly ```text Needs your decision — 1 of 3 shown -U-7K4M research/old-results.md +DOC-7A4C2E9F research/old-results.md Not linked from the maintained map; no build marker and no date record. -Role: unresolved Protected: no Repository: tracked +Role: unresolved Never-edit: no Repository: tracked Reply with a role: -U-7K4M maintained -U-7K4M preserved -U-7K4M generated -U-7K4M historical - -Optional, and separate from the role — either can be added to the same line: -U-7K4M protect keep it visible but never treat it -U-7K4M exclude stop reading its body - -Leave it out and it stays unresolved. +DOC-7A4C2E9F maintained +DOC-7A4C2E9F preserved +DOC-7A4C2E9F generated +DOC-7A4C2E9F historical +DOC-7A4C2E9F leave unresolved ``` -Role and the two optional actions are different answers to different questions, so a reply may set a role, an action, or both. Several handles may be resolved in one reply. Large unresolved sets are summarized and batchable — the product never dumps dozens of unexplained paths on a human. Map writes nothing; a resolution persists only through a separately approved write. +Role and `never-edit` are different answers to different questions, so one policy preview may set a role, a `never-edit` choice, or both. Several handles may be resolved in one reply. Large unresolved sets receive deterministic directory-group handles with exact counts and complete route lists; an exact-file decision overrides its group. Ignored documentation receives no classification card because its body was not read. Map writes nothing; a resolution persists only through a separate exact `DOCS-R-` approval. The code is recomputed from current map bytes and supplied decisions, so it survives a process boundary while changed bytes or decisions invalidate it. -## Health always shows its denominator +## Readiness always shows its facts and denominator -The health denominator is exactly: +The readiness denominator is exactly: -> **maintained** AND **not protected** AND **not ignored/excluded** +> **maintained** AND **not ignored/excluded** ```text -Health — maintained, not protected, not ignored -5 of 6 healthy — 83% -denominator: 8 maintained − 2 protected − 0 ignored = 6 +Readiness v3 — maintained, not ignored +6 of 8 ready — 75% +eligible: 8 maintained − 0 ignored = 8 Trust: partial — 1 current-truth route is unverified ``` -The percentage stays, but it always names its numerator and denominator and shows how the denominator was reached. Preserved, generated, historical, unresolved, protected, and ignored documents never silently enter it. Trust is reported separately from structural health. +The fraction is unweighted. A denominator member is ready only when it is reachable from the selected map, all repository-internal links and anchors in its body resolve, its deterministic title identity passes, and its path is confined. Every failed fact is named on that document. A missing selected map, incomplete measurement, or empty denominator yields no percentage. -Membership in the denominator is decided **only** by the three classification fields. Whether a document is reachable, well-linked, or correctly titled is a *scored property* of a member, never a condition for being counted — a broken page must stay in the denominator, or repairing it could not raise the score. +Preserved, generated, historical, unresolved, and ignored documents never silently enter the denominator. A maintained `never-edit` document remains in it; `never-edit` changes treatment eligibility only. Trust and freshness remain separate facts and never enter readiness. -**The numerator is a named open decision, not an implementer's choice.** Which per-document conditions make a maintained document *healthy* — and how that per-document verdict relates to the existing weighted rubric in [EVALUATION.md](EVALUATION.md) — is assigned to the classification-and-scoring step of the completion sequence in [ROADMAP.md](ROADMAP.md#completing-the-018-candidate). Until the owner settles it there, an implementation may not invent a predicate and present it as the contract. What is already fixed, and what the refinement may not undo: the denominator rule above, and the requirement that both numerator and denominator stay visible, and identical across commands **for the same boundary**. Two commands measuring the same boundary may not produce different denominators; a deliberately narrower human-chosen boundary, or a labeled component map, legitimately has its own — and says so. A hidden or command-dependent denominator may not return. +Map, Check, Doctor, Audit, and Init consume the same versioned documentation and readiness result for the same manifest identity and measured boundary. A deliberately narrower human-chosen boundary, or a labeled component map, legitimately has its own fraction and says so. Historical rubric-v2 state remains readable for compatibility, but reads never compare its percentage with readiness v3 or rewrite that state. ## Consent and approval @@ -184,8 +179,8 @@ Consequential approval belongs at the write boundary, and there is one gate: - The engine returns a short handle such as `PLAN-3B9T`. Humans never copy raw SHA-256 fingerprints for routine work. - The short handle is **not** the security control. Immediately before writing, the engine resolves it to the full internal receipt and revalidates the repository root, evidence, approved files, and proposed result. - The model never generates, interprets, or substitutes an approval identifier. -- A code is single-use. Changed evidence or a newer proposal invalidates it. Codes have no arbitrary clock expiry. -- Full hashes remain available under advanced details. +- A code is single-use. The engine recomputes its full internal digest from current evidence and supplied decisions, so changed evidence or decisions invalidate it. Codes have no arbitrary clock or process-lifetime expiry. +- Full hashes stay internal; the normal approval surface carries only the short code and exact proposed diff. Two identities do two jobs and may not grow into a general identity chain: a **normalized** manifest identity, so equivalent documentation is not treated as a different repository merely because Windows and Linux differ on line endings or path case; and an **exact raw-byte** binding on every approved target, so any local byte change after approval refuses the write. @@ -275,16 +270,16 @@ Role Repository status ----------------------- -- ----------------------- -- total 24 total 24 -Protection — a flag, not a role, so these overlap the roles above - protected 3 (2 maintained, 1 preserved) +Never-edit — user-owned write policy, not a role or readiness exclusion + never-edit 3 (2 maintained, 1 preserved) -Health — maintained, not protected, not ignored - 5 of 6 healthy — 83% - denominator: 8 maintained − 2 protected − 0 ignored = 6 +Readiness v3 — maintained, not ignored + 6 of 8 ready — 75% + eligible: 8 maintained − 0 ignored = 8 Trust: partial — 1 current-truth route is unverified -The 2 protected maintained documents are kept current but are never -treated and never scored, so they stay out of the denominator. +The 2 maintained never-edit documents remain in readiness, but Doctor +will not propose content edits to them. 1 ignored path is listed by name only; its body was not read, so its role is honestly unresolved. @@ -298,12 +293,12 @@ Doctor — entire repository 24 documentation files accounted for roles 8 maintained, 7 preserved, 5 generated, 3 historical, 1 unresolved - protection 3 protected (2 maintained, 1 preserved) — a flag, not a role + never-edit 3 (2 maintained, 1 preserved) — write policy, not a role repository 21 tracked, 2 untracked, 1 ignored or excluded -Health — maintained, not protected, not ignored - 5 of 6 healthy — 83% - denominator: 8 maintained − 2 protected − 0 ignored = 6 +Readiness v3 — maintained, not ignored + 6 of 8 ready — 75% + eligible: 8 maintained − 0 ignored = 8 Trust: partial — 1 current-truth route is unverified Treatment 1 of 1 — Reconnect the orphaned setup guide @@ -327,8 +322,8 @@ Files this treatment would change Nothing is moved, renamed, or deleted. No other file is touched. Intended outcome - docs/setup.md becomes reachable from the map, and maintained health - becomes 6 of 6 — 100%. + docs/setup.md becomes reachable from the map, and readiness becomes + 7 of 8 — 88%. Nothing has changed yet. This is a read-only proposal. @@ -356,21 +351,21 @@ A treatment that moves, renames, or removes a file is the highest-consequence ca docs/how-to/README.md. Nothing is discarded without a disposition. ``` -Handle prefixes name what is being answered — `U-` an unresolved document, `DOC-` a Doctor treatment, `PLAN-` a whole proposed change set. Prefixes are engine-owned and the codes behind them are independent; two handles that share the same suffix are unrelated. The exact alphabet, length, and uniqueness scope are engine implementation details settled with the receipt work, not product policy, but they must guarantee that a code is single-use and unambiguous within one proposal. +Unresolved cards use engine-owned document and directory handles; the separate map-policy approval uses `DOCS-R-`. Doctor treatments and whole proposed change sets keep their own receipt namespaces. Codes from different workflows are unrelated, and every approval code must be single-use and unambiguous within its proposal. **Init** — a read-only plan first, then the per-file diff shown above, then one short-code approval, then the applied-and-verified result. Nothing changes before that approval. ## What the current candidate does not do yet -The contract above is the target. Measured against commit `da02ba2` on this repository, the shipped candidate does not meet it. These gaps are assigned to later steps of the 0.1.8 completion program and are **not** implemented: +The contract above is the target. The current 0.1.8 candidate does not meet all of it. These are the remaining gaps and the closed defects whose old public claims must not return: -1. ~~**Commands do not share one selected reality.**~~ **Closed by the selection-policy step.** One engine-owned policy now selects the boundary, map, entry, and current-truth routes per repository snapshot, and every read command consumes it. The default boundary is the repository, an explicit narrower request is reported as partial, and recorded Init state supplies adopted intent rather than a second reality. Evidence and exact commands are in [current state](docs/STATE.md). -2. **The public result does not expose the engine-owned repository inventory.** The checker builds the full manifest internally and publishes its identity, selected routes, and completeness, but not the path ledger itself. A displayed map tree is therefore still assembled by the model instead of supplied by the engine. -3. **A link out of the measured surface validates while its target disappears.** The link counts as valid; the target never enters the inventory, reachability, or the score. +1. ~~**The main repository read routes do not share one selected reality.**~~ **Closed by the selection-policy step.** One engine-owned policy now selects the boundary, map, entry, and current-truth routes per repository snapshot for Map, Check, Doctor, Audit, and Init. The default boundary is the repository, an explicit narrower request is reported as partial, and recorded Init state supplies adopted intent rather than a second reality. The separate deterministic `context` entrypoint remains gap 12. Evidence and exact commands are in [current state](docs/STATE.md). +2. ~~**The public result does not expose the engine-owned repository inventory.**~~ **Closed by the classification/readiness step.** The checker now publishes one ordered `documentation` result containing the complete path ledger, independent role/`never-edit`/repository-status facts, deterministic groups, policy provenance, current-truth routes, coverage, and identity. The installed Map presentation does not consume that ledger yet; that presentation gap remains assigned to the beginner-interface step. +3. **An out-of-surface link relationship is not yet fully presented.** Its target no longer disappears: repository accounting includes and classifies the path, and a maintained target outside an explicit component is excluded from that component's readiness with a named reason. A link can still validate because the target exists while the installed presentation does not explain that boundary relationship. 4. ~~**Root-document recognition uses a fixed 12-name allowlist as an existence gate.**~~ **Closed by the manifest and selection steps.** Every normal nonignored Markdown document is inventoried regardless of its filename or Git staging state, so `GLOSSARY.md` and an uncommitted `TROUBLESHOOTING.md` cannot disappear. Conventional names may help select an entry from the complete inventory; they never decide whether a document exists. -5. **The percentage is not presented with one maintained numerator and denominator.** The engine carries per-category counts, but no single visible line names the scored set — and the denominator changes with the route that selected it. -6. **The three-field classification model is not implemented.** Today one label carries purpose, protection, and Git state at once, so preserved, generated, and historical pages can surface as ordinary navigation defects. -7. **The unresolved contract, the short approval handle, and the per-file diff displays are not implemented.** Approval today uses the mechanisms described in [COMMANDS.md](COMMANDS.md). +5. ~~**The engine does not produce one maintained numerator and denominator.**~~ **Closed by readiness v3.** The unweighted result names ready and eligible counts and every per-document failed fact, and is shared across Map, Check, Doctor, Audit, and Init. The envelope temporarily still carries the legacy rubric-v2 percentage beside readiness v3; the beginner-interface step must remove that second presented percentage. +6. ~~**The three-field classification model is not implemented.**~~ **Closed by the classification/readiness step.** Every accounted document now carries independent role, user-owned `never-edit`, and repository status facts. Names and folders may supply hints but cannot silently assign a role. +7. **The installed command presentation does not yet consume all resolution and treatment facts.** The engine now emits unresolved document and directory-group cards, previews and exactly applies one map-policy edit through a single-use `DOCS-R-` code, reports write eligibility, and evaluates purpose-backed navigation. Final plain-English ordering, the single-percentage display, and the installed Map tree remain assigned to the beginner-interface step. 8. **The shipped command surface is larger than the set above**, and absorbed verbs still exist as separate routes. 9. ~~**The shipped prompt still asserts the disproven claim.**~~ **Closed by the selection-policy step.** `skills/docs/SKILL.md`, its `references/` contracts, and the generated adapter copies now state the selection the engine keeps, and the prompt no longer asks the host to supply a boundary, map, or current-truth route. The historical `0.1.3` entry in [CHANGELOG.md](CHANGELOG.md) still records the claim as it was believed then; a dated changelog entry is not rewritten. @@ -391,9 +386,9 @@ Independent read-only reviews of commit `da02ba2` were completed on 2026-07-27, | 1 | Shared-measurement claim is true of the measurement function and false of the selected files | **Accept — proven** | Reproduced independently; corrected in this documentation set and in the shipped prompt (gaps 1 and 9 closed) | | 2 | One engine-owned documentation manifest should feed every read command | **Accept** | Target contract above | | 3 | Separate repository accounting from the scored maintained set | **Accept** | The four boundaries | -| 4 | A percentage without a visible, stable denominator is not trustworthy | **Accept** | Health contract above | +| 4 | A percentage without a visible, stable denominator is not trustworthy | **Accept — implemented in engine** | Readiness contract above; installed single-percentage presentation remains pending | | 5 | Filename conventions may classify but must not gate existence | **Accept** | Classification model | -| 6 | Out-of-surface links must be reported, never silently validated | **Accept** | Gap 3 above | +| 6 | Out-of-surface links must be reported, never silently validated | **Accept — partially implemented** | Repository accounting and readiness name the target; installed relationship presentation remains gap 3 | | 7 | Preserved, generated, and historical material needs deterministic classification | **Accept** | Classification model | | 8 | Optional local state must never be required for read-only discovery | **Accept** | Local state and privacy | | 9 | Consolidate, do not rewrite | **Accept** | The safety layer is retained intact | diff --git a/README.md b/README.md index 3e858ae..b7f266e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Diátaxis Docs gives a repository's humans and coding agents one trustworthy map of its documentation, then finds and applies only the exact improvements the human approves. The full contract is the [product constitution](PRODUCT.md). > [!NOTE] -> **Public alpha, and the 0.1.8 candidate is not ready for daily-driver use.** Map, Check, Doctor, Audit, and Init now measure the same repository reality, but classification, scoring, and the displayed map still need the remaining 0.1.8 work. Doctor starts read-only and stops before any documentation change — review the proposed treatment and use the same Git safeguards you use for coding agents. What is proven today is in [current state](docs/STATE.md). +> **Public alpha, and the 0.1.8 candidate is not ready for daily-driver use.** Map, Check, Doctor, Audit, and Init now share one engine-owned documentation and readiness result. Doctor starts read-only. The installed interface still needs to render that inventory and remove transitional rubric-v2 health, so review every proposed treatment and use the same Git safeguards you use for coding agents. What is proven today is in [current state](docs/STATE.md). ## Try it in 60 seconds @@ -62,7 +62,7 @@ Doctor returns the map, measured findings, and exact proposed scope when it can ## Project status -Diátaxis Docs is an actively developed public alpha. The 0.1.8 candidate is unreleased. Its read commands now share one repository selection locally; classification, scoring, presentation, and the remaining completion phases are still in progress. The target is the [product constitution](PRODUCT.md) and the measured reality is [current state](docs/STATE.md). +Diátaxis Docs is an actively developed public alpha. The 0.1.8 candidate is unreleased. Its main read commands now share repository selection, evidence-bounded classification, readiness v3, and treatment-policy facts locally; beginner presentation, deterministic `context`/`write`/`update` entrypoints, and the remaining completion phases are still in progress. The target is the [product constitution](PRODUCT.md) and the measured reality is [current state](docs/STATE.md). Codex and the Claude Code desktop app — plugin picker are the live-tested paths; every host is labeled by its actual evidence tier in [Compatibility](COMPATIBILITY.md). Technical deep dives live in the [project wiki](https://github.com/Statusnone420/Skills/wiki), and changing release facts live in the [changelog](CHANGELOG.md). diff --git a/ROADMAP.md b/ROADMAP.md index afac1f2..1e4f0be 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,13 +4,13 @@ Diátaxis Docs is an actively developed public alpha. Priorities are evidence-dr ## Completing the 0.1.8 candidate -0.1.8 is unreleased and blocked on one reproduced architecture defect: commands can select different documentation boundaries on the same repository ([current state](docs/STATE.md)). The work is sequenced, and every step is part of 0.1.8 — none of it is deferred to a later version: +0.1.8 is unreleased. The reproduced command-selection defect and the classification/readiness engine slice are repaired locally; beginner presentation, the remaining deterministic command entrypoints, lifecycle consolidation, packaging, and final cross-platform proof still block release ([current state](docs/STATE.md)). The work is sequenced, and every step is part of 0.1.8 — none of it is deferred to a later version: -1. **Settle the contract.** The [product constitution](PRODUCT.md) defines the promise, boundaries, classifications, health denominator, approval model, and support language. *This step.* -2. **Lock the failure into tests.** Journey regressions that compare the selected boundary across commands, plus one additive engine-owned documentation manifest. -3. **One selection policy.** Every read command, including `context`, consumes that manifest for a given repository snapshot; the silent `docs` default is removed from repository-level Map. -4. **Honest classification and scoring.** Role, protection, and repository status are classified deterministically as three separate fields; every percentage names its maintained denominator. This step also settles the one health question the constitution deliberately leaves open — which per-document conditions make a maintained document *healthy*, and how that verdict relates to the existing weighted rubric. The denominator rule is already fixed and this step may not reopen it. -5. **A beginner-usable interface.** Plain-English engine-supplied results, short approval handles, per-file diffs, deterministic `write` and `update` plans, and a command surface that matches the constitution. +1. **Settle the contract — complete.** The [product constitution](PRODUCT.md) defines the promise, boundaries, classification facts, readiness denominator, approval model, and support language. +2. **Lock the failure into tests — complete.** Journey regressions compare selected boundaries across commands, and one additive engine-owned documentation manifest accounts for the repository. +3. **One selection policy — complete for Map, Check, Doctor, Audit, and Init.** Those routes consume one manifest for a given repository snapshot, and the silent `docs` default is removed from repository-level Map. A matching deterministic `context` entrypoint remains with the interface work. +4. **Honest classification and readiness — engine complete.** Role, user-owned `never-edit`, and repository status are independent facts. Readiness v3 is the unweighted fraction of maintained, nonignored documents that pass reachability, internal-link/anchor, title, and confinement checks; Trust and historical rubric-v2 state remain separate. +5. **A beginner-usable interface.** Render the engine-supplied inventory, classification, readiness, and resolution facts in plain English; present one percentage; keep short approvals and per-file diffs easy to inspect; add deterministic `context`, `write`, and `update` plans; and make the command surface match the constitution. 6. **Consolidation, lifecycle proof, and packaging.** Duplicate and unreachable code removed; transaction machinery kept or replaced only behind crash-injection evidence. 7. **Final candidate proof.** Full cross-platform suites, fresh installed-package runs, and a disposable-repository matrix before any real-repository trial. diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/doctor_baseline.py b/adapters/claude/skills/docs/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/doctor_baseline.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py b/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_closeout.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_closeout.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/measurement.py b/adapters/claude/skills/docs/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/measurement.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/adapters/claude/skills/docs/scripts/check.py b/adapters/claude/skills/docs/scripts/check.py index bd7676e..941d352 100644 --- a/adapters/claude/skills/docs/scripts/check.py +++ b/adapters/claude/skills/docs/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/adapters/claude/skills/docs/scripts/init_closeout.py b/adapters/claude/skills/docs/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/adapters/claude/skills/docs/scripts/init_closeout.py +++ b/adapters/claude/skills/docs/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/adapters/copilot/scripts/_docs_checker/doctor_baseline.py b/adapters/copilot/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/adapters/copilot/scripts/_docs_checker/doctor_baseline.py +++ b/adapters/copilot/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/adapters/copilot/scripts/_docs_checker/document_classification.py b/adapters/copilot/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/adapters/copilot/scripts/_docs_checker/documentation_result.py b/adapters/copilot/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/adapters/copilot/scripts/_docs_checker/init_adoption.py b/adapters/copilot/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/adapters/copilot/scripts/_docs_checker/init_adoption.py +++ b/adapters/copilot/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/adapters/copilot/scripts/_docs_checker/init_closeout.py b/adapters/copilot/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/adapters/copilot/scripts/_docs_checker/init_closeout.py +++ b/adapters/copilot/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/adapters/copilot/scripts/_docs_checker/manifest.py b/adapters/copilot/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/adapters/copilot/scripts/_docs_checker/manifest.py +++ b/adapters/copilot/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/adapters/copilot/scripts/_docs_checker/measurement.py b/adapters/copilot/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/adapters/copilot/scripts/_docs_checker/measurement.py +++ b/adapters/copilot/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/adapters/copilot/scripts/_docs_checker/readiness.py b/adapters/copilot/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/adapters/copilot/scripts/_docs_checker/treatment_policy.py b/adapters/copilot/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/adapters/copilot/scripts/check.py b/adapters/copilot/scripts/check.py index bd7676e..941d352 100644 --- a/adapters/copilot/scripts/check.py +++ b/adapters/copilot/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/adapters/copilot/scripts/init_closeout.py b/adapters/copilot/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/adapters/copilot/scripts/init_closeout.py +++ b/adapters/copilot/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/adapters/cursor/scripts/_docs_checker/doctor_baseline.py b/adapters/cursor/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/adapters/cursor/scripts/_docs_checker/doctor_baseline.py +++ b/adapters/cursor/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/adapters/cursor/scripts/_docs_checker/document_classification.py b/adapters/cursor/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/adapters/cursor/scripts/_docs_checker/documentation_result.py b/adapters/cursor/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/adapters/cursor/scripts/_docs_checker/init_adoption.py b/adapters/cursor/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/adapters/cursor/scripts/_docs_checker/init_adoption.py +++ b/adapters/cursor/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/adapters/cursor/scripts/_docs_checker/init_closeout.py b/adapters/cursor/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/adapters/cursor/scripts/_docs_checker/init_closeout.py +++ b/adapters/cursor/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/adapters/cursor/scripts/_docs_checker/manifest.py b/adapters/cursor/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/adapters/cursor/scripts/_docs_checker/manifest.py +++ b/adapters/cursor/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/adapters/cursor/scripts/_docs_checker/measurement.py b/adapters/cursor/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/adapters/cursor/scripts/_docs_checker/measurement.py +++ b/adapters/cursor/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/adapters/cursor/scripts/_docs_checker/readiness.py b/adapters/cursor/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/adapters/cursor/scripts/_docs_checker/treatment_policy.py b/adapters/cursor/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/adapters/cursor/scripts/check.py b/adapters/cursor/scripts/check.py index bd7676e..941d352 100644 --- a/adapters/cursor/scripts/check.py +++ b/adapters/cursor/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/adapters/cursor/scripts/init_closeout.py b/adapters/cursor/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/adapters/cursor/scripts/init_closeout.py +++ b/adapters/cursor/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/adapters/grok/scripts/_docs_checker/doctor_baseline.py b/adapters/grok/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/adapters/grok/scripts/_docs_checker/doctor_baseline.py +++ b/adapters/grok/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/adapters/grok/scripts/_docs_checker/document_classification.py b/adapters/grok/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/adapters/grok/scripts/_docs_checker/documentation_result.py b/adapters/grok/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/adapters/grok/scripts/_docs_checker/init_adoption.py b/adapters/grok/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/adapters/grok/scripts/_docs_checker/init_adoption.py +++ b/adapters/grok/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/adapters/grok/scripts/_docs_checker/init_closeout.py b/adapters/grok/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/adapters/grok/scripts/_docs_checker/init_closeout.py +++ b/adapters/grok/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/adapters/grok/scripts/_docs_checker/manifest.py b/adapters/grok/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/adapters/grok/scripts/_docs_checker/manifest.py +++ b/adapters/grok/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/adapters/grok/scripts/_docs_checker/measurement.py b/adapters/grok/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/adapters/grok/scripts/_docs_checker/measurement.py +++ b/adapters/grok/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/adapters/grok/scripts/_docs_checker/readiness.py b/adapters/grok/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/adapters/grok/scripts/_docs_checker/treatment_policy.py b/adapters/grok/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/adapters/grok/scripts/check.py b/adapters/grok/scripts/check.py index bd7676e..941d352 100644 --- a/adapters/grok/scripts/check.py +++ b/adapters/grok/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/adapters/grok/scripts/init_closeout.py b/adapters/grok/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/adapters/grok/scripts/init_closeout.py +++ b/adapters/grok/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/doctor_baseline.py b/adapters/plugin/skills/docs/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/doctor_baseline.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py b/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_closeout.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_closeout.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/measurement.py b/adapters/plugin/skills/docs/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/measurement.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/adapters/plugin/skills/docs/scripts/check.py b/adapters/plugin/skills/docs/scripts/check.py index bd7676e..941d352 100644 --- a/adapters/plugin/skills/docs/scripts/check.py +++ b/adapters/plugin/skills/docs/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/adapters/plugin/skills/docs/scripts/init_closeout.py b/adapters/plugin/skills/docs/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/adapters/plugin/skills/docs/scripts/init_closeout.py +++ b/adapters/plugin/skills/docs/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/docs/STATE.md b/docs/STATE.md index 9cb9fcd..2af3df1 100644 --- a/docs/STATE.md +++ b/docs/STATE.md @@ -3,7 +3,7 @@ Diátaxis Docs 0.1.8 is an unreleased public-alpha candidate. The canonical source is `skills/docs/`; Codex, Claude, static, and web packages are generated from it and parity-checked. > [!WARNING] -> **Not ready for daily-driver use.** Commands now select one documentation reality, but classification, scoring, and the presented map still do not meet the target. The target contract is in the [product constitution](../PRODUCT.md); this page records what the code does today. +> **Not ready for daily-driver use.** Commands now select one documentation reality and the engine emits shared classification and readiness facts, but the installed presentation still shows transitional legacy health and does not render the engine-owned map ledger. The target contract is in the [product constitution](../PRODUCT.md); this page records what the code does today. ## The repaired selection defect @@ -11,19 +11,19 @@ Default command routes used to measure different documentation. Reproduced on th | Route | Command | Result then | Result now | | --- | --- | --- | --- | -| Checker default | `check.py . --json --agent` | boundary `docs` — 2 maintained documents, 100%, `clean`, 0 findings | boundary `.` — 61 maintained documents, 89%, `needs-attention`, 27 findings | -| Human override | `check.py . --json --agent --scope .` | boundary `.` — 89%, `needs-attention`, 27 findings | unchanged, and now identical to the default | -| Doctor first-contact baseline | `check.py . --json --agent --doctor-baseline` | `unavailable` — two candidate roots, no boundary, no score | boundary `.` — 61 maintained documents, 89%, 27 findings; provisional entry candidate with no treatment authority | +| Checker default | `check.py . --json --agent` | boundary `docs` — 2 maintained documents, 100%, `clean`, 0 findings | boundary `.` — legacy rubric v2: 61 measured documents, 89%, `needs-attention`, 27 findings | +| Human override | `check.py . --json --agent --scope .` | boundary `.` — 89%, `needs-attention`, 27 findings | unchanged boundary, now identical to the default | +| Doctor first-contact baseline | `check.py . --json --agent --doctor-baseline` | `unavailable` — two candidate roots, no boundary, no score | boundary `.` — same documentation/readiness result; provisional entry candidate with no treatment authority | Rows 1 and 3 were the two *default* routes — no boundary was supplied to either, and they disagreed about whether the repository had any documentation problem. Row 2 was a human override, shown because it revealed what row 1 hid. The measurement *function* was deterministic; the *files handed to it* were not. All three routes now measure the same boundary and manifest identity. Doctor still labels this uninitialized repository a provisional `existing-entry-candidate`, recommends Map, and withholds treatment authority. Removing that extra step without guessing the document's purpose remains the Doctor-independence work in [product constitution](../PRODUCT.md) gap 11. -One engine-owned selection policy now answers the question once per repository snapshot: which boundary is measured, which document is its map, which route a reader enters from, and which routes the map declares as current truth. Map, Check, Doctor, Audit, and Init consume that answer. An omitted boundary is the repository, not `docs/`. Only an explicit human request narrows it, and that result reports itself as a partial component view. Recorded Init state contributes adopted intent, which is reported beside the measurement rather than replacing it, so an initialized repository and a fresh clone of the same documentation produce one manifest identity. Each read result carries that identity, the selected routes, and whether the accounting behind it is complete. +One engine-owned selection policy now answers the question once per repository snapshot: which boundary is measured, which document is its map, which route a reader enters from, and which routes the map declares as current truth. Map, Check, Doctor, Audit, and Init consume that answer. An omitted boundary is the repository, not `docs/`. Only an explicit human request narrows it, and that result reports itself as a partial component view. Recorded Init state contributes adopted intent, which is reported beside the measurement rather than replacing it, so an initialized repository and a fresh clone of the same documentation produce one manifest identity. Each of those results carries that identity, the selected routes, and whether the accounting behind it is complete. Earlier documentation claimed that Doctor, Map, Check, Audit, and Init share one deterministic documentation measurement. **The claim as written was wrong** — the commands shared a measurement function, not a selection — and it has been replaced everywhere it shipped, including `skills/docs/SKILL.md`, its `references/` contracts, and the generated adapter copies. The historical `0.1.3` entry in [CHANGELOG.md](../CHANGELOG.md) still records the claim as it was believed then; a dated changelog entry is not rewritten. -What the repair does **not** yet fix: the result envelope still carries no document inventory, so a displayed map tree is model-authored; a link out of an explicitly narrowed component boundary still validates while its target stays outside that measurement; and roles such as preserved, generated, and historical are still `unresolved`. Init's later setup proposal also still uses conventional-name evidence after the shared repository measurement; that evidence may guide setup, but it cannot hide documents or replace repository reality. The full gap list is in the [product constitution](../PRODUCT.md#what-the-current-candidate-does-not-do-yet). +The shared result now carries the ordered documentation ledger, independent role/`never-edit`/repository-status facts, policy provenance, deterministic groups, unresolved cards, and readiness v3. What remains: the installed Map contract does not render that ledger; the envelope still carries the legacy rubric-v2 percentage beside readiness v3; a link out of an explicitly narrowed component boundary can still validate while its target stays outside that measurement; and `context`, `write`, and `update` still lack matching deterministic engine entrypoints. The full gap list is in the [product constitution](../PRODUCT.md#what-the-current-candidate-does-not-do-yet). ## What is verified today @@ -35,13 +35,19 @@ What the repair does **not** yet fix: the result envelope still carries no docum - Markdown destinations wrapped in angle brackets can contain spaces without becoming false broken-link findings. - MarkdownX is scanned as inert text. Unsupported navigation providers are unmeasured rather than guessed. -- An additive engine-owned documentation manifest now accounts for every in-policy repository document exactly once — tracked, untracked, and ignored or excluded — with ignored bodies unread, roles left `unresolved` rather than guessed, and named exclusions. It records the map route and, where a navigation provider proves one, the separate human entry route; for Mintlify those differ, because `docs.json` is authority and is not itself readable documentation. Its identity binds the normalized content of every inspected body, so a real documentation change moves it while a re-encoded line ending, a recomposed character, or a different spelling of path case does not, and it survives a fresh clone. Ignored bodies are never read and so never digested. Every read command now builds exactly one of these per invocation and publishes its identity, its selected routes, and whether the accounting behind it is complete. +- An additive engine-owned documentation manifest accounts for every in-policy repository document exactly once — tracked, untracked, and ignored or excluded — with ignored bodies unread and named exclusions. Its identity binds normalized content for inspected bodies, so a real documentation change moves it while line-ending, Unicode-composition, or path-case spelling differences do not, and it survives a fresh clone. +- One engine-owned `documentation` result publishes that manifest's ordered path ledger. Every document has an evidence-bounded role, independent user-owned `never_edit` fact, independent repository status, write eligibility, and the reason for any exclusion. The selected map and declared current-truth routes are maintained evidence; ordinary links and filename or folder hints cannot silently assign a role. Invalid or conflicting `docs:policy` input fails closed. +- The selected map's current working-tree `docs:policy` bytes are authoritative for the current snapshot. The result separately reports whether those bytes match Git and are therefore durable; a non-Git repository receives the honest no-saved-version notice. Exact-file policy rules override directory rules, and rendering preserves the existing line order while touching only edited entries. +- Readiness v3 is the unweighted fraction of eligible maintained documents whose reachability, internal links and anchors, title identity, and confinement all pass. It exists only for a complete measurement with a valid selected map, names every failed fact, and never consumes Trust or compares itself with stored rubric-v2 state. +- Unresolved nonignored documents receive deterministic individual or directory-group cards. A resolution previews one exact map-policy diff behind a single-use stateless `DOCS-R-` approval; preview and apply can cross a process boundary, while changed map bytes, changed supplied decisions, or reuse after a successful apply refuses without writing. Init persists approved `never-edit` choices into the map, and treatment policy keeps `never-edit`, ignored, unresolved, and non-maintained reasons separate from readiness. - One Windows-visible behavior change came with it. Repository-relative path validation rejected a rooted path such as `/docs/README.md` on POSIX but accepted it on Windows, where a rooted path carries no drive letter. It is now rejected on both. A rooted `--map` or `--hot` argument therefore returns one clean input error on Windows instead of the previous wrong answer — a `missing-map` finding for a file that exists, or a silently discarded hot path. A rooted `--scope` was already refused and is unchanged. - Map, Check, Doctor, Audit, and Init consume one engine-owned selection. On one unchanged snapshot the default checker route, the first-contact Doctor baseline, and an initialized Doctor produce the same measured boundary and the same manifest identity, and a component request is reported as partial. Selection reads directory listings and the one repository inventory the manifest already needs, so a command inventories the repository once rather than twice. Each item above is proven for the route that owns it. +On this worktree, the repository checker accounts for 93 documentation paths: 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stay unread. Readiness v3 reports 2 of 2 eligible maintained documents ready. That 100% is only the eligible fraction beside 59 nonignored unresolved documents; it is not whole-repository health. The transitional legacy rubric remains separately visible until the presentation work removes it. + The same ownership gap also affects `context`, `write`, and `update`: the current routes do not have matching deterministic engine entrypoints for evidence selection or mutation planning. Until those entrypoints exist, the model can be asked to choose work that the product constitution assigns to the engine. The latest measured results and remaining coverage are in [BENCHMARK.md](../BENCHMARK.md). Release history is in [CHANGELOG.md](../CHANGELOG.md); host support is in [COMPATIBILITY.md](../COMPATIBILITY.md). diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/doctor_baseline.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/doctor_baseline.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_closeout.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_closeout.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/measurement.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/measurement.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/check.py b/plugins/diataxis-docs/skills/docs/scripts/check.py index bd7676e..941d352 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/check.py +++ b/plugins/diataxis-docs/skills/docs/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/plugins/diataxis-docs/skills/docs/scripts/init_closeout.py b/plugins/diataxis-docs/skills/docs/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/init_closeout.py +++ b/plugins/diataxis-docs/skills/docs/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/skills/docs/scripts/_docs_checker/doctor_baseline.py b/skills/docs/scripts/_docs_checker/doctor_baseline.py index f5b03a1..803db99 100644 --- a/skills/docs/scripts/_docs_checker/doctor_baseline.py +++ b/skills/docs/scripts/_docs_checker/doctor_baseline.py @@ -9,6 +9,8 @@ from dataclasses import replace from .discovery import discover_init_scope, prepare_init_discovery +from .documentation_policy import InvalidDocumentationPolicy +from .documentation_result import assemble_documentation_result from .health import health_summary from .init_closeout import inspect_initialization_preflight from .navigation import NavigationBoundary @@ -107,7 +109,11 @@ def _adopted_intent(map_route, boundary, selection, entry_route, manifest): # A separate entry route exists only where a provider proved one. # For a Markdown map it repeats the map, including the placeholder. current.add(_path_identity(entry_route)) - measured = {_path_identity(route) for route in manifest.maintained_surface} + measured = { + _path_identity(document.path) + for document in manifest.documents + if document.body_inspected + } return { "map": map_route, "boundary": boundary, @@ -128,6 +134,12 @@ def measured_result_envelope( measured, who chose it, and whether it is a partial component view — so a reader can never mistake a component map for the repository map. """ + result = assemble_documentation_result( + measurements["manifest"], + measurements, + findings, + ) + published_findings = result["findings"] health = health_summary( measurements, findings=measurements["active_findings"], @@ -138,8 +150,8 @@ def measured_result_envelope( health["surface"] = measurements["navigation"]["scope"] health["provider"] = measurements["navigation"]["provider"] envelope = { - "status": "findings" if findings else "clean", - "has_findings": bool(findings), + "status": "findings" if published_findings else "clean", + "has_findings": bool(published_findings), "root": ".", "scope": scope, "map": map_path, @@ -147,7 +159,9 @@ def measured_result_envelope( "hot_path": hot_path, "navigation": measurements["navigation"], "health": health, - "findings": findings, + "documentation": result["documentation"], + "readiness": result["readiness"], + "findings": published_findings, } if boundary is not None: envelope["boundary"] = boundary @@ -203,6 +217,8 @@ def _lifecycle_baseline(root, discovery, check_measurements, selection, inventor ) except NavigationBoundary as exc: return _unavailable(discovery, "navigation-unavailable", navigation=exc.result) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable(discovery, "measurement-unavailable") if any(item.get("kind") == "state-conflict" for item in findings): @@ -315,6 +331,8 @@ def doctor_orientation_baseline(root, check_measurements): "navigation-unavailable", navigation=exc.result, ) + except InvalidDocumentationPolicy: + raise except (OSError, UnicodeError, ValueError): return _unavailable( discovery, diff --git a/skills/docs/scripts/_docs_checker/document_classification.py b/skills/docs/scripts/_docs_checker/document_classification.py new file mode 100644 index 0000000..5a13534 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/document_classification.py @@ -0,0 +1,113 @@ +"""Evidence-bounded role and ``never-edit`` classification.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from .documentation_policy import ( + InvalidDocumentationPolicy, + never_edit_rule_for, + role_rule_for, + route_key, +) + + +COLD_ROLES = frozenset({"preserved", "generated", "historical"}) +_YEAR_SHAPED = re.compile(r"(?:^|[-_])(19|20)\d{2}(?:[-_.]|$)") + + +@dataclass(frozen=True, slots=True) +class DocumentClassification: + role: str + never_edit: bool + role_evidence: str + hint: str | None + + +def _hint_for(route): + path = PurePosixPath(route) + parts = tuple(part.casefold() for part in path.parts) + stem = path.stem.casefold() + if any(part in {"generated", "adapters", "autogen"} for part in parts[:-1]): + return "name-or-directory-suggests-generated" + if ( + any(part in {"archive", "history", "historical"} for part in parts[:-1]) + or stem.startswith(("history", "archive")) + or _YEAR_SHAPED.search(stem) + ): + return "name-or-directory-suggests-historical" + if any(part in {"references", "reference", "reviews"} for part in parts[:-1]): + return "name-or-directory-suggests-preserved" + return None + + +def _authoritative_routes(map_route, map_exists, current_truth_routes): + routes = [] + if map_exists: + routes.append((map_route, "selected-map")) + routes.extend( + (row["route"], "declared-current-truth") for row in current_truth_routes + ) + return tuple(routes) + + +def classify_documents( + routes, + repository_status, + policy, + *, + map_route, + map_exists, + current_truth_routes, +): + """Classify every accounted route once from explicit evidence only.""" + authoritative = _authoritative_routes( + map_route, + map_exists, + current_truth_routes, + ) + authoritative_by_key = {} + for route, evidence in authoritative: + authoritative_by_key.setdefault(route_key(route), evidence) + rule = role_rule_for(policy, route) + if rule is not None and rule.value != "maintained": + raise InvalidDocumentationPolicy( + "authoritative-role-conflict", + map_route=map_route, + conflict_route=route, + ) + + result = {} + for route in routes: + status = repository_status[route] + never_rule = never_edit_rule_for(policy, route) + if route_key(route) in authoritative_by_key: + role = "maintained" + evidence = authoritative_by_key[route_key(route)] + else: + role_rule = role_rule_for(policy, route) + if role_rule is not None: + role = role_rule.value + evidence = f"working-tree-policy:{role_rule.selector}" + elif status == "ignored or excluded": + role = "unresolved" + evidence = "ignored-body-unread" + else: + role = "unresolved" + evidence = "insufficient-role-evidence" + result[route] = DocumentClassification( + role=role, + never_edit=never_rule is not None, + role_evidence=evidence, + hint=_hint_for(route) if role == "unresolved" else None, + ) + return result + + +__all__ = ( + "COLD_ROLES", + "DocumentClassification", + "classify_documents", +) diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py new file mode 100644 index 0000000..212415c --- /dev/null +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -0,0 +1,599 @@ +"""Working-snapshot documentation-policy parsing and deterministic precedence. + +The canonical map may carry one hidden ``docs:policy`` block. Only the +selected map's current bytes are policy authority for a read. Git provenance +reports whether those bytes have a saved version; it never substitutes a +committed blob for the snapshot being measured. +""" + +from __future__ import annotations + +import re +import subprocess +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +from .paths import normalize_repo_relative, safe_path + + +POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +_POLICY_START = "[ \t]*(?=\r?$)", + re.MULTILINE | re.DOTALL, +) +_GLOB_CHARACTERS = frozenset("*?[]{}") + + +def route_key(value): + """Return one cross-platform identity for a normalized repository route.""" + return unicodedata.normalize("NFC", value).casefold() + + +@dataclass(frozen=True, slots=True) +class PolicyRule: + selector: str + directory: bool + value: str + + +@dataclass(frozen=True, slots=True) +class DocumentationPolicy: + source_route: str + role_rules: tuple[PolicyRule, ...] + never_edit_rules: tuple[PolicyRule, ...] + present: bool + source: str = "working-tree" + git: str = "unavailable" + durable: bool = False + notice: str | None = "no-saved-version-to-restore" + + +class InvalidDocumentationPolicy(ValueError): + """One safe, named refusal for malformed or contradictory policy.""" + + def __init__(self, reason, *, map_route, conflict_route=None): + super().__init__("invalid-policy") + self.classification = "invalid-policy" + self.reason = reason + self.map_route = map_route + self.conflict_route = conflict_route + + +def _invalid(reason, map_route, *, conflict_route=None): + raise InvalidDocumentationPolicy( + reason, + map_route=map_route, + conflict_route=conflict_route, + ) + + +_UNSHARED_SOURCE = object() + + +def _git_map_state(root, map_route, *, git_available=None): + """Return whether Git considers the current map saved in ``HEAD``.""" + if git_available is False: + return "unavailable" + if git_available is None: + try: + repository = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if repository.returncode or repository.stdout.strip() != b"true": + return "unavailable" + try: + saved = subprocess.run( + ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + status = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + map_route, + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError: + return "unavailable" + if not saved.returncode and not status.returncode and not status.stdout: + return "committed" + return "uncommitted" + + +def _current_map_bytes(root, map_route): + """Read one confined current map when no shared scan bytes were supplied.""" + try: + path = safe_path(Path(root) / map_route, root) + if not path.is_file(): + return None + return path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + return None + + +def _policy_text(source_bytes, map_route): + if source_bytes is None: + return "" + try: + return source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + if _POLICY_START.encode("ascii") in source_bytes: + raise InvalidDocumentationPolicy( + "policy-source-not-utf8", + map_route=map_route, + ) from exc + return "" + + +def _provenance(root, map_route, *, git_available=None): + git_state = _git_map_state( + root, + map_route, + git_available=git_available, + ) + if git_state == "unavailable": + return { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + } + if git_state == "committed": + return { + "source": "working-tree", + "git": "committed", + "durable": True, + "notice": None, + } + return { + "source": "working-tree", + "git": "uncommitted", + "durable": False, + "notice": "not-yet-durable", + } + + +def _selector(root, raw, map_route): + directory = raw.endswith("/") + if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): + _invalid("invalid-selector", map_route) + candidate = raw[:-1] if directory else raw + try: + normalized = normalize_repo_relative(candidate, "policy selector") + except (TypeError, UnicodeError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unconfined-selector", + map_route=map_route, + ) from exc + canonical = unicodedata.normalize("NFC", normalized) + expected = canonical + "/" if directory else canonical + if normalized == "." or raw != expected: + _invalid("non-normalized-selector", map_route) + try: + target = safe_path(Path(root) / canonical, root) + except (OSError, ValueError) as exc: + raise InvalidDocumentationPolicy( + "unsafe-selector", + map_route=map_route, + conflict_route=canonical, + ) from exc + if (directory and not target.is_dir()) or ( + not directory and not target.is_file() + ): + _invalid( + "policy-selector-unbound", + map_route, + conflict_route=canonical, + ) + return canonical, directory + + +def parse_documentation_policy(text, *, root, map_route, provenance=None): + """Parse one current policy block or return an explicit empty policy.""" + if not isinstance(text, str): + raise TypeError("policy source must be text") + provenance = { + "source": "working-tree", + "git": "unavailable", + "durable": False, + "notice": "no-saved-version-to-restore", + **({} if provenance is None else provenance), + } + starts = text.count(_POLICY_START) + if starts == 0: + return DocumentationPolicy( + map_route, + (), + (), + False, + **provenance, + ) + matches = list(_POLICY_BLOCK.finditer(text)) + if starts != 1 or len(matches) != 1: + _invalid( + "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", + map_route, + ) + + role_rules = {} + never_edit_rules = {} + for raw_line in matches[0].group(1).splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + _invalid("malformed-role-directive", map_route) + role = parts[1] + if role not in POLICY_ROLES: + _invalid("unknown-role", map_route) + selector, directory = _selector(root, parts[2], map_route) + key = (route_key(selector), directory) + previous = role_rules.get(key) + if previous is not None and previous.value != role: + _invalid( + "conflicting-role-rules", + map_route, + conflict_route=selector, + ) + role_rules[key] = PolicyRule(selector, directory, role) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + _invalid("malformed-never-edit-directive", map_route) + selector, directory = _selector(root, parts[1], map_route) + key = (route_key(selector), directory) + never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + continue + _invalid("unknown-directive", map_route) + + def ordered(rules): + return tuple( + sorted( + rules.values(), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + ) + + return DocumentationPolicy( + source_route=map_route, + role_rules=ordered(role_rules), + never_edit_rules=ordered(never_edit_rules), + present=True, + **provenance, + ) + + +def load_documentation_policy( + root, + map_route, + *, + source_bytes=_UNSHARED_SOURCE, + git_available=None, +): + """Load policy from exact current map bytes and report saved provenance.""" + root = Path(root).absolute() + map_route = normalize_repo_relative(map_route, "map route") + if source_bytes is _UNSHARED_SOURCE: + source_bytes = _current_map_bytes(root, map_route) + elif source_bytes is not None and not isinstance(source_bytes, bytes): + raise TypeError("policy source bytes must be bytes or absent") + provenance = _provenance( + root, + map_route, + git_available=git_available, + ) + return parse_documentation_policy( + _policy_text(source_bytes, map_route), + root=root, + map_route=map_route, + provenance=provenance, + ) + + +def load_committed_documentation_policy(root, map_route): + """Compatibility alias for callers migrating to current-snapshot policy.""" + return load_documentation_policy(root, map_route) + + +def documentation_policy_payload(policy): + """Return public, deterministic policy source and durability facts.""" + return { + "source": policy.source, + "source_route": policy.source_route, + "present": policy.present, + "git": policy.git, + "durable": policy.durable, + "notice": policy.notice, + } + + +def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): + """Render one canonical policy block from already-validated rules.""" + if newline not in ("\n", "\r\n"): + raise ValueError("policy newline must be LF or CRLF") + lines = ["") + return newline.join(lines) + newline + + +def replace_documentation_policy( + text, + role_rules, + never_edit_rules, +): + """Return map text with one minimally edited policy block. + + Existing directive lines retain their exact spelling and order unless the + corresponding rule changes. New rules append after the hand-authored + directives in one deterministic order. + """ + if not isinstance(text, str): + raise TypeError("policy source must be text") + role_rules = tuple(role_rules) + never_edit_rules = tuple(never_edit_rules) + matches = list(_POLICY_BLOCK.finditer(text)) + if not matches and not role_rules and not never_edit_rules: + return text + newline = "\r\n" if "\r\n" in text else "\n" + if matches: + if len(matches) != 1 or text.count(_POLICY_START) != 1: + raise ValueError("policy source must contain one policy block") + match = matches[0] + desired_roles = { + (route_key(rule.selector), rule.directory): rule + for rule in role_rules + } + desired_never_edit = { + (route_key(rule.selector), rule.directory): rule + for rule in never_edit_rules + } + seen_roles = set() + seen_never_edit = set() + retained = [] + for raw_line in match.group(1).splitlines(keepends=True): + line = raw_line.strip() + if not line: + retained.append(raw_line) + continue + if line.startswith("role"): + parts = line.split(maxsplit=2) + if len(parts) != 3 or parts[0] != "role": + raise ValueError("policy source contains an invalid role directive") + selector = parts[2] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_roles.get(key) + if desired is None: + continue + seen_roles.add(key) + expected = ( + f"role {desired.value} {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + if line.startswith("never-edit"): + parts = line.split(maxsplit=1) + if len(parts) != 2 or parts[0] != "never-edit": + raise ValueError( + "policy source contains an invalid never-edit directive" + ) + selector = parts[1] + directory = selector.endswith("/") + key = (route_key(selector[:-1] if directory else selector), directory) + desired = desired_never_edit.get(key) + if desired is None: + continue + seen_never_edit.add(key) + expected = ( + f"never-edit {desired.selector}" + f"{'/' if desired.directory else ''}" + ) + retained.append( + raw_line + if line == expected + else expected + ( + "\r\n" + if raw_line.endswith("\r\n") + else "\n" + if raw_line.endswith("\n") + else "" + ) + ) + continue + raise ValueError("policy source contains an unknown directive") + + missing_roles = sorted( + ( + rule + for key, rule in desired_roles.items() + if key not in seen_roles + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.value, + rule.selector, + ), + ) + missing_never_edit = sorted( + ( + rule + for key, rule in desired_never_edit.items() + if key not in seen_never_edit + ), + key=lambda rule: ( + route_key(rule.selector), + not rule.directory, + rule.selector, + ), + ) + additions = [ + f"role {rule.value} {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_roles + ] + additions.extend( + f"never-edit {rule.selector}" + f"{'/' if rule.directory else ''}{newline}" + for rule in missing_never_edit + ) + content = "".join(retained) + if additions and content and not content.endswith(("\n", "\r")): + content += newline + content += "".join(additions) + return text[: match.start(1)] + content + text[match.end(1) :] + block = render_documentation_policy( + role_rules, + never_edit_rules, + newline=newline, + ) + separator = "" if not text or text.endswith(("\n", "\r")) else newline + blank = "" if not text or text.endswith(newline * 2) else newline + return text + separator + blank + block + + +def _matching_rule(rules, route): + route_identity = route_key(route) + exact = [ + rule + for rule in rules + if not rule.directory and route_key(rule.selector) == route_identity + ] + if exact: + return exact[0] + directories = [ + rule + for rule in rules + if rule.directory + and route_identity.startswith(route_key(rule.selector) + "/") + ] + if not directories: + return None + return max( + directories, + key=lambda rule: ( + len(Path(rule.selector).parts), + len(route_key(rule.selector)), + route_key(rule.selector), + ), + ) + + +def role_rule_for(policy, route): + return _matching_rule(policy.role_rules, route) + + +def never_edit_rule_for(policy, route): + return _matching_rule(policy.never_edit_rules, route) + + +def invalid_policy_envelope(error, *, doctor=False): + """Return the standard zero-authority public refusal for invalid policy.""" + navigation = { + "status": "unmeasured", + "provider": "markdown-map", + "authority": error.map_route, + "classification": error.classification, + "reason": error.reason, + } + if error.conflict_route is not None: + navigation["conflict_route"] = error.conflict_route + result = { + "status": "error", + "has_findings": False, + "root": ".", + "scope": None, + "map": error.map_route, + "error": error.classification, + "navigation": navigation, + "findings": [], + } + if doctor: + result["doctor_baseline"] = { + "status": "unavailable", + "reason": error.classification, + "label": "Doctor baseline unavailable", + "authority_kind": None, + "maintained_map": None, + "treatment_authority": False, + "writes": 0, + "recommendation": None, + } + return result + + +__all__ = ( + "POLICY_ROLES", + "DocumentationPolicy", + "InvalidDocumentationPolicy", + "PolicyRule", + "documentation_policy_payload", + "invalid_policy_envelope", + "load_committed_documentation_policy", + "load_documentation_policy", + "never_edit_rule_for", + "parse_documentation_policy", + "render_documentation_policy", + "replace_documentation_policy", + "role_rule_for", + "route_key", +) diff --git a/skills/docs/scripts/_docs_checker/documentation_result.py b/skills/docs/scripts/_docs_checker/documentation_result.py new file mode 100644 index 0000000..f1ea3a9 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/documentation_result.py @@ -0,0 +1,117 @@ +"""Assembly of the one public documentation and readiness result.""" + +from __future__ import annotations + +from .document_classification import COLD_ROLES +from .documentation_policy import documentation_policy_payload +from .manifest import manifest_payload +from .readiness import calculate_readiness +from .resolution import unresolved_cards +from .treatment_policy import document_write_facts + + +GROUP_ORDER = ( + "preserved", + "generated", + "historical", + "unresolved", + "ignored or excluded", +) + + +def _coverage(manifest, measurements): + selection = measurements.get("selection") + selection_reasons = ( + () + if selection is None + else tuple(getattr(selection, "incomplete_reasons", ())) + ) + reasons = sorted( + {*manifest.coverage.incomplete_reasons, *selection_reasons} + ) + payload = { + "complete": manifest.coverage.complete and not selection_reasons, + "accounted_documents": manifest.coverage.accounted_documents, + "status_source": manifest.coverage.status_source, + "incomplete_reasons": reasons, + "dropped": [ + {"reason": item.reason, "count": item.count} + for item in manifest.coverage.dropped + ], + } + return payload + + +def _groups(manifest): + routes = {kind: [] for kind in GROUP_ORDER} + for document in manifest.documents: + if document.repository_status == "ignored or excluded": + kind = "ignored or excluded" + elif document.role in routes: + kind = document.role + else: + continue + routes[kind].append(document.path) + return [ + { + "kind": kind, + "count": len(routes[kind]), + "routes": sorted( + routes[kind], + key=lambda value: (value.casefold(), value), + ), + } + for kind in GROUP_ORDER + ] + + +def _published_findings(findings, manifest): + """Suppress only cold-role reachability repairs owned by no denominator.""" + cold = { + document.path + for document in manifest.documents + if document.role in COLD_ROLES + } + return [ + finding + for finding in findings + if not ( + finding.get("kind") == "unreachable" + and finding.get("path") in cold + ) + ] + + +def assemble_documentation_result(manifest, measurements, findings): + """Return route-neutral documentation/readiness facts and public findings.""" + documentation = manifest_payload(manifest) + by_path = { + document.path: document + for document in manifest.documents + } + for document in documentation["documents"]: + document.update(document_write_facts(by_path[document["path"]])) + # Who invoked the engine is envelope evidence, not documentation reality. + # Omitting it keeps default, Doctor, explicit-repository, and Init routes + # byte-identical for one snapshot and measured boundary. + documentation["boundary"].pop("source", None) + documentation["policy"] = documentation_policy_payload(manifest.policy) + documentation["coverage"] = _coverage(manifest, measurements) + documentation["groups"] = _groups(manifest) + documentation["unresolved"] = unresolved_cards(manifest) + complete = documentation["coverage"]["complete"] + return { + "documentation": documentation, + "readiness": calculate_readiness( + manifest, + measurements, + complete=complete, + ), + "findings": _published_findings(findings, manifest), + } + + +__all__ = ( + "GROUP_ORDER", + "assemble_documentation_result", +) diff --git a/skills/docs/scripts/_docs_checker/init_adoption.py b/skills/docs/scripts/_docs_checker/init_adoption.py index 8a55451..997c2a0 100644 --- a/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/skills/docs/scripts/_docs_checker/init_adoption.py @@ -16,16 +16,23 @@ from .health import HEALTH_RUBRIC_VERSION from .init_closeout import ( InitCloseoutError, + _run_git, apply_response, prepare_initialization_closeout, preview_response, validate_public_request, ) +from .documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, +) from .navigation import ( canonical_navigation_evidence, ) from .measurement import measure_documentation -from .paths import _path_identity, safe_path +from .formats import is_document_path +from .paths import _path_identity, normalize_repo_relative, safe_path from .selection import ( REPOSITORY_BOUNDARY, boundary_evidence, @@ -35,6 +42,7 @@ SKILL_VERSION = "0.1.8" +_SUGGEST_NEVER_EDIT = object() def canonical_request_bytes(value): @@ -61,6 +69,95 @@ def _raw_digest(path): return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() +def _never_edit_rules(values): + return tuple( + PolicyRule( + value[:-1] if value.endswith("/") else value, + value.endswith("/"), + "never-edit", + ) + for value in values + ) + + +def _policy_projection(root, measurements, map_path, map_creation, never_edit): + """Return exact proposed map bytes and whether an existing map changes.""" + if map_creation is None: + source_bytes = measurements.get("_map_source_bytes") + if source_bytes is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + try: + current = safe_path(root / map_path, root).read_bytes() + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) from exc + if current != source_bytes: + raise InitCloseoutError( + "stale-preview", + "map-policy-source-drift", + "adoption-policy", + ) + else: + source_bytes = map_creation["bytes"] + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise InitCloseoutError( + "waiting", + "map-policy-source-not-utf8", + "adoption-policy", + ) from exc + policy = parse_documentation_policy( + source_text, + root=root, + map_route=map_path, + ) + never_edit_rules = { + (_path_identity(rule.selector), rule.directory): rule + for rule in policy.never_edit_rules + } + for rule in _never_edit_rules(never_edit): + never_edit_rules.setdefault( + (_path_identity(rule.selector), rule.directory), + rule, + ) + proposed = replace_documentation_policy( + source_text, + policy.role_rules, + tuple(never_edit_rules.values()), + ).encode("utf-8") + return source_bytes, proposed, map_creation is None and proposed != source_bytes + + +def _map_git_recovery(root, map_path, source_bytes): + """Bind an in-place Init map edit to its existing Git recovery evidence.""" + try: + commit = _run_git(root, "rev-parse", "HEAD").decode("ascii", "strict") + blob = _run_git(root, "rev-parse", f"HEAD:{map_path}").decode( + "ascii", + "strict", + ) + except (InitCloseoutError, UnicodeDecodeError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "git", + "commit": commit, + "blob": blob, + "digest": "sha256:" + hashlib.sha256(source_bytes).hexdigest(), + } + + def measure_adoption_repository( root, measure, @@ -142,9 +239,9 @@ def _adoption_corpus(measurements): ) selected_scope = selection.measured_boundary paths = [ - route - for route in manifest.maintained_surface - if _in_scope(route, selected_scope) + document.path + for document in manifest.documents + if document.body_inspected and _in_scope(document.path, selected_scope) ] paths.sort(key=lambda item: (item.casefold(), item)) coverage_mode = ( @@ -175,6 +272,48 @@ def _repository_binding(repository_response): } +def _never_edit_choices(root, requested): + """Return exact user choices, or the bounded first-run suggestion.""" + root = Path(root).absolute() + if requested is _SUGGEST_NEVER_EDIT: + try: + github = safe_path(root / ".github", root) + except ValueError: + values = () + else: + values = (".github/",) if github.is_dir() else () + else: + if isinstance(requested, (str, bytes)) or requested is None: + raise TypeError("never-edit choices must be a sequence of paths") + values = tuple(requested) + + normalized = {} + for index, raw in enumerate(values): + if not isinstance(raw, str) or not raw: + raise ValueError(f"never-edit choice {index} is invalid") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + f"never-edit choice {index}", + ) + if route == ".": + raise ValueError("never-edit may not select the repository root") + target = safe_path(root / route, root) + if directory: + if not target.is_dir(): + raise ValueError("never-edit directory is unavailable") + value = route + "/" + else: + if not target.is_file() or not is_document_path(target): + raise ValueError("never-edit document is unavailable") + value = route + normalized.setdefault(_path_identity(value), value) + return sorted( + normalized.values(), + key=lambda value: (value.casefold(), value), + ) + + def _adoption_trust_coverage(health, current_truth_routes): """Bind measured current-truth routes to the Init state they will become.""" coverage = copy.deepcopy(health["coverage"]) @@ -248,6 +387,7 @@ def _build_adoption_request( explicit_scope=None, completed_at=None, repository_context=None, + never_edit=_SUGGEST_NEVER_EDIT, ): root = Path(root).absolute() if repository_context is None: @@ -316,6 +456,7 @@ def _build_adoption_request( completed_at=completed_at, map_path=map_path, map_creation=map_creation, + never_edit=_never_edit_choices(root, never_edit), ) return request, health, { "repository": repository_context, @@ -330,6 +471,7 @@ def build_adoption_request( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, ): """Build one strict schema-3 adoption request from measured repository facts.""" repository_context = measure_adoption_repository( @@ -341,6 +483,7 @@ def build_adoption_request( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) return request, health @@ -354,12 +497,24 @@ def _request_from_context( completed_at=None, map_path, map_creation=None, + never_edit=(), ): setup_measurements = setup_context["measurements"] setup_response = setup_context["response"] selection = setup_measurements["selection"] selected_scope = selection.measured_boundary paths = starting_scan["paths"] + if map_creation is not None: + map_creation = copy.deepcopy(map_creation) + map_source, proposed_map, existing_map_policy_edit = _policy_projection( + root, + setup_measurements, + map_path, + map_creation, + never_edit, + ) + if map_creation is not None: + map_creation["bytes"] = proposed_map navigation = setup_measurements["navigation"] navigation_evidence = canonical_navigation_evidence(root, navigation) # Init evidence binds ``scope`` to the approved setup boundary. A nested @@ -389,6 +544,33 @@ def _request_from_context( } for relative in paths ] + if existing_map_policy_edit: + map_item_id = f"{map_path}#" + map_disposition = next( + ( + item + for item in dispositions + if item["item_id"] == map_item_id + ), + None, + ) + if map_disposition is None: + raise InitCloseoutError( + "waiting", + "map-policy-source-unavailable", + "adoption-policy", + ) + map_disposition.update( + { + "disposition": "MIGRATED", + "reason": ( + "Init will persist the approved never-edit policy in the " + "canonical documentation map." + ), + "target": map_path, + "recovery": _map_git_recovery(root, map_path, map_source), + } + ) document_changes = [] result_health = health if map_creation is not None: @@ -410,6 +592,18 @@ def _request_from_context( map_creation["bytes"], selected_scope, ) + elif existing_map_policy_edit: + document_changes.append( + { + "operation": "REPLACE", + "path": map_path, + "reason": ( + "Persist the exact approved never-edit policy during Init." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(proposed_map).decode("ascii"), + } + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): @@ -444,6 +638,11 @@ def _request_from_context( after_provenance.sort( key=lambda item: (item["route"].casefold(), item["route"]) ) + elif existing_map_policy_edit: + for item in after_provenance: + if _path_identity(item["route"]) == _path_identity(map_path): + item["bytes"] = len(proposed_map) + break hot_path_before = { "value": sum(item["bytes"] for item in before_provenance), "unit": "bytes", @@ -460,6 +659,7 @@ def _request_from_context( "inspected_scope": selected_scope, "map_path": map_path, "current_truth_routes": current_truth_routes, + "never_edit": list(never_edit), "rubric_version": HEALTH_RUBRIC_VERSION, "score_before": health["percentage"], "score_after": result_health["percentage"], @@ -493,6 +693,11 @@ def _request_from_context( "summary": ( "Establish the documentation map and initialize documentation memory." if map_creation is not None + else ( + "Initialize documentation memory and persist the approved " + "never-edit policy." + ) + if existing_map_policy_edit else "Initialize documentation memory without changing existing documents." ), }, @@ -520,6 +725,7 @@ def adoption_preview( *, explicit_scope=None, completed_at=None, + never_edit=_SUGGEST_NEVER_EDIT, _repository_context=None, ): repository_context = _repository_context @@ -533,6 +739,7 @@ def adoption_preview( explicit_scope=explicit_scope, completed_at=completed_at, repository_context=repository_context, + never_edit=never_edit, ) prepared = prepare_initialization_closeout( root, @@ -580,6 +787,7 @@ def adoption_apply(root, receipt_request, approval, *, _repository_context=None) explicit_scope=evidence["selected_scope"], completed_at=evidence["event"]["completed_at"], repository_context=repository_context, + never_edit=tuple(evidence.get("never_edit", ())), ) if canonical_request_bytes(fresh_request) != canonical_request_bytes( receipt_request diff --git a/skills/docs/scripts/_docs_checker/init_closeout.py b/skills/docs/scripts/_docs_checker/init_closeout.py index f208ab4..8a2abef 100644 --- a/skills/docs/scripts/_docs_checker/init_closeout.py +++ b/skills/docs/scripts/_docs_checker/init_closeout.py @@ -840,7 +840,11 @@ def _normalize_repository_measurement_v3(value): def _normalize_evidence_v3(evidence): - optional_fields = {"navigation_evidence", "repository_measurement"} + optional_fields = { + "navigation_evidence", + "never_edit", + "repository_measurement", + } if ( type(evidence) is not dict or not _EVIDENCE_FIELDS.issubset(evidence) @@ -877,6 +881,23 @@ def _normalize_evidence_v3(evidence): boundary=selected_scope, ), ) + never_edit = None + if "never_edit" in evidence: + def normalize_never_edit(item, name): + if type(item) is not str: + _invalid(f"invalid-{name}") + directory = item.endswith("/") + route = _normalize_shared_path_v3( + item[:-1] if directory else item, + name, + ) + return route + "/" if directory else route + + never_edit = _normalize_sorted_strings( + evidence["never_edit"], + "never-edit", + normalizer=normalize_never_edit, + ) cold_paths = _normalize_sorted_strings( evidence["cold_paths"], "cold-paths", @@ -1056,6 +1077,8 @@ def _normalize_evidence_v3(evidence): "approvals": approvals, "source_changes": copy.deepcopy(changes), } + if never_edit is not None: + normalized["never_edit"] = never_edit normalized["event"] = _normalize_public_event_v3(evidence["event"], normalized) if "navigation_evidence" in evidence: normalized["navigation_evidence"] = _normalize_navigation_evidence_v3( diff --git a/skills/docs/scripts/_docs_checker/manifest.py b/skills/docs/scripts/_docs_checker/manifest.py index 84adb4d..349e1c4 100644 --- a/skills/docs/scripts/_docs_checker/manifest.py +++ b/skills/docs/scripts/_docs_checker/manifest.py @@ -16,8 +16,10 @@ a separate, recorded value. * Nonignored documentation participates whether or not Git has committed it. * Ignored or excluded documentation stays visible by path, name, and count, and - its body is never read, so its role stays ``unresolved``. -* Protection is an independent flag that overlays a role, never a role itself. + its body is never read. Explicit non-body evidence may still establish its + independent role; otherwise it stays ``unresolved``. +* ``never-edit`` is a committed user-owned fact independent of role and + repository status. Existing protected-surface evidence remains separate. * A role is only assigned from evidence the engine already produces. Anything else is ``unresolved`` rather than guessed from a filename. * The repository root is a hard boundary; every route is repository-relative. @@ -39,9 +41,10 @@ Identity covers only meaningful evidence: normalized repository-relative paths, the normalized content digest of every document whose body was inspected, roles, -protection, repository status, the measured boundary and entry route, the -maintained surface, current-truth routes, exclusions, and coverage. No absolute -path, timestamp, host value, byte count, or document body enters it. +role evidence, ``never-edit``, protection, repository status, the measured +boundary and entry route, the maintained surface, current-truth routes, +exclusions, and coverage. No absolute path, timestamp, host value, byte count, +or document body enters it. Two normalizations make that identity describe a *documentation landscape* rather than one checkout's spelling of it: @@ -78,6 +81,8 @@ from types import MappingProxyType from .discovery_policy import prune_reason +from .document_classification import classify_documents +from .documentation_policy import DocumentationPolicy, load_documentation_policy from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -135,7 +140,6 @@ } ) -_UNRESOLVED = "unresolved" _MAINTAINED = "maintained" @@ -164,6 +168,9 @@ class DocumentRecord: path: str role: str + never_edit: bool + role_evidence: str + hint: str | None protected: bool protection_reason: str repository_status: str @@ -254,6 +261,7 @@ class DocumentationManifest: schema_version: int ordering_version: str boundary: ManifestBoundary + policy: DocumentationPolicy documents: tuple maintained_surface: tuple current_truth_routes: tuple @@ -545,9 +553,9 @@ def build_documentation_manifest( accounted_set = frozenset(accounted) protection = _protection_policy(accounted) - maintained = [] + inspected = [] for raw in measurements["maintained_routes"]: - route = normalize_repo_relative(raw, "maintained route") + route = normalize_repo_relative(raw, "inspected route") if route not in accounted_set: if route not in excluded and route not in limit_dropped: # Reserved for a genuine inventory disagreement. A route the @@ -561,8 +569,45 @@ def build_documentation_manifest( # precedence wins rather than silently scoring an unread document. notes.incomplete.add("maintained-route-ignored") continue - maintained.append(route) - maintained_set = frozenset(maintained) + inspected.append(route) + inspected_set = frozenset(inspected) + inspected_surface = tuple(sorted(inspected_set, key=_route_order)) + + current_truth = tuple( + sorted( + { + CurrentTruthRoute( + route=normalize_repo_relative(row["route"], "current-truth route"), + marker=row["marker"], + ) + for row in measurements["map_current_routes"] + }, + key=lambda row: (_route_order(row.route), row.marker), + ) + ) + policy_arguments = {} + if "_map_source_bytes" in measurements: + policy_arguments["source_bytes"] = measurements["_map_source_bytes"] + policy_arguments["git_available"] = status_source == "git" + policy = load_documentation_policy(root, map_route, **policy_arguments) + classifications = classify_documents( + accounted, + status_of, + policy, + map_route=map_route, + map_exists=measurements.get("map_exists") is True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in current_truth + ], + ) + maintained_set = frozenset( + route + for route, classification in classifications.items() + if classification.role == _MAINTAINED + and status_of[route] != "ignored or excluded" + and route in inspected_set + ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) # Only inspected bodies are digested, so an ignored or excluded document is @@ -571,7 +616,7 @@ def build_documentation_manifest( # surface so two runs of the same snapshot open the same files in the same # order. digests = {} - for route in maintained_surface: + for route in inspected_surface: digests[route] = _evidence_digest(root, route) if digests[route] is None: notes.incomplete.add("evidence-digest-unavailable") @@ -579,27 +624,18 @@ def build_documentation_manifest( documents = tuple( DocumentRecord( path=route, - role=_MAINTAINED if route in maintained_set else _UNRESOLVED, + role=classifications[route].role, + never_edit=classifications[route].never_edit, + role_evidence=classifications[route].role_evidence, + hint=classifications[route].hint, protected=protection[route.casefold()][0], protection_reason=protection[route.casefold()][1], repository_status=status_of[route], - body_inspected=route in maintained_set, + body_inspected=route in inspected_set, content_digest=digests.get(route), ) for route in accounted ) - current_truth = tuple( - sorted( - { - CurrentTruthRoute( - route=normalize_repo_relative(row["route"], "current-truth route"), - marker=row["marker"], - ) - for row in measurements["map_current_routes"] - }, - key=lambda row: (_route_order(row.route), row.marker), - ) - ) manifest = DocumentationManifest( schema_version=MANIFEST_SCHEMA_VERSION, ordering_version=MANIFEST_ORDERING_VERSION, @@ -611,6 +647,7 @@ def build_documentation_manifest( source=boundary_source, provider=provider, ), + policy=policy, documents=documents, maintained_surface=maintained_surface, current_truth_routes=current_truth, @@ -676,6 +713,8 @@ def manifest_identity_payload(manifest): { "path": _identity_route(document.path), "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, "protected": document.protected, "repository_status": document.repository_status, "content_digest": document.content_digest, @@ -721,6 +760,9 @@ def manifest_payload(manifest): { "path": document.path, "role": document.role, + "never_edit": document.never_edit, + "role_evidence": document.role_evidence, + "hint": document.hint, "protected": document.protected, "protection_reason": document.protection_reason, "repository_status": document.repository_status, diff --git a/skills/docs/scripts/_docs_checker/measurement.py b/skills/docs/scripts/_docs_checker/measurement.py index 1ded57d..dcdb8f0 100644 --- a/skills/docs/scripts/_docs_checker/measurement.py +++ b/skills/docs/scripts/_docs_checker/measurement.py @@ -125,6 +125,7 @@ def measure_documentation( { "active_findings": [*findings, *active_findings], "baseline": baseline, + "stored_rubric": None if state is None else state["rubric"], "freshness": freshness, "coverage": coverage, "navigation": navigation, diff --git a/skills/docs/scripts/_docs_checker/readiness.py b/skills/docs/scripts/_docs_checker/readiness.py new file mode 100644 index 0000000..026df3e --- /dev/null +++ b/skills/docs/scripts/_docs_checker/readiness.py @@ -0,0 +1,229 @@ +"""Unweighted readiness-v3 facts for classified maintained documentation.""" + +from __future__ import annotations + +import unicodedata +from collections.abc import Mapping + + +READINESS_VERSION = 3 +READINESS_FACTS = ("reachability", "links", "title", "confinement") + + +def _normalized_text(value): + if not isinstance(value, str): + return "" + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _route_identity(value): + return unicodedata.normalize("NFC", value.replace("\\", "/")).casefold() + + +def _within_boundary(route, boundary): + route_key = _route_identity(route) + boundary_key = _route_identity(boundary) + return boundary_key == "." or ( + route_key == boundary_key or route_key.startswith(boundary_key + "/") + ) + + +def _evidence_rows(measurements): + rows = {} + for row in measurements.get("document_evidence", ()): + if not isinstance(row, Mapping) or not isinstance(row.get("path"), str): + continue + rows[row["path"]] = row + return rows + + +def _fact(passed, **evidence): + return {"passed": bool(passed), **evidence} + + +def _title_facts(rows, evidence): + duplicate_candidates = {} + for path, row in rows.items(): + observed = evidence.get(path, {}) + h1s = [ + value + for value in observed.get("h1s", ()) + if isinstance(value, str) and value.strip() + ] + if not observed: + row["facts"]["title"] = _fact( + False, + classification="title-unmeasured", + ) + elif len(h1s) == 0: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) + elif len(h1s) > 1: + row["facts"]["title"] = _fact( + False, + classification="multiple-titles", + titles=h1s, + ) + else: + normalized = _normalized_text(h1s[0]) + row["facts"]["title"] = _fact( + True, + classification="unique-title", + title=h1s[0], + ) + duplicate_candidates.setdefault(normalized, []).append(path) + + for paths in duplicate_candidates.values(): + if len(paths) < 2: + continue + labels = {} + for path in paths: + values = { + _normalized_text(label) + for label in evidence.get(path, {}).get("map_labels", ()) + if _normalized_text(label) + } + labels[path] = sorted(values) + distinct = ( + all(len(values) == 1 for values in labels.values()) + and len({values[0] for values in labels.values()}) == len(paths) + ) + for path in paths: + title = rows[path]["facts"]["title"].get("title") + rows[path]["facts"]["title"] = _fact( + distinct, + classification=( + "distinct-map-label" if distinct else "ambiguous-title" + ), + title=title, + map_labels=labels[path], + ) + + +def _baseline_fact(stored_rubric): + if not isinstance(stored_rubric, Mapping): + return None + version = stored_rubric.get("version") + if not isinstance(version, int) or isinstance(version, bool): + return None + return { + "rubric_version": version, + "comparable": version == READINESS_VERSION, + } + + +def calculate_readiness(manifest, measurements, *, complete): + """Return one deterministic, fact-backed readiness-v3 result.""" + evidence = _evidence_rows(measurements) + boundary = manifest.boundary.measured_boundary + rows = {} + excluded = [] + + for document in manifest.documents: + if ( + document.role != "maintained" + or document.repository_status == "ignored or excluded" + or not _within_boundary(document.path, boundary) + ): + if document.repository_status == "ignored or excluded": + reason = "ignored-or-excluded" + elif document.role != "maintained": + reason = f"role-{document.role}" + else: + reason = "outside-measured-boundary" + excluded.append( + { + "path": document.path, + "role": document.role, + "reason": reason, + } + ) + continue + + observed = evidence.get(document.path) + link_failures = ( + [] + if observed is None + else [ + dict(failure) + for failure in observed.get("link_failures", ()) + if isinstance(failure, Mapping) + ] + ) + facts = { + "reachability": _fact( + observed is not None and observed.get("reachable") is True, + reason=( + None + if observed is not None and observed.get("reachable") is True + else "not-reachable-from-selected-map" + if observed is not None + else "reachability-unmeasured" + ), + ), + "links": _fact( + observed is not None and not link_failures, + failures=link_failures, + reason=None if observed is not None else "links-unmeasured", + ), + "title": _fact(False, classification="title-unmeasured"), + # Accounted document paths have already passed the manifest's + # repository confinement and reparse checks. + "confinement": _fact(True, classification="repository-confined"), + } + rows[document.path] = { + "path": document.path, + "facts": facts, + } + + _title_facts(rows, evidence) + documents = [] + for path in sorted(rows, key=lambda value: (value.casefold(), value)): + row = rows[path] + failed = [ + {"fact": name, **row["facts"][name]} + for name in READINESS_FACTS + if row["facts"][name]["passed"] is not True + ] + documents.append( + { + **row, + "ready": not failed, + "failed_facts": failed, + } + ) + + eligible_count = len(documents) + ready_count = sum(row["ready"] for row in documents) + valid_map = measurements.get("map_exists") is True + percentage = ( + round(100 * ready_count / eligible_count) + if complete and valid_map and eligible_count + else None + ) + result = { + "version": READINESS_VERSION, + "ready_count": ready_count, + "eligible_count": eligible_count, + "percentage": percentage, + "measurement_complete": bool(complete), + "selected_map_valid": valid_map, + "documents": documents, + "excluded": sorted( + excluded, + key=lambda row: (row["path"].casefold(), row["path"]), + ), + } + baseline = _baseline_fact(measurements.get("stored_rubric")) + if baseline is not None: + result["baseline"] = baseline + return result + + +__all__ = ( + "READINESS_FACTS", + "READINESS_VERSION", + "calculate_readiness", +) diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py new file mode 100644 index 0000000..2c51236 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -0,0 +1,426 @@ +"""Deterministic unresolved cards plus exact map-policy preview and apply.""" + +from __future__ import annotations + +import difflib +import hashlib +import json +import os +import stat +import tempfile +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path, PurePosixPath + +from .document_classification import classify_documents +from .documentation_policy import ( + POLICY_ROLES, + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + route_key, +) +from .formats import is_document_path +from .paths import ( + normalize_repo_relative, + repository_document_inventory, + safe_path, +) + + +LEAVE_UNRESOLVED = "leave unresolved" +RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +_GROUP_MINIMUM = 3 + + +def _canonical_bytes(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _handle(kind, path, routes): + digest = hashlib.sha256( + _canonical_bytes( + { + "kind": kind, + "path": path, + "routes": list(routes), + } + ) + ).hexdigest() + prefix = "DIR" if kind == "directory" else "DOC" + return f"{prefix}-{digest[:8].upper()}" + + +def _reason(evidence): + reasons = { + "insufficient-role-evidence": ( + "No explicit evidence assigns this documentation a role." + ), + "ignored-body-unread": ( + "The repository excludes this document and its body was not read." + ), + } + return reasons.get( + evidence, + "The available repository evidence does not assign this documentation a role.", + ) + + +def _card(path, routes, *, kind, evidence): + handle = _handle(kind, path, routes) + result = { + "path": path, + "reason": _reason(evidence), + "choices": list(RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + if kind == "directory": + result.update(count=len(routes), routes=list(routes)) + return result + + +def unresolved_cards(manifest): + """Return stable individual cards plus batchable directory groups.""" + unresolved = [ + document + for document in manifest.documents + if document.role == "unresolved" + and document.repository_status != "ignored or excluded" + ] + ordered = sorted( + unresolved, + key=lambda document: (document.path.casefold(), document.path), + ) + by_directory = defaultdict(list) + for document in ordered: + parent = PurePosixPath(document.path).parent.as_posix() + if parent != ".": + by_directory[parent].append(document) + + groups = [] + grouped_routes = set() + for directory in sorted( + by_directory, + key=lambda value: (value.casefold(), value), + ): + documents = by_directory[directory] + if len(documents) < _GROUP_MINIMUM: + continue + routes = tuple(document.path for document in documents) + grouped_routes.update(routes) + groups.append( + _card( + directory + "/", + routes, + kind="directory", + evidence="insufficient-role-evidence", + ) + ) + + individuals = [ + _card( + document.path, + (document.path,), + kind="document", + evidence=document.role_evidence, + ) + for document in ordered + if document.path not in grouped_routes + ] + return [*groups, *individuals] + + +def _normalized_selector(root, raw): + if not isinstance(raw, str) or not raw: + raise ValueError("resolution selector must be a non-empty string") + directory = raw.endswith("/") + route = normalize_repo_relative( + raw[:-1] if directory else raw, + "resolution selector", + ) + if route == ".": + raise ValueError("resolution selector may not be the repository root") + target = safe_path(Path(root) / route, root) + if directory: + if not target.is_dir(): + raise ValueError("resolution directory is unavailable") + elif not target.is_file() or not is_document_path(target): + raise ValueError("resolution document is unavailable") + return route, directory + + +def _decisions(root, values): + if not isinstance(values, Mapping): + raise TypeError("resolution decisions must be a mapping") + normalized = [] + seen = set() + for raw_selector, raw_decision in values.items(): + route, directory = _normalized_selector(root, raw_selector) + identity = (route_key(route), directory) + if identity in seen: + raise ValueError("duplicate-resolution-selector") + seen.add(identity) + if not isinstance(raw_decision, Mapping): + raise TypeError("resolution decision must be a mapping") + if set(raw_decision) - {"role", "never_edit"}: + raise ValueError("resolution decision fields are invalid") + role = raw_decision.get("role") + if role is not None and role not in (*POLICY_ROLES, LEAVE_UNRESOLVED): + raise ValueError("resolution role is invalid") + never_edit = raw_decision.get("never_edit") + if never_edit is not None and type(never_edit) is not bool: + raise TypeError("never-edit decision must be true or false") + if role is None and never_edit is None: + raise ValueError("resolution decision is empty") + normalized.append( + { + "selector": route, + "directory": directory, + "role": role, + "never_edit": never_edit, + } + ) + return sorted( + normalized, + key=lambda row: ( + route_key(row["selector"]), + not row["directory"], + row["selector"], + ), + ) + + +def _rule_key(selector, directory): + return route_key(selector), directory + + +def _proposed_rules(policy, decisions): + role_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.role_rules + } + never_edit_rules = { + _rule_key(rule.selector, rule.directory): rule + for rule in policy.never_edit_rules + } + for decision in decisions: + key = _rule_key(decision["selector"], decision["directory"]) + role = decision["role"] + if role is not None and role != LEAVE_UNRESOLVED: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) + never_edit = decision["never_edit"] + if never_edit is True: + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + "never-edit", + ) + elif never_edit is False: + never_edit_rules.pop(key, None) + return tuple(role_rules.values()), tuple(never_edit_rules.values()) + + +def _selected_snapshot(root): + # Imports stay local so documentation-result assembly can use card + # generation without creating a manifest/selection import cycle. + from .measurement import measure_documentation + from .selection import select_documentation + + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _findings, _hot_path, measurements = measure_documentation( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"], measurements.get("_map_source_bytes") + + +def _refusal(reason, *, map_route=None): + result = { + "status": "refused", + "writes": 0, + "reason": reason, + } + if map_route is not None: + result["map"] = map_route + return result + + +def _resolution_plan(root, decisions): + """Recompute one exact policy edit and its stateless approval binding.""" + root = Path(root).absolute() + manifest, source_bytes = _selected_snapshot(root) + map_route = manifest.boundary.map_route + if source_bytes is None: + raise ValueError("selected map is unavailable") + try: + source_text = source_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("selected map must be UTF-8") from exc + normalized = _decisions(root, decisions) + role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + proposed_text = replace_documentation_policy( + source_text, + role_rules, + never_edit_rules, + ) + proposed_policy = parse_documentation_policy( + proposed_text, + root=root, + map_route=map_route, + ) + classify_documents( + [document.path for document in manifest.documents], + { + document.path: document.repository_status + for document in manifest.documents + }, + proposed_policy, + map_route=map_route, + map_exists=True, + current_truth_routes=[ + {"route": row.route, "marker": row.marker} + for row in manifest.current_truth_routes + ], + ) + map_diff = "".join( + difflib.unified_diff( + source_text.splitlines(keepends=True), + proposed_text.splitlines(keepends=True), + fromfile=f"a/{map_route}", + tofile=f"b/{map_route}", + ) + ) + binding = { + "map": map_route, + "source": source_text, + "proposed": proposed_text, + "decisions": normalized, + } + digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() + approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None + return { + "map": map_route, + "map_diff": map_diff, + "approval": approval, + "source_bytes": source_bytes, + "proposed_bytes": proposed_text.encode("utf-8"), + } + + +def resolution_preview(root, decisions): + """Preview one exact current-map policy edit and write nothing.""" + plan = _resolution_plan(root, decisions) + return { + "status": "approval-required" if plan["map_diff"] else "no-change", + "writes": 0, + "map": plan["map"], + "map_diff": plan["map_diff"], + "approval": plan["approval"], + } + + +def resolution_apply(root, decisions, approval): + """Apply only an exact stateless preview binding recomputed from snapshot.""" + root = Path(root).absolute() + if not isinstance(approval, str): + return _refusal("approval-code-not-current") + try: + plan = _resolution_plan(root, decisions) + except (OSError, TypeError, UnicodeError, ValueError): + return _refusal("approval-code-not-current") + map_route = plan["map"] + if plan["approval"] is None or approval != plan["approval"]: + return _refusal("approval-code-not-current", map_route=map_route) + + try: + target = safe_path(root / map_route, root) + if not target.is_file() or target.is_symlink(): + return _refusal("selected-map-drift", map_route=map_route) + target_mode = stat.S_IMODE(target.stat().st_mode) + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + + staged = None + try: + descriptor, staged_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-", + suffix=".tmp", + dir=target.parent, + ) + staged = Path(staged_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["proposed_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + os.chmod(staged, target_mode) + if target.read_bytes() != plan["source_bytes"]: + return _refusal("selected-map-drift", map_route=map_route) + os.replace(staged, target) + staged = None + except (OSError, ValueError): + return _refusal("selected-map-write-failed", map_route=map_route) + finally: + if staged is not None: + try: + staged.unlink() + except OSError: + pass + + try: + installed = target.read_bytes() + except OSError: + installed = None + if installed != plan["proposed_bytes"]: + return { + "status": "post-apply-conflict", + "writes": 1, + "reason": "selected-map-installed-result-drift", + "map": map_route, + } + return { + "status": "applied", + "writes": 1, + "map": map_route, + "approval": approval, + } + + +__all__ = ( + "LEAVE_UNRESOLVED", + "RESOLUTION_CHOICES", + "resolution_apply", + "resolution_preview", + "unresolved_cards", +) diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index e555c3c..963d51d 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -26,6 +26,7 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") +LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -339,14 +340,30 @@ def shared_markdown_route(relative): anchors = {} titles = {} first_h1 = {} + h1_values = {} texts = {} + map_source_bytes = None for path in files: - raw_text = path.read_text(encoding="utf-8", errors="replace") + if path == mapfile: + map_source_bytes = path.read_bytes() + # Match ``Path.read_text`` universal-newline behavior for all + # existing scan regexes while retaining the map's exact bytes for + # policy provenance. + raw_text = ( + map_source_bytes.decode("utf-8", errors="replace") + .replace("\r\n", "\n") + .replace("\r", "\n") + ) + else: + raw_text = path.read_text(encoding="utf-8", errors="replace") text = strip_fences(raw_text) texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} - first_h1[path] = next((heading.strip() for heading in H1.findall(text)), None) + h1_values[path] = [ + heading.strip() for heading in H1.findall(text) if heading.strip() + ] + first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) title = first_h1[path] or metadata["values"].get("title") if isinstance(title, str) and title.strip(): @@ -360,10 +377,11 @@ def shared_markdown_route(relative): checked_anchors = 0 valid_anchors = 0 valid_navigation_destinations = set() + map_link_labels = {} for path in files: links[path] = [] text = texts[path] - for raw_target in LINK.findall(text): + for link_label, raw_target in LABELED_LINK.findall(text): raw_target = _unwrap_angle_link_destination(unquote(raw_target)) source_relative = _relative_posix(path, root) if navigation and navigation.get("provider") == "mintlify": @@ -473,6 +491,17 @@ def shared_markdown_route(relative): continue valid_links += 1 links[path].append(destination) + if ( + path == mapfile + and destination in files + and destination.is_file() + and is_document_path(destination) + ): + destination_relative = _relative_posix(destination, root) + label = " ".join( + unicodedata.normalize("NFC", link_label).split() + ) + map_link_labels.setdefault(destination_relative, set()).add(label) if navigation and navigation.get("provider") == "mintlify": if destination_relative in navigation.get("navigated_pages", ()): valid_navigation_destinations.add(destination) @@ -715,6 +744,51 @@ def shared_markdown_route(relative): } hot_path = hot_path_summary(root, shared_hot_paths, selected_reparse_paths) map_text = texts.get(mapfile, "") + link_failure_kinds = { + "missing-link", + "missing-anchor", + "outside-link", + "unsupported-navigation-link", + } + link_failures = {} + for finding in findings: + route = finding.get("path") + if finding.get("kind") not in link_failure_kinds or route not in file_route_names: + continue + failure = { + key: finding[key] + for key in ("kind", "target") + if key in finding + } + link_failures.setdefault(route, []).append(failure) + document_evidence = [] + for path in sorted( + files, + key=lambda item: ( + _relative_posix(item, root).casefold(), + _relative_posix(item, root), + ), + ): + route = _relative_posix(path, root) + failures = sorted( + link_failures.get(route, ()), + key=lambda row: ( + str(row.get("kind", "")), + str(row.get("target", "")), + ), + ) + document_evidence.append( + { + "path": route, + "reachable": path in reachable, + "h1s": list(h1_values.get(path, ())), + "map_labels": sorted( + map_link_labels.get(route, ()), + key=lambda value: (value.casefold(), value), + ), + "link_failures": failures, + } + ) measurements = { "map_exists": map_exists, "map_has_h1": bool(first_h1.get(mapfile)), @@ -745,6 +819,10 @@ def shared_markdown_route(relative): "hot_bytes": hot_path["bytes"], "hot_path_files": list(hot_path["files"]), "map_current_routes": map_current_routes, + "document_evidence": document_evidence, + # Internal only: policy parsing consumes the exact selected-map bytes + # this scan measured instead of reopening a potentially changed file. + "_map_source_bytes": map_source_bytes, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } @@ -755,6 +833,7 @@ def shared_markdown_route(relative): "H1", "H2", "HEADING", + "LABELED_LINK", "LINK", "CURRENT_ROUTE_LINK", "discover_markdown", diff --git a/skills/docs/scripts/_docs_checker/treatment_policy.py b/skills/docs/scripts/_docs_checker/treatment_policy.py new file mode 100644 index 0000000..a82d714 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -0,0 +1,208 @@ +"""Pure write and navigation-treatment eligibility policy.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata + + +_SENSITIVE_FILENAMES = frozenset( + { + "agents.md", + "code_of_conduct.md", + "contributing.md", + "governance.md", + "security.md", + "support.md", + } +) +_RESOLUTION_CHOICES = ( + "maintained", + "preserved", + "generated", + "historical", + "leave unresolved", +) + + +def _route_key(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _warning(document): + filename = document.path.rsplit("/", 1)[-1].casefold() + if document.protected: + return ( + "This is a protected or sensitive documentation surface; " + "show its exact approved change before editing." + ) + if filename in _SENSITIVE_FILENAMES: + return ( + "This conventional filename may carry sensitive repository " + "policy; show its exact approved change before editing." + ) + return None + + +def document_write_facts(document): + """Return independent write eligibility without changing role/readiness.""" + warning = _warning(document) + if document.repository_status == "ignored or excluded": + eligible = False + reason = "ignored-or-excluded" + elif document.role != "maintained": + eligible = False + reason = f"role-{document.role}" + elif document.never_edit: + eligible = False + reason = "user-owned-never-edit" + else: + eligible = True + reason = None + result = { + "write_eligible": eligible, + "write_ineligible_reason": reason, + } + if warning is not None: + result["warning"] = warning + return result + + +def _documents(manifest): + return { + _route_key(document.path): document + for document in manifest.documents + } + + +def _classification_card(document): + digest = hashlib.sha256( + json.dumps( + { + "kind": "document", + "path": document.path, + "routes": [document.path], + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + handle = f"DOC-{digest[:8].upper()}" + return { + "path": document.path, + "reason": "No explicit evidence assigns this documentation a role.", + "choices": list(_RESOLUTION_CHOICES), + "handle": handle, + "reply": ( + f"Resolve {handle} as maintained, preserved, generated, " + "historical, or leave unresolved." + ), + } + + +def evaluate_link_proposal( + manifest, + *, + source, + target, + purpose, + wording, + repair_existing=False, +): + """Decide one exact navigation proposal from manifest facts only.""" + repair_existing = bool(repair_existing) + details = { + "source": source, + "target": target, + "purpose": purpose, + "wording": wording, + "repair_existing": repair_existing, + } + if not all(isinstance(value, str) for value in (source, target, purpose, wording)): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-proposal-fields-must-be-text", + } + if repair_existing: + details["purpose"] = ( + f"Repair the existing broken confined link from {source} to {target}." + ) + details["purpose_source"] = "engine" + else: + details["purpose_source"] = "human" + if not repair_existing and not purpose.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "navigation-purpose-required-not-score-only", + } + if not wording.strip(): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-link-wording-required", + } + documents = _documents(manifest) + source_document = documents.get(_route_key(source)) + target_document = documents.get(_route_key(target)) + if source_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-not-in-confined-documentation", + } + source_facts = document_write_facts(source_document) + if not source_facts["write_eligible"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "source-" + source_facts["write_ineligible_reason"], + } + if target_document is None: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-not-in-confined-documentation", + } + if target_document.repository_status == "ignored or excluded": + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-ignored-or-excluded", + } + if target_document.role == "unresolved": + result = { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "target-role-unresolved", + } + result["classification_card"] = _classification_card(target_document) + return result + return { + **details, + "allowed": True, + "readiness_credit": target_document.role == "maintained", + "reason": ( + "existing-broken-confined-link-repair" + if repair_existing + else "purpose-backed-navigation" + ), + } + + +__all__ = ( + "document_write_facts", + "evaluate_link_proposal", +) diff --git a/skills/docs/scripts/check.py b/skills/docs/scripts/check.py index bd7676e..941d352 100644 --- a/skills/docs/scripts/check.py +++ b/skills/docs/scripts/check.py @@ -65,6 +65,10 @@ doctor_orientation_baseline, measured_result_envelope, ) +from _docs_checker.documentation_policy import ( + InvalidDocumentationPolicy, + invalid_policy_envelope, +) from _docs_checker.init_closeout import inspect_initialization_preflight from _docs_checker.continuation import decode_continuation_token from _docs_checker.identity import ( @@ -462,6 +466,21 @@ def main(argv=None): requested_map=namespace.map, hot_paths=hot, ) + except InvalidDocumentationPolicy as exc: + policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) + if namespace.json: + print( + json.dumps( + policy_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + else: + print("error: invalid-policy") + return 2 except NavigationBoundary as exc: if namespace.json or recovery_mode: print( @@ -581,31 +600,38 @@ def main(argv=None): if namespace.init_discovery: print(json.dumps(discovery, ensure_ascii=True)) return 2 if discovery.get("status") == "state-conflict" else 0 + selection = measurements["selection"] + response = measured_result_envelope( + findings, + hot_path, + measurements, + scope=selection.measured_boundary, + # The reader's entry, which is the map itself unless a provider proves + # a separate one. The manifest keeps the authority route; both travel + # in ``boundary``. + map_path=measurements["navigation"].get("entry") or selection.map_route, + boundary=boundary_evidence(selection, measurements["manifest"]), + ) + published_findings = response["findings"] if namespace.json: - selection = measurements["selection"] + if not namespace.agent: + # The additive engine result is the shared agent-command contract. + # Preserve the existing plain JSON compatibility surface until + # generated adapters and public presentation move in Stage 4. + response.pop("documentation", None) + response.pop("readiness", None) print( json.dumps( - measured_result_envelope( - findings, - hot_path, - measurements, - scope=selection.measured_boundary, - # The reader's entry, which is the map itself unless a - # provider proves a separate one. The manifest keeps the - # authority route; both travel in ``boundary``. - map_path=measurements["navigation"].get("entry") - or selection.map_route, - boundary=boundary_evidence(selection, measurements["manifest"]), - ), + response, ensure_ascii=True, ) ) - elif findings: - for finding in findings: + elif published_findings: + for finding in published_findings: print(f"{finding['kind']}: {finding}") else: print("clean") - return 0 if namespace.agent else (1 if findings else 0) + return 0 if namespace.agent else (1 if published_findings else 0) if __name__ == "__main__": diff --git a/skills/docs/scripts/init_closeout.py b/skills/docs/scripts/init_closeout.py index 71c5951..0de01b8 100644 --- a/skills/docs/scripts/init_closeout.py +++ b/skills/docs/scripts/init_closeout.py @@ -215,6 +215,12 @@ def main(argv=None): "--scope", help="explicit shared documentation scope when discovery is ambiguous", ) + parser.add_argument( + "--never-edit", + action="append", + dest="never_edit", + help="repeat one exact repository-relative initial never-edit path", + ) arguments = parser.parse_args(argv) validated_request = None @@ -229,6 +235,11 @@ def main(argv=None): request, response = adoption_preview( arguments.root, explicit_scope=arguments.scope, + **( + {} + if arguments.never_edit is None + else {"never_edit": tuple(arguments.never_edit)} + ), _repository_context=repository_context, ) _write_adoption_receipt( diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 08f7dec..b045652 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -123,7 +123,13 @@ def test_canonical_checker_registry_covers_package_and_generated_bundles(self): "discovery.py", "scan.py", "measurement.py", + "documentation_policy.py", + "document_classification.py", "manifest.py", + "readiness.py", + "documentation_result.py", + "resolution.py", + "treatment_policy.py", "selection.py", "runtime.py", "identity.py", diff --git a/tests/test_docs_checker_architecture.py b/tests/test_docs_checker_architecture.py index fc68fee..bf0b14d 100644 --- a/tests/test_docs_checker_architecture.py +++ b/tests/test_docs_checker_architecture.py @@ -32,7 +32,13 @@ "doctor_baseline", "health", "navigation", + "documentation_policy", + "document_classification", "manifest", + "readiness", + "documentation_result", + "resolution", + "treatment_policy", "selection", ) @@ -250,12 +256,56 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): self.assertLessEqual(graph["memory"], {"formats", "paths", "identity"}) self.assertEqual( graph["doctor_baseline"], - {"discovery", "health", "navigation", "paths", "selection"}, + { + "discovery", + "documentation_policy", + "documentation_result", + "health", + "navigation", + "paths", + "selection", + }, ) self.assertEqual(graph["navigation"], {"formats", "paths"}) + self.assertEqual(graph["documentation_policy"], {"paths"}) + self.assertEqual( + graph["document_classification"], + {"documentation_policy"}, + ) + self.assertEqual( + graph["manifest"], + { + "discovery_policy", + "document_classification", + "documentation_policy", + "health", + "paths", + "surfaces", + }, + ) + self.assertEqual(graph["readiness"], set()) + self.assertEqual( + graph["documentation_result"], + { + "document_classification", + "documentation_policy", + "manifest", + "readiness", + "resolution", + "treatment_policy", + }, + ) self.assertEqual( - graph["manifest"], {"discovery_policy", "health", "paths", "surfaces"} + graph["resolution"], + { + "document_classification", + "documentation_policy", + "formats", + "paths", + "selection", + }, ) + self.assertEqual(graph["treatment_policy"], set()) # Selection owns the boundary policy and nothing else: it reads the # documentation-root and format policies, confines routes, and builds # the manifest. It must never reach into discovery, lifecycle, or diff --git a/tests/test_doctor_map_journey_parity.py b/tests/test_doctor_map_journey_parity.py index 05f3db5..48bc17b 100644 --- a/tests/test_doctor_map_journey_parity.py +++ b/tests/test_doctor_map_journey_parity.py @@ -38,9 +38,8 @@ #: Every strict expected failure below names its owning phase here as well as in #: its docstring, so the owner can find its work by grepping one table. EXPECTED_FAILURE_OWNERS = { - "test_preserved_and_generated_material_is_classified_not_maintained": "PR 3", - "test_the_measured_result_envelope_carries_the_repository_inventory": "PR 3", "test_the_installed_map_contract_presents_the_engine_inventory": "PR 4", + "test_the_presented_result_carries_exactly_one_percentage": "PR 4", } @@ -179,8 +178,8 @@ def test_default_routes_agree_about_the_measured_boundary_and_the_score(self): payloads["map"]["health"]["percentage"], ) - def test_the_measured_envelope_reports_its_boundary_but_not_yet_an_inventory(self): - """PR 2 publishes the selection; PR 3 owns publishing the inventory.""" + def test_the_measured_envelope_reports_its_boundary_and_inventory(self): + """PR 3 publishes the engine inventory beside the PR 2 selection.""" with tempfile.TemporaryDirectory() as td: root = Path(td) / "repo" root.mkdir() @@ -200,6 +199,8 @@ def test_the_measured_envelope_reports_its_boundary_but_not_yet_an_inventory(sel "hot_path", "navigation", "health", + "documentation", + "readiness", "findings", "boundary", }, @@ -209,12 +210,12 @@ def test_the_measured_envelope_reports_its_boundary_but_not_yet_an_inventory(sel ) self.assertTrue(payload["boundary"]["identity"].startswith("sha256:")) self.assertTrue(payload["boundary"]["documentation_complete"]) - # Ordinary untracked documentation participates normally. The - # ignored body and path stay out of this pre-PR-3 envelope, and the - # complete inventory itself is still not published here. + # Ordinary untracked documentation participates normally. Ignored + # documentation is visible by path while its body stays unread. serialized = json.dumps(payload) self.assertIn("local-note.md", serialized) - self.assertNotIn("private/secret.md", serialized) + self.assertIn("private/secret.md", serialized) + self.assertNotIn("PRIVATE_BODY_SENTINEL", serialized) def test_an_out_of_boundary_link_target_now_participates_in_the_measurement(self): """The reproduced ``GLOSSARY.md`` defect. @@ -239,7 +240,13 @@ def test_an_out_of_boundary_link_target_now_participates_in_the_measurement(self ) for name, manifest in manifests.items(): with self.subTest(route=name): - self.assertIn("GLOSSARY.md", manifest.maintained_surface) + glossary = [ + document + for document in manifest.documents + if document.path == "GLOSSARY.md" + ] + self.assertEqual(len(glossary), 1) + self.assertEqual(glossary[0].role, "unresolved") def test_private_material_never_reaches_any_command_result(self): with tempfile.TemporaryDirectory() as td: @@ -389,15 +396,33 @@ def test_the_shipped_prompt_no_longer_asserts_the_disproven_shared_measurement(s class ClassificationAndEnvelopeParityTests(unittest.TestCase): """Contracts the constitution promises that classification does not keep.""" - @unittest.expectedFailure def test_preserved_and_generated_material_is_classified_not_maintained(self): """PRODUCT.md 'Classification is three separate facts': preserved and generated pages carry their own role and never silently become part of - the maintained surface. Owned by PR 3.""" + the maintained surface. + + PRODUCT.md also says a filename alone is never sufficient evidence, so + this fixture commits an approved ``docs:policy`` block into its map: + the roles asserted below are backed by that committed human policy, + not guessed from ``archive/``, ``generated/``, or a dated name. + Owned by PR 3.""" with tempfile.TemporaryDirectory() as td: root = Path(td) / "repo" root.mkdir() cartographer_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "policy") manifests, _ = journey_manifests(root) roles = { @@ -409,10 +434,14 @@ def test_preserved_and_generated_material_is_classified_not_maintained(self): self.assertEqual(roles["docs/generated/api.md"], "generated") self.assertEqual(roles["HISTORY-2024.md"], "historical") - @unittest.expectedFailure def test_the_measured_result_envelope_carries_the_repository_inventory(self): """PRODUCT.md gap 2: the engine computes a repository inventory and - discards it, so a displayed map tree is model-authored. Owned by PR 3.""" + discards it, so a displayed map tree is model-authored. + + The envelope must publish the approved engine-owned ``documentation`` + result — not a competing top-level ``manifest`` field — so this + marker comes off when the PR 3 Stage 2 documentation result lands. + Owned by PR 3.""" with tempfile.TemporaryDirectory() as td: root = Path(td) / "repo" root.mkdir() @@ -420,12 +449,45 @@ def test_the_measured_result_envelope_carries_the_repository_inventory(self): payload = run_checker(root) - self.assertIn("manifest", payload) + self.assertIn("documentation", payload) + documented = { + document["path"] + for document in payload["documentation"]["documents"] + } + self.assertIn("GLOSSARY.md", documented) class InstalledPresentationParityTests(unittest.TestCase): """Contracts the constitution promises that installed prompts do not keep.""" + @unittest.expectedFailure + def test_the_presented_result_carries_exactly_one_percentage(self): + """PRODUCT.md makes readiness v3 the factual documentation fraction; + after PR 4 the presented result carries that one percentage rather + than also presenting transitional legacy v2 health. Owned by PR 4.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + cartographer_fixture(root) + + payload = run_checker(root) + + percentages = [] + + def collect(value, route=()): + if isinstance(value, dict): + for key, child in value.items(): + child_route = route + (key,) + if key == "percentage": + percentages.append(child_route) + collect(child, child_route) + elif isinstance(value, list): + for index, child in enumerate(value): + collect(child, route + (str(index),)) + + collect(payload) + self.assertEqual(percentages, [("readiness", "percentage")]) + @unittest.expectedFailure def test_the_installed_map_contract_presents_the_engine_inventory(self): """PRODUCT.md 'What Map covers' plus gap 2: the shipped map contract diff --git a/tests/test_documentation_manifest.py b/tests/test_documentation_manifest.py index c49a8f6..d5d2931 100644 --- a/tests/test_documentation_manifest.py +++ b/tests/test_documentation_manifest.py @@ -266,8 +266,9 @@ def test_conventional_docs_layout_reports_its_maintained_surface(self): self.assertEqual(routes(manifest), ["docs/guide.md", "docs/README.md"]) self.assertEqual( - manifest.maintained_surface, ("docs/guide.md", "docs/README.md") + manifest.maintained_surface, ("docs/README.md",) ) + self.assertEqual(record_of(manifest, "docs/guide.md").role, "unresolved") self.assertEqual(manifest.boundary.coverage, "repository") self.assertEqual(manifest.boundary.measured_boundary, "docs") @@ -289,7 +290,14 @@ def test_root_only_documentation_is_accounted_without_a_docs_directory(self): ) self.assertEqual( manifest.maintained_surface, - ("CONTRIBUTING.md", "GLOSSARY.md", "README.md"), + ("README.md",), + ) + self.assertEqual( + { + record_of(manifest, path).role + for path in ("CONTRIBUTING.md", "GLOSSARY.md") + }, + {"unresolved"}, ) def test_multi_root_repository_accounts_for_every_package_surface(self): @@ -316,11 +324,9 @@ def test_multi_root_repository_accounts_for_every_package_surface(self): ], ) - def test_generated_and_preserved_material_stays_visible_and_carries_the_measured_role(self): - """Generated and preserved pages must be accounted for, and the manifest - must report the role the engine actually measured. Inside the measured - boundary today that role is ``maintained`` — the classification defect - this manifest makes visible rather than one it hides.""" + def test_name_shaped_generated_and_preserved_material_stays_visible_unresolved(self): + """Generated- and archive-shaped pages remain visible, but names alone + are hints and never role evidence.""" with tempfile.TemporaryDirectory() as td: root = Path(td) / "repo" root.mkdir() @@ -332,7 +338,8 @@ def test_generated_and_preserved_material_stays_visible_and_carries_the_measured with self.subTest(path=path): document = record_of(manifest, path) self.assertIn(document.role, DOCUMENT_ROLES) - self.assertEqual(document.role, "maintained") + self.assertEqual(document.role, "unresolved") + self.assertTrue(document.hint) self.assertTrue(document.body_inspected) def test_a_role_is_never_guessed_from_a_filename_or_directory(self): @@ -1079,9 +1086,9 @@ def test_protection_is_an_independent_flag_rather_than_a_role(self): security = record_of(manifest, "SECURITY.md") self.assertTrue(security.protected) - self.assertIn(security.role, DOCUMENT_ROLES) + self.assertEqual(security.role, "unresolved") self.assertTrue(security.protection_reason) - self.assertIn("SECURITY.md", manifest.maintained_surface) + self.assertNotIn("SECURITY.md", manifest.maintained_surface) def test_the_manifest_records_the_selection_it_was_given(self): with tempfile.TemporaryDirectory() as td: diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py new file mode 100644 index 0000000..1b90461 --- /dev/null +++ b/tests/test_documentation_result_contract.py @@ -0,0 +1,1390 @@ +"""PR 3 Stage 2 contract tests: the engine-owned documentation result. + +Every test in this module asserts a contract the PR 3 charter and the product +constitution promise but the current candidate does not keep. They are meant +to FAIL on the PR 3 base commit — each for the one missing engine behavior its +docstring names, never because of a broken fixture or a platform assumption. +Stage 2 implements the smallest canonical behavior that turns them green. +Stage 3 resolution and treatment contracts live separately, as strict expected +failures, in ``tests/test_resolution_treatment_contract.py``. + +Names proposed here that do not exist yet — the envelope's ``documentation`` +and ``readiness`` sections, the ``invalid-policy`` failure classification, the +``ambiguous-title`` readiness fact, the manifest's ``never_edit`` fact, and +the readiness fact keys ``reachability`` / ``links`` / ``title`` / +``confinement`` — are the Stage 1 proposal of the Stage 2 public contract. +``documentation`` is deliberately not called ``classification`` because the +existing engine already uses ``classification`` as a scalar failure code in +its failure envelopes. Renaming any proposal is a review decision, not an +implementation accident. + +Contract sources: the PR 3 charter acceptance criteria (local control plan), +PRODUCT.md "Classification is three separate facts" and "Health always shows +its denominator", and the owner-approved policy-block, readiness, write +authority, and purpose-backed navigation gates. +""" + +import difflib +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SCRIPTS = ROOT / "skills" / "docs" / "scripts" +CHECKER = SCRIPTS / "check.py" +sys.path.insert(0, str(SCRIPTS)) + +import check as docs_checker # noqa: E402 +from _docs_checker.manifest import manifest_payload # noqa: E402 +from _docs_checker.paths import repository_document_inventory # noqa: E402 +from _docs_checker.selection import select_documentation # noqa: E402 + + +#: The bounded persistable role choices decision 13 offers a human for one +#: unresolved document. The Stage 3 resolution contracts consume these from +#: here so both stages share one vocabulary. +ROLE_CHOICES = frozenset({"maintained", "preserved", "generated", "historical"}) + +#: The owner-approved reply that keeps the honest default. It is a human +#: choice, never a persisted role rule: replying ``leave unresolved`` writes +#: no policy entry and the document simply stays ``unresolved``. +LEAVE_UNRESOLVED_CHOICE = "leave unresolved" +RESOLUTION_CHOICES = frozenset(ROLE_CHOICES | {LEAVE_UNRESOLVED_CHOICE}) + +#: The owner-approved committed policy block for the classified fixture. +#: Directory rules batch groups, the deepest directory rule wins, and an exact +#: path overrides its containing directory. ``never-edit`` is not a role. +POLICY_BLOCK = ( + "\n" +) + +#: What the committed policy plus the constitution's evidence rules make true +#: of the classified fixture. Exercises every precedence rule at least once. +EXPECTED_ROLES = { + "docs/README.md": "maintained", + "docs/STATE.md": "maintained", + "docs/guide.md": "maintained", + "docs/frozen.md": "maintained", + "docs/lonely.md": "maintained", + "docs/archive/2023-review.md": "preserved", + "docs/archive/current-migration.md": "maintained", + "docs/archive/old/2019-review.md": "historical", + "docs/generated/api.md": "generated", + "docs/generated/cli.md": "generated", + "GLOSSARY.md": "maintained", + "HISTORY-2024.md": "historical", +} + +#: The grouped cold material the classified fixture must expose. Keys are the +#: group kinds correction 7 requires; the ignored kind matches the repository +#: status bucket name because ignore precedence is a status, not a role. +EXPECTED_GROUPS = { + "preserved": ["docs/archive/2023-review.md"], + "generated": ["docs/generated/api.md", "docs/generated/cli.md"], + "historical": ["docs/archive/old/2019-review.md", "HISTORY-2024.md"], + "unresolved": [".github/SUPPORT.md", "local-note.md"], + "ignored or excluded": ["private/secret.md"], +} + +#: Charter gate 1: eligible = maintained AND not ignored/excluded, unweighted. +#: ``docs/guide.md`` fails its link fact and ``docs/lonely.md`` is unreachable, +#: and the secondary percentage rounds 100 * 5 / 7 to the nearest whole number. +EXPECTED_ELIGIBLE = 7 +EXPECTED_READY = 5 +EXPECTED_PERCENTAGE = round(100 * EXPECTED_READY / EXPECTED_ELIGIBLE) + +#: The four per-document facts every eligible readiness row must expose. +READINESS_FACTS = frozenset({"reachability", "links", "title", "confinement"}) + + +def git(root, *arguments): + return subprocess.run( + ["git", "-C", str(root), *arguments], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=True, + ) + + +def write(root, relative, text): + target = Path(root) / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8", newline="\n") + return target + + +def _init_git(root): + git(root, "init", "--quiet") + git(root, "config", "user.email", "fixture@example.invalid") + git(root, "config", "user.name", "Fixture") + + +def hinted_fixture(root): + """Name-shaped documents with no qualifying role evidence at all. + + ``docs/archive/``, ``docs/generated/``, and a dated root filename are + hints. No generator declaration and no committed policy backs them, so + the constitution requires every one of them to stay ``unresolved``. The + selected map and its declared current-truth route are the only documents + with maintained evidence. + """ + _init_git(root) + write(root, ".gitignore", "private/\n") + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Current state](STATE.md) \n" + "- [Guide](guide.md)\n" + "- [Glossary](../GLOSSARY.md)\n", + ) + write(root, "docs/STATE.md", "# Current state\n\nWhat is true today.\n") + write(root, "docs/guide.md", "# Guide\n\nHow to use the project.\n") + write(root, "docs/archive/2023-review.md", "# 2023 review\n\nPreserved record.\n") + write(root, "docs/generated/api.md", "# API\n\nProduced by the build.\n") + write(root, "GLOSSARY.md", "# Glossary\n\nTerms this project uses.\n") + write(root, "HISTORY-2024.md", "# History 2024\n\nA dated record.\n") + write(root, "private/secret.md", "# PRIVATE_BODY_SENTINEL\n") + git(root, "add", "--", ".gitignore", "docs", "GLOSSARY.md", "HISTORY-2024.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + +def classified_fixture(root): + """The Cartographer shape plus the owner-approved committed policy block.""" + _init_git(root) + write(root, ".gitignore", "private/\n") + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Current state](STATE.md) \n" + "- [Guide](guide.md)\n" + "- [Frozen contract](frozen.md)\n" + "- [Current migration](archive/current-migration.md)\n" + "- [2023 review](archive/2023-review.md)\n" + "- [Glossary](../GLOSSARY.md)\n" + "\n" + POLICY_BLOCK, + ) + write(root, "docs/STATE.md", "# Current state\n\nWhat is true today.\n") + write( + root, + "docs/guide.md", + "# Guide\n\nSee the [missing page](missing-page.md).\n", + ) + write(root, "docs/frozen.md", "# Frozen contract\n\nStable agreed text.\n") + write(root, "docs/lonely.md", "# Lonely\n\nNothing links here.\n") + write(root, "docs/archive/2023-review.md", "# 2023 review\n\nPreserved record.\n") + write( + root, + "docs/archive/current-migration.md", + "# Current migration\n\nStill maintained by exact exception.\n", + ) + write(root, "docs/archive/old/2019-review.md", "# 2019 review\n\nOld record.\n") + write(root, "docs/generated/api.md", "# API\n\nProduced by the build.\n") + write(root, "docs/generated/cli.md", "# CLI\n\nProduced by the build.\n") + write(root, "GLOSSARY.md", "# Glossary\n\nTerms this project uses.\n") + write(root, "HISTORY-2024.md", "# History 2024\n\nA dated record.\n") + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask questions.\n") + write(root, "local-note.md", "# Local note\n\nUncommitted working page.\n") + write(root, "private/secret.md", "# PRIVATE_BODY_SENTINEL\n") + git( + root, + "add", + "--", + ".gitignore", + ".github", + "docs", + "GLOSSARY.md", + "HISTORY-2024.md", + ) + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + +def run_checker_route_process(root, *arguments): + return subprocess.run( + [sys.executable, "-B", str(CHECKER), str(root), *arguments], + capture_output=True, + text=True, + cwd=str(ROOT), + ) + + +def run_checker_process(root, *arguments): + result = run_checker_route_process( + root, + "--json", + "--agent", + *arguments, + ) + return json.loads(result.stdout), result.returncode + + +def run_checker(root, *arguments): + payload, _returncode = run_checker_process(root, *arguments) + return payload + + +def build_manifest(root): + """Build the engine manifest exactly as the default command route does.""" + inventory = repository_document_inventory(root, ".") + selection = select_documentation( + root, + requested_boundary=None, + route="checker-default", + inventory=inventory, + ) + _, _, measurements = docs_checker.check( + root, + selection.map_route, + None, + selection.measured_boundary, + _measurements=True, + _selection=selection, + _inventory=inventory, + ) + return measurements["manifest"] + + +def manifest_documents(root): + return manifest_payload(build_manifest(root))["documents"] + + +def section(payload, name): + """Return one engine-owned envelope section, or fail with the contract.""" + if name not in payload: + raise AssertionError( + f"the measured result envelope carries no engine-owned {name!r} " + "section; PR 3 acceptance requires every command route to consume " + "one engine documentation and readiness result" + ) + return payload[name] + + +def make_reparse_point(link, target): + """Create a directory reparse path cross-platform, or skip honestly.""" + if os.name == "nt": + quoted_link = str(link).replace("'", "''") + quoted_target = str(target).replace("'", "''") + command = ( + "New-Item -ItemType Junction " + f"-Path '{quoted_link}' -Target '{quoted_target}' | Out-Null" + ) + result = subprocess.run( + ["powershell", "-NoProfile", "-Command", command], + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + raise unittest.SkipTest( + f"junction creation failed: {result.stderr.strip()}" + ) + return + try: + Path(link).symlink_to(target, target_is_directory=True) + except OSError as error: # pragma: no cover - environment-specific + raise unittest.SkipTest(f"symlink creation failed: {error}") + + +class ThreeFactDocumentationContracts(unittest.TestCase): + """Charter AC 1: role, ``never-edit``, and repository status are independent.""" + + def test_every_document_carries_role_never_edit_and_status_independently(self): + """Every in-policy document appears exactly once carrying an + independent role, an independent user-owned ``never_edit`` fact, and an + independent repository status; root ``GLOSSARY.md`` stays visible.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + documents = manifest_documents(root) + paths = [document["path"] for document in documents] + + self.assertEqual(len(paths), len(set(paths))) + self.assertIn("GLOSSARY.md", paths) + self.assertEqual( + {document["repository_status"] for document in documents}, + {"tracked", "untracked", "ignored or excluded"}, + ) + for document in documents: + with self.subTest(path=document["path"]): + self.assertIn( + "never_edit", + document, + "the manifest document record carries no independent " + "user-owned never-edit fact; role, never-edit, and " + "repository status must be three separate fields", + ) + + +class RoleEvidenceContracts(unittest.TestCase): + """Charter AC 2 and the locked role rule: the selected map file and its + declared current-truth routes are maintained evidence; an ordinary map + link is not role evidence; names and directories are hints only.""" + + def test_map_and_current_truth_are_maintained_and_map_links_are_not(self): + """In a policy-less repository only the selected map and its declared + current-truth route carry maintained evidence. A linked but + policy-less ``GLOSSARY.md`` remains ``unresolved`` — visible, its map + link still valid, but never silently maintained.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + payload = run_checker(root) + links = payload["health"]["categories"]["links"]["raw"] + self.assertEqual(links["valid"], links["checked"]) + + roles = { + document["path"]: document["role"] + for document in manifest_documents(root) + } + self.assertEqual(roles["docs/README.md"], "maintained") + self.assertEqual(roles["docs/STATE.md"], "maintained") + for path in ( + "GLOSSARY.md", + "docs/guide.md", + "docs/archive/2023-review.md", + "docs/generated/api.md", + "HISTORY-2024.md", + ): + with self.subTest(path=path): + self.assertEqual( + roles[path], + "unresolved", + "an ordinary map link, a filename, or a directory " + "name is not role evidence", + ) + + def test_hints_are_recorded_beside_unresolved_roles_never_as_roles(self): + """A name-shaped unresolved document records its hint separately from + its role — a suggestion for the human, never a classification.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + for path in ("docs/archive/2023-review.md", "HISTORY-2024.md"): + with self.subTest(hint=path): + self.assertEqual(records[path].get("role"), "unresolved") + self.assertIn( + "hint", + records[path], + "a name-shaped unresolved document must record its " + "hint separately from its role", + ) + + +class CommittedPolicyContracts(unittest.TestCase): + """Charter AC 3: one committed, deterministic, fail-closed policy block.""" + + def test_directory_rules_exact_exceptions_and_deepest_rule_win(self): + """Directory rules classify groups, the deepest directory rule wins, + and an exact path overrides its containing directory.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + roles = { + document["path"]: document["role"] + for document in manifest_documents(root) + } + + for path, expected in sorted(EXPECTED_ROLES.items()): + with self.subTest(path=path): + self.assertEqual( + roles[path], + expected, + "the committed docs:policy block must decide this " + "role deterministically", + ) + + def test_the_committed_policy_survives_a_fresh_clone(self): + """Git carries role decisions across clones: a fresh clone classifies + identically with no local state at all.""" + with tempfile.TemporaryDirectory() as td: + origin = Path(td) / "origin" + origin.mkdir() + classified_fixture(origin) + clone = Path(td) / "clone" + git(td, "clone", "--quiet", str(origin), str(clone)) + + roles = { + document["path"]: document["role"] + for document in manifest_documents(clone) + } + + for path, expected in sorted(EXPECTED_ROLES.items()): + with self.subTest(path=path): + self.assertEqual(roles.get(path), expected) + provenance = section(run_checker(clone), "documentation")["policy"] + self.assertEqual(provenance["git"], "committed") + self.assertTrue(provenance["durable"]) + self.assertIsNone(provenance["notice"]) + + def test_policy_replacement_preserves_crlf_map_bytes(self): + """A Windows checkout can preview a policy update without mixing line + endings or making a clean clone look locally edited.""" + from _docs_checker.documentation_policy import ( + PolicyRule, + replace_documentation_policy, + ) + + source = ( + "# Documentation\r\n\r\n" + "\r\n" + ) + proposed = replace_documentation_policy( + source, + ( + PolicyRule("docs", True, "maintained"), + PolicyRule("HISTORY-2024.md", False, "historical"), + ), + (), + ) + + self.assertNotIn("\n", proposed.replace("\r\n", "")) + self.assertNotIn("\r", proposed.replace("\r\n", "")) + self.assertIn( + "role historical HISTORY-2024.md\r\n", + proposed, + ) + + def test_policy_replacement_preserves_the_approved_block_order(self): + """PR 3 Stage 3B review fix B2: adding one never-edit entry to the + plan-approved example preserves every existing line byte-for-byte and + produces exactly one added diff line.""" + from _docs_checker.documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + ) + + source = ( + "\n" + ) + with tempfile.TemporaryDirectory() as td: + root = Path(td) + for directory in ( + "docs/archive", + "references/reviews", + "adapters", + ".github", + ): + (root / directory).mkdir(parents=True, exist_ok=True) + write(root, "docs/archive/current-migration.md", "# Migration\n") + write(root, "docs/frozen.md", "# Frozen\n") + write(root, "AGENTS.md", "# Instructions\n") + policy = parse_documentation_policy( + source, + root=root, + map_route="docs/README.md", + ) + proposed = replace_documentation_policy( + source, + policy.role_rules, + ( + *policy.never_edit_rules, + PolicyRule("docs/frozen.md", False, "never-edit"), + ), + ) + + changed = [ + line + for line in difflib.unified_diff( + source.splitlines(keepends=True), + proposed.splitlines(keepends=True), + ) + if line.startswith(("+", "-")) + and not line.startswith(("+++", "---")) + ] + self.assertEqual(changed, ["+never-edit docs/frozen.md\n"]) + self.assertEqual( + proposed.replace("never-edit docs/frozen.md\n", ""), + source, + "the hand-authored policy block must otherwise remain byte-for-byte", + ) + + def test_an_uncommitted_policy_edit_applies_and_is_not_yet_durable(self): + """The selected map's current working-tree bytes are policy truth for + this read, while provenance says an uncommitted decision is not yet + durable.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n" + + "\n", + encoding="utf-8", + newline="\n", + ) + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + + self.assertEqual(records["HISTORY-2024.md"]["role"], "historical") + self.assertEqual(documentation["policy"]["git"], "uncommitted") + self.assertFalse(documentation["policy"]["durable"]) + self.assertEqual( + documentation["policy"]["notice"], + "not-yet-durable", + ) + + def test_a_non_git_repository_applies_policy_with_no_saved_version_notice(self): + """Without Git, current map bytes still decide policy and provenance + honestly says there is no saved version to restore.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Current state](STATE.md) \n\n" + "\n", + ) + write(root, "docs/STATE.md", "# Current state\n\nCurrent truth.\n") + write(root, "HISTORY-2024.md", "# History 2024\n\nOld record.\n") + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + + self.assertEqual(records["HISTORY-2024.md"]["role"], "historical") + self.assertEqual(documentation["policy"]["git"], "unavailable") + self.assertFalse(documentation["policy"]["durable"]) + self.assertEqual( + documentation["policy"]["notice"], + "no-saved-version-to-restore", + ) + + def _assert_invalid_policy_envelope(self, payload, returncode): + """The existing failure-envelope contract, applied to policy input.""" + self.assertEqual( + returncode, + 2, + "an invalid docs:policy block must fail closed with the " + "engine's failure envelope, not measure normally", + ) + self.assertEqual(payload.get("error"), "invalid-policy") + self.assertEqual( + payload.get("navigation", {}).get("classification"), + "invalid-policy", + ) + # Fail-closed means no readiness percentage and no treatment + # authority may survive the refused input. + self.assertNotIn("readiness", payload) + self.assertNotIn("health", payload) + self.assertFalse( + payload.get("doctor_baseline", {}).get("treatment_authority") + ) + + def test_invalid_policy_fails_closed_as_the_invalid_policy_classification(self): + """Absolute paths, traversal, globs, unknown directives, unknown + roles, conflicting duplicates, and a second block each fail closed + with the existing failure-envelope classification ``invalid-policy`` + — never a silent partial apply, a readiness percentage, or treatment + authority.""" + second_block = "\n" + invalid_blocks = { + "absolute-path": "\n", + "parent-traversal": "\n", + "glob-selector": "\n", + "unknown-directive": "\n", + "unknown-role": "\n", + "conflicting-duplicate": ( + "\n" + ), + "second-policy-block": second_block + second_block, + } + for name, block in sorted(invalid_blocks.items()): + with self.subTest(invalid=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + "\n" + block, + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", name) + + payload, returncode = run_checker_process(root) + self._assert_invalid_policy_envelope(payload, returncode) + + baseline, baseline_returncode = run_checker_process( + root, "--doctor-baseline" + ) + self._assert_invalid_policy_envelope( + baseline, baseline_returncode + ) + + def _assert_unbound_policy_selector(self, directive): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + f"\n\n", + encoding="utf-8", + newline="\n", + ) + + payload, returncode = run_checker_process(root) + + self._assert_invalid_policy_envelope(payload, returncode) + self.assertEqual( + payload.get("navigation", {}).get("reason"), + "policy-selector-unbound", + ) + + def test_unbound_policy_file_selector_fails_closed(self): + """A file selector must bind to one existing confined working-tree + file; an inert typo is invalid policy rather than a silent no-op.""" + self._assert_unbound_policy_selector( + "role preserved docs/missing.md" + ) + + def test_unbound_policy_directory_selector_fails_closed(self): + """A trailing-slash selector must bind to an existing confined + working-tree directory for role and never-edit policy alike.""" + self._assert_unbound_policy_selector( + "never-edit docs/missing/" + ) + + def test_trailing_junk_role_selector_fails_closed(self): + """Extra role-line text may not turn an existing document path into + a different inert selector that silently binds nothing.""" + self._assert_unbound_policy_selector( + "role preserved docs/guide.md extra" + ) + + def test_policy_may_not_cool_the_selected_map_or_a_current_truth_route(self): + """The committed policy may not classify the selected map or a + declared current-truth route as preserved, generated, or historical: + both are maintained evidence by the locked role rule, so the + contradiction fails closed with the ``invalid-policy`` envelope and a + conflict reason that names the conflicting route.""" + conflicts = { + "selected-map-cooled": ( + "\n", + "docs/README.md", + ), + "current-truth-cooled": ( + "\n", + "docs/STATE.md", + ), + } + for name, (block, route) in sorted(conflicts.items()): + with self.subTest(conflict=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + "\n" + block, + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", name) + + payload, returncode = run_checker_process(root) + + self._assert_invalid_policy_envelope(payload, returncode) + self.assertIn( + route, + json.dumps(payload), + "the conflict reason must name the conflicting route", + ) + + def test_a_policy_selector_may_not_traverse_a_reparse_path(self): + """A policy selector naming a symlink, junction, or other reparse + route is refused with the same fail-closed ``invalid-policy`` + envelope: the committed policy cannot make the engine classify + documentation the repository boundary already refuses to read.""" + with tempfile.TemporaryDirectory() as td: + outside = Path(td) / "outside" + outside.mkdir() + write(outside, "escaped.md", "# Escaped\n\nOutside the repository.\n") + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + make_reparse_point(root / "docs" / "linked", outside) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "reparse") + + payload, returncode = run_checker_process(root) + + self._assert_invalid_policy_envelope(payload, returncode) + self.assertNotIn("Escaped", json.dumps(payload)) + + +class SharedResultContracts(unittest.TestCase): + """Charter AC 4 and 14: one engine result for every route and every host.""" + + def test_map_check_audit_doctor_and_init_consume_one_result(self): + """Map, Check, and Audit are presentations of the same flagless + default engine route, so one invocation of that route speaks for all + three. Doctor's first contact is ``--doctor-baseline``, Init records + the same measurement under ``repository_measurement``, and an explicit + repository-wide override changes nothing. Every route publishes one + identical engine-owned documentation and readiness result for the + same snapshot.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + init_payload = run_checker(root, "--init-discovery") + envelopes = { + "map-check-audit-default": run_checker(root), + "doctor-first-contact": run_checker(root, "--doctor-baseline"), + "human-repository-override": run_checker(root, "--scope", "."), + "init-repository-measurement": init_payload[ + "repository_measurement" + ], + } + + rendered = { + name: ( + json.dumps(section(payload, "documentation"), sort_keys=True), + json.dumps(section(payload, "readiness"), sort_keys=True), + ) + for name, payload in envelopes.items() + } + self.assertEqual(len(set(rendered.values())), 1, sorted(rendered)) + + def test_an_explicit_component_route_stays_visibly_partial(self): + """A human-requested component keeps its own boundary, reports itself + partial, and still carries the engine documentation and readiness + sections rather than reverting to a model-authored view.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + payload = run_checker(root, "--scope", "docs") + + self.assertEqual(payload["boundary"]["mode"], "component") + self.assertIs(payload["boundary"]["partial"], True) + section(payload, "documentation") + section(payload, "readiness") + + def test_one_snapshot_produces_one_result_no_matter_who_asks(self): + """Two invocations of the same route on one unchanged snapshot return + byte-identical documentation and readiness sections; host models can + rephrase the result but cannot change it.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + first = run_checker(root) + second = run_checker(root) + + self.assertEqual( + json.dumps(section(first, "documentation"), sort_keys=True), + json.dumps(section(second, "documentation"), sort_keys=True), + ) + self.assertEqual( + json.dumps(section(first, "readiness"), sort_keys=True), + json.dumps(section(second, "readiness"), sort_keys=True), + ) + + +class ReadinessContracts(unittest.TestCase): + """Charter AC 5, 6, and 7: one unweighted, fact-backed, honest fraction.""" + + def test_readiness_is_an_unweighted_fraction_with_visible_counts(self): + """Readiness equals ready maintained documents over eligible + maintained documents — no weights, no partial credit, the counts stay + authoritative, and the secondary percentage rounds the fraction to + the nearest whole number.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + readiness = section(run_checker(root), "readiness") + + self.assertEqual(readiness.get("eligible_count"), EXPECTED_ELIGIBLE) + self.assertEqual(readiness.get("ready_count"), EXPECTED_READY) + self.assertEqual(readiness.get("percentage"), EXPECTED_PERCENTAGE) + for weighted in ("weights", "earned_weight", "categories"): + self.assertNotIn( + weighted, + readiness, + "the readiness result must not carry weighted-rubric " + "machinery", + ) + + def test_every_eligible_document_exposes_all_four_facts(self): + """Every eligible document row exposes its reachability, + internal-link/anchor, title, and confinement facts individually, so + the counts are traceable to named evidence.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + self.assertEqual(len(rows), EXPECTED_ELIGIBLE) + for path, row in sorted(rows.items()): + with self.subTest(path=path): + facts = row.get("facts") + self.assertIsInstance(facts, dict) + self.assertEqual(set(facts), set(READINESS_FACTS)) + + def test_every_unready_document_names_each_failed_fact(self): + """An unready document names the exact facts that failed — the broken + link target, the unreachable verdict — never a bare deduction.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + guide = rows["docs/guide.md"] + self.assertIs(guide.get("ready"), False) + self.assertIn("missing-page.md", json.dumps(guide)) + lonely = rows["docs/lonely.md"] + self.assertIs(lonely.get("ready"), False) + self.assertIn("reachability", json.dumps(lonely.get("failed_facts"))) + + def test_a_repository_without_a_selected_map_reports_no_percentage(self): + """No selected map means no readiness percentage — an honest no-map + state, never a misleading low number.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write(root, "notes/ideas.md", "# Ideas\n\nLoose notes only.\n") + write(root, "notes/plans.md", "# Plans\n\nMore loose notes.\n") + git(root, "add", "--", "notes") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + readiness = section(run_checker(root), "readiness") + + self.assertIsNone( + readiness.get("percentage"), + "a missing selected map must yield no percentage; today the " + "weighted rubric presents a number for this repository", + ) + + def test_incomplete_accounting_publishes_no_percentage(self): + """When documentation accounting is incomplete, the readiness + percentage is absent or null: a partial inventory must never publish + a complete-looking fraction.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + # No Git: the filesystem accounting prunes ``node_modules`` + # without inspecting it, so repository coverage is honestly + # incomplete on this snapshot. + write(root, "docs/README.md", "# Documentation\n\n- [Guide](guide.md)\n") + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + write(root, "node_modules/vendor.md", "# Vendor\n\nPruned body.\n") + + manifest = manifest_payload(build_manifest(root)) + self.assertFalse(manifest["coverage"]["complete"]) + + readiness = section(run_checker(root), "readiness") + + self.assertIsNone( + readiness.get("percentage"), + "incomplete accounting must never publish a complete-looking " + "readiness fraction", + ) + + def test_a_broken_internal_anchor_fails_the_links_fact(self): + """A repository-internal link whose anchor does not resolve fails the + source document's link fact by name, exactly like a missing target.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Alpha](a.md)\n" + "- [Beta](b.md)\n" + "\n\n", + ) + write(root, "docs/a.md", "# Alpha\n\nSee [Beta](b.md#missing-anchor).\n") + write(root, "docs/b.md", "# Beta\n\nBody.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "anchor") + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + self.assertIs(rows["docs/b.md"].get("ready"), True) + self.assertIs(rows["docs/a.md"].get("ready"), False) + self.assertIn("missing-anchor", json.dumps(rows["docs/a.md"])) + + def test_duplicate_titles_are_normalized_across_case_and_whitespace(self): + """Title identity is normalized: two H1s differing only by letter + case and internal whitespace are duplicates, so identical map labels + fail both routes with ``ambiguous-title``.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](a.md)\n" + "- [Guide](b.md)\n" + "\n\n", + ) + write(root, "docs/a.md", "# Guide\n\nFirst body.\n") + write(root, "docs/b.md", "# GUIDE \n\nSecond body.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "normalized") + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + for path in ("docs/a.md", "docs/b.md"): + with self.subTest(path=path): + self.assertIs(rows[path].get("ready"), False) + self.assertIn("ambiguous-title", json.dumps(rows[path])) + + def test_duplicate_titles_need_distinct_map_labels_to_pass(self): + """The deterministic title contract: duplicate H1s pass only when + every duplicate route has a distinct human-facing label in the + selected map; otherwise each affected route fails ``ambiguous-title``. + The third case pins that distinctness is judged among the map labels + themselves — two routes sharing one label fail even when that label + differs from the duplicated H1.""" + cases = { + "distinct-labels": ("[Alpha guide](a.md)", "[Beta guide](b.md)", True), + "identical-labels": ("[Guide](a.md)", "[Guide](b.md)", False), + "duplicate-labels-differing-from-title": ( + "[Alpha guide](a.md)", + "[Alpha guide](b.md)", + False, + ), + } + for name, (first_label, second_label, expect_ready) in sorted(cases.items()): + with self.subTest(case=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + f"- {first_label}\n" + f"- {second_label}\n" + "\n\n", + ) + write(root, "docs/a.md", "# Guide\n\nFirst body.\n") + write(root, "docs/b.md", "# Guide\n\nSecond body.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", name) + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + for path in ("docs/a.md", "docs/b.md"): + with self.subTest(path=path): + self.assertIs(rows[path].get("ready"), expect_ready) + if not expect_ready: + self.assertIn( + "ambiguous-title", json.dumps(rows[path]) + ) + + def test_trust_stays_out_of_readiness(self): + """The selected map, reader entry, and current-truth declarations stay + visible facts, and Trust never enters the readiness fraction.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + payload = run_checker(root) + self.assertEqual(payload["navigation"]["authority"], "docs/README.md") + self.assertIn("trust_status", payload["health"]) + + readiness = section(payload, "readiness") + for trust_field in ("trust", "trust_status", "freshness", "coverage"): + self.assertNotIn( + trust_field, + readiness, + "Trust is reported separately and may never enter the " + "readiness fraction", + ) + + +class RubricCompatibilityContracts(unittest.TestCase): + """Charter AC 13: stored rubric-v2 state stays readable, uncompared, and + unmodified. Uses the real public Init journey via the existing lifecycle + fixture helpers, with the stored evidence pinned to rubric version 2.""" + + @staticmethod + def apply_v2_doctor_fixture(root): + """Initialize a repository whose stored baseline is a rubric-v2 score.""" + import copy as _copy + + from tests.test_init_closeout import ( + build_doctor_request, + build_repository, + run_closeout, + ) + + build_repository(root) + links = "\n".join( + f"- [Verified page {index:03d}](page-{index:03d}.md)" + for index in range(71, 103) + ) + (root / "docs" / "README.md").write_text( + "# Verified page 000\n\nUnicode fact: café — ✓.\n\n" + "## Current routes\n\n" + links + "\n", + encoding="utf-8", + newline="\n", + ) + preview_request = build_doctor_request(root) + preview_request["evidence"]["rubric_version"] = 2 + preview_process = run_closeout(root, preview_request) + if preview_process.returncode != 0: + raise AssertionError(preview_process.stdout) + preview = json.loads(preview_process.stdout) + apply_request = _copy.deepcopy(preview_request) + apply_request.update(operation="apply", approval=preview["approval"]) + apply_process = run_closeout(root, apply_request) + if apply_process.returncode != 0: + raise AssertionError(apply_process.stdout) + + def test_a_stored_v2_baseline_stays_readable_without_a_v3_delta(self): + """An initialized repository whose state stores a rubric-v2 baseline + remains readable, the versioned readiness result publishes no + v2-to-v3 delta while making the incompatibility visible, and the + read-only routes modify no state file.""" + from tests.test_init_closeout import tree_snapshot + + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self.apply_v2_doctor_fixture(root) + stored = json.loads( + (root / ".diataxis" / "state.json").read_text(encoding="utf-8") + ) + self.assertEqual(stored["rubric"]["version"], 2) + before = tree_snapshot(root) + + baseline_payload, baseline_returncode = run_checker_process( + root, "--doctor-baseline" + ) + default_payload, default_returncode = run_checker_process(root) + + self.assertEqual(baseline_returncode, 0) + self.assertEqual(default_returncode, 0) + self.assertEqual( + baseline_payload["doctor_baseline"]["status"], "measured" + ) + self.assertEqual( + tree_snapshot(root), + before, + "a read-only command modified an operational state file", + ) + + readiness = section(default_payload, "readiness") + self.assertEqual(readiness.get("version"), 3) + self.assertNotIn( + "delta", + readiness, + "readiness v3 must not fabricate a delta against a stored " + "rubric-v2 baseline", + ) + baseline_info = readiness.get("baseline") + self.assertIsNotNone( + baseline_info, + "the stored-baseline incompatibility must be visible, not " + "silently dropped", + ) + self.assertEqual(baseline_info.get("rubric_version"), 2) + self.assertIs(baseline_info.get("comparable"), False) + + +class NeverEditFactContracts(unittest.TestCase): + """Charter AC 8: never-edit changes writes, never visibility or scoring. + Write-eligibility reporting itself is a Stage 3 treatment contract.""" + + def test_never_edit_is_a_visible_fact_that_keeps_the_document_scored(self): + """A maintained ``never-edit`` document stays visible with its + ``never_edit`` fact true, and stays inside the readiness fraction — + it never disappears and never leaves the denominator.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + payload = run_checker(root) + documentation = section(payload, "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + frozen = records["docs/frozen.md"] + + self.assertEqual(frozen.get("role"), "maintained") + self.assertIs(frozen.get("never_edit"), True) + readiness = section(payload, "readiness") + self.assertEqual(readiness.get("eligible_count"), EXPECTED_ELIGIBLE) + readiness_rows = {row["path"] for row in readiness["documents"]} + self.assertIn("docs/frozen.md", readiness_rows) + + +class GroupingContracts(unittest.TestCase): + """Charter AC 9: deterministic groups with exact counts and full routes.""" + + def test_every_cold_group_is_deterministic_with_counts_and_routes(self): + """Preserved, generated, historical, unresolved, and ignored/excluded + material is grouped by the engine deterministically — exact counts, + complete route lists, no model-invented summaries — and two runs + agree byte for byte.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + first = section(run_checker(root), "documentation") + second = section(run_checker(root), "documentation") + + self.assertEqual( + json.dumps(first.get("groups"), sort_keys=True), + json.dumps(second.get("groups"), sort_keys=True), + ) + groups = first.get("groups") + self.assertTrue(groups, "the documentation result carries no groups") + by_kind = {} + for group in groups: + kind = group.get("kind") + routes = list(group.get("routes", ())) + self.assertEqual(group.get("count"), len(routes)) + by_kind.setdefault(kind, []).extend(routes) + for kind, expected_routes in sorted(EXPECTED_GROUPS.items()): + with self.subTest(kind=kind): + self.assertEqual(list(by_kind.get(kind, ())), expected_routes) + + def test_reaching_an_accounting_limit_reports_incomplete_not_complete(self): + """More ignored documentation than the accounting limit yields an + honest ``coverage incomplete`` documentation result naming what was + dropped — never a claim of complete accounting.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + for index in range(2060): + write( + root, + f"private/bulk/note-{index:04d}.md", + f"# Note {index:04d}\n", + ) + + manifest = manifest_payload(build_manifest(root)) + self.assertFalse(manifest["coverage"]["complete"]) + self.assertIn( + "ignored-document-limit", + manifest["coverage"]["incomplete_reasons"], + ) + + payload = run_checker(root) + documentation = section(payload, "documentation") + coverage = documentation.get("coverage", {}) + self.assertIs(coverage.get("complete"), False) + self.assertIn( + "ignored-document-limit", + json.dumps(coverage), + "the documentation result must name the limit that made its " + "accounting incomplete", + ) + self.assertIsNone( + section(payload, "readiness").get("percentage"), + "incomplete accounting must never publish a complete-looking " + "readiness fraction", + ) + + +class ColdMaterialContracts(unittest.TestCase): + """Charter AC 8 and 12, read-result half: classified cold material leaves + readiness with a named reason instead of surfacing as repair findings.""" + + def test_classified_cold_documents_are_excluded_with_reasons_not_findings(self): + """Policy-classified preserved, generated, and historical documents + stop surfacing as ``unreachable`` repair findings; each leaves the + readiness fraction with a visible named reason instead.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + payload = run_checker(root) + cold_paths = { + path + for path, role in EXPECTED_ROLES.items() + if role in {"preserved", "generated", "historical"} + } + misfired = [ + finding + for finding in payload["findings"] + if finding.get("kind") == "unreachable" + and finding.get("path") in cold_paths + ] + self.assertEqual( + misfired, + [], + "policy-classified cold documents must not surface as " + "navigation repair findings", + ) + + excluded = { + row["path"]: row + for row in section(payload, "readiness").get("excluded", ()) + } + for path in sorted(cold_paths | {"private/secret.md"}): + with self.subTest(path=path): + self.assertIn(path, excluded) + self.assertTrue(str(excluded[path].get("reason", "")).strip()) + + def test_all_routes_use_the_same_published_findings_and_exit_status(self): + """P1-1: agent JSON, plain JSON, and text all consume the published + suppressed findings set instead of independently using raw findings.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Current state](STATE.md) \n\n" + "\n", + ) + write(root, "docs/STATE.md", "# Current state\n\nCurrent truth.\n") + write(root, "docs/archive/old.md", "# Old record\n\nPreserved.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + agent = run_checker_route_process(root, "--json", "--agent") + plain_json = run_checker_route_process(root, "--json") + text = run_checker_route_process(root) + agent_payload = json.loads(agent.stdout) + plain_payload = json.loads(plain_json.stdout) + + self.assertEqual(agent_payload["findings"], []) + self.assertEqual(plain_payload["findings"], agent_payload["findings"]) + self.assertEqual(text.stdout.strip(), "clean") + self.assertEqual(agent.returncode, 0) + self.assertEqual(plain_json.returncode, 0) + self.assertEqual(text.returncode, 0) + + +class SafetyContracts(unittest.TestCase): + """Charter AC 15: the new result weakens no existing guarantee.""" + + def test_documentation_and_readiness_are_zero_write_and_private(self): + """Producing the documentation and readiness result writes nothing, + never reads an ignored body, and leaks no private content into any + route's published envelope.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + before = git(root, "status", "--porcelain").stdout + + payloads = { + "first_doctor": run_checker(root, "--doctor-baseline"), + "default": run_checker(root), + "human_override": run_checker(root, "--scope", "."), + } + + self.assertEqual(git(root, "status", "--porcelain").stdout, before) + for name, payload in payloads.items(): + with self.subTest(route=name): + self.assertNotIn("PRIVATE_BODY_SENTINEL", json.dumps(payload)) + documentation = section(payload, "documentation") + ignored = [ + document + for document in documentation["documents"] + if document["path"] == "private/secret.md" + ] + self.assertTrue(ignored) + self.assertIs(ignored[0].get("body_inspected"), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_init_adoption_cli.py b/tests/test_init_adoption_cli.py index ba19fd9..f309d5f 100644 --- a/tests/test_init_adoption_cli.py +++ b/tests/test_init_adoption_cli.py @@ -212,7 +212,15 @@ def test_engine_owned_preview_is_all_unchanged_and_zero_repository_write(self): ) self.assertEqual(sum(row["available"] for row in payload["score_receipt"]["categories"].values()), 100) self.assertNotIn("PRIVATE_SENTINEL_DO_NOT_READ", serialized) - self.assertNotIn("docs/local/private.md", serialized) + ignored = next( + document + for document in payload["repository_measurement"]["documentation"][ + "documents" + ] + if document["path"] == "docs/local/private.md" + ) + self.assertEqual(ignored["repository_status"], "ignored or excluded") + self.assertIs(ignored["body_inspected"], False) self.assertNotIn("Shared guidance.", serialized) self.assertTrue(receipt.is_file()) self.assertEqual(_snapshot(repo), before) diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py new file mode 100644 index 0000000..bd7f9ab --- /dev/null +++ b/tests/test_resolution_treatment_contract.py @@ -0,0 +1,859 @@ +"""PR 3 Stage 3 contracts: unresolved resolution and treatment policy. + +These are the resolution and treatment-eligibility promises of the PR 3 +charter — Stage 3 work, deliberately separated from the Stage 2 canonical +documentation-result contracts in ``tests/test_documentation_result_contract.py`` +so Stage 2 can finish with its owned tests green while these stay visibly +pending. + +These tests began as strict expected failures so an owning stage could not +leave an obsolete marker behind. Stage 3 removes each marker and its +bookkeeping entry only with the implemented behavior green. + +The engine unit names probed here — ``_docs_checker.resolution`` and +``_docs_checker.treatment_policy`` — and the ``write_eligible`` field are the +Stage 1 proposal of the Stage 3 contract surface, open to rename in review. +""" + +import importlib +import json +import subprocess +import sys +import tempfile +import unittest +from collections.abc import Mapping +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SCRIPTS = ROOT / "skills" / "docs" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from tests.test_documentation_result_contract import ( # noqa: E402 + LEAVE_UNRESOLVED_CHOICE, + RESOLUTION_CHOICES, + _init_git, + build_manifest, + classified_fixture, + git, + hinted_fixture, + run_checker, + section, + write, +) + + +#: Every strict expected failure below names its owning phase here as well as +#: in its docstring, so the owner can find its work by grepping one table. +EXPECTED_FAILURE_OWNERS = {} + + +def run_resolution_process(root, action, decisions, approval=None): + script = """ +import json +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[1]) +from _docs_checker.resolution import resolution_apply, resolution_preview + +root = Path(sys.argv[2]) +decisions = json.loads(sys.argv[4]) +if sys.argv[3] == "preview": + result = resolution_preview(root, decisions) +else: + result = resolution_apply(root, decisions, sys.argv[5]) +print(json.dumps(result, sort_keys=True)) +""" + return subprocess.run( + [ + sys.executable, + "-B", + "-c", + script, + str(SCRIPTS), + str(root), + action, + json.dumps(decisions, sort_keys=True), + "" if approval is None else approval, + ], + capture_output=True, + text=True, + cwd=str(ROOT), + ) + + +class UnresolvedResolutionContracts(unittest.TestCase): + """Decision 13 and charter AC 10: unresolved asks the human plainly and + persists nothing before one approved write.""" + + def test_an_unresolved_document_offers_reason_choices_and_a_handle(self): + """PRODUCT.md 'Unresolved asks the human plainly': every unresolved + document carries its exact path, one plain-English reason, the bounded + role choices, an engine-generated handle, and a copy-paste reply that + quotes the handle. Owned by PR 3 Stage 3.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + documentation = section(run_checker(root), "documentation") + cards = {card["path"]: card for card in documentation["unresolved"]} + + self.assertIn("HISTORY-2024.md", cards) + card = cards["HISTORY-2024.md"] + self.assertTrue(str(card.get("reason", "")).strip()) + # The choices are the four persistable roles plus the + # owner-approved ``leave unresolved`` reply, which is a human + # choice and never becomes a persisted role rule. + self.assertEqual(set(card.get("choices", ())), set(RESOLUTION_CHOICES)) + handle = card.get("handle", "") + self.assertTrue(isinstance(handle, str) and handle.strip()) + self.assertIn(handle, str(card.get("reply", ""))) + + def test_gitignored_documents_never_receive_classification_cards(self): + """P2-2: ignored documentation stays visible only by inventory name + and count; its exact path may not appear in any classification card.""" + from _docs_checker.resolution import unresolved_cards + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + cards = unresolved_cards(build_manifest(root)) + serialized = json.dumps(cards, sort_keys=True) + + self.assertNotIn("private/secret.md", serialized) + for card in cards: + self.assertNotEqual(card.get("path"), "private/secret.md") + self.assertNotIn( + "private/secret.md", + card.get("routes", ()), + ) + + def test_directory_groups_resolve_in_one_reply_with_exact_exceptions(self): + """PRODUCT.md 'Unresolved asks the human plainly': large unresolved + sets are summarized and batchable. A directory of unresolved + documents receives one deterministic group handle with its exact + count and complete route list, one group reply classifies the whole + directory, and an exact-file reply overrides its containing group. + Owned by PR 3 Stage 3.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + for index in range(1, 4): + write( + root, + f"research/note-{index:02d}.md", + f"# Note {index:02d}\n\nRecorded result.\n", + ) + git(root, "add", "--", "research") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "research") + + first = section(run_checker(root), "documentation") + second = section(run_checker(root), "documentation") + groups = [ + card + for card in first["unresolved"] + if card.get("path") == "research/" + ] + self.assertEqual( + [card for card in second["unresolved"] if card.get("path") == "research/"], + groups, + "group handles must be deterministic across runs", + ) + self.assertEqual(len(groups), 1) + group = groups[0] + self.assertEqual(group.get("count"), 3) + self.assertEqual( + sorted(group.get("routes", ())), + [ + "research/note-01.md", + "research/note-02.md", + "research/note-03.md", + ], + ) + self.assertTrue(str(group.get("handle", "")).strip()) + + try: + resolution = importlib.import_module("_docs_checker.resolution") + except ImportError: + self.fail( + "the engine has no unresolved-resolution unit yet; a " + "directory group must resolve in one reply" + ) + preview = resolution.resolution_preview( + root, + { + "research/": {"role": "preserved"}, + "research/note-02.md": {"role": "historical"}, + }, + ) + map_diff = str(preview.get("map_diff", "")) + self.assertIn("role preserved research/", map_diff) + self.assertIn("role historical research/note-02.md", map_diff) + + def test_resolving_a_role_previews_an_exact_map_diff_with_a_short_code(self): + """PRODUCT.md 'Consent and approval': a role resolution produces an + exact proposed map-policy diff and a short engine-owned approval code, + and persists nothing before that approval; full SHA-256 receipts stay + internal. ``leave unresolved`` is a human choice, not a role rule: + it contributes nothing to the proposed policy edit and the document + simply stays unresolved. Owned by PR 3 Stage 3.""" + try: + resolution = importlib.import_module("_docs_checker.resolution") + except ImportError: + self.fail( + "the engine has no unresolved-resolution unit yet; role and " + "never-edit decisions must preview one approved policy edit" + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + before = git(root, "status", "--porcelain").stdout + + preview = resolution.resolution_preview( + root, + { + "HISTORY-2024.md": {"role": "historical"}, + "docs/archive/2023-review.md": { + "role": LEAVE_UNRESOLVED_CHOICE + }, + }, + ) + + map_diff = str(preview.get("map_diff", "")) + self.assertTrue(map_diff) + self.assertIn("HISTORY-2024.md", map_diff) + self.assertNotIn( + "2023-review.md", + map_diff, + "leave unresolved persists no role rule for the document", + ) + approval = str(preview.get("approval", "")) + self.assertTrue(approval.strip()) + self.assertNotRegex(approval, r"[0-9a-f]{64}") + self.assertEqual(git(root, "status", "--porcelain").stdout, before) + + def test_leave_unresolved_alone_proposes_no_policy_write(self): + """The owner-approved unresolved choice is not a durable role rule and + therefore produces no diff or approval when it is the only reply.""" + from _docs_checker.resolution import resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + before = map_path.read_bytes() + + preview = resolution_preview( + root, + { + "HISTORY-2024.md": { + "role": LEAVE_UNRESOLVED_CHOICE + } + }, + ) + + self.assertEqual(preview["status"], "no-change") + self.assertEqual(preview["map_diff"], "") + self.assertIsNone(preview["approval"]) + self.assertEqual(map_path.read_bytes(), before) + + def test_resolution_apply_is_exact_single_use_and_visible_on_a_fresh_run(self): + """Stage 3B resolution apply changes only the selected map, consumes + its exact code once, and leaves the working-tree policy immediately + authoritative with honest uncommitted provenance.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + before = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.parts + } + + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + + self.assertEqual(applied["status"], "applied") + self.assertEqual(applied["writes"], 1) + after = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.parts + } + self.assertEqual( + [ + path + for path in sorted(before) + if before[path] != after[path] + ], + ["docs/README.md"], + ) + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + self.assertEqual(records["HISTORY-2024.md"]["role"], "historical") + self.assertEqual(documentation["policy"]["git"], "uncommitted") + + reused = resolution_apply(root, decisions, preview["approval"]) + self.assertEqual(reused["status"], "refused") + self.assertEqual(reused["writes"], 0) + self.assertTrue(str(reused.get("reason", "")).strip()) + + def test_resolution_receipt_survives_a_process_boundary(self): + """P2-1: a DOCS-R receipt is a stateless digest binding, so preview + and apply may run in separate Python processes without local memory.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + decisions = {"HISTORY-2024.md": {"role": "historical"}} + + preview_process = run_resolution_process( + root, + "preview", + decisions, + ) + self.assertEqual( + preview_process.returncode, + 0, + preview_process.stderr, + ) + preview = json.loads(preview_process.stdout) + + apply_process = run_resolution_process( + root, + "apply", + decisions, + preview["approval"], + ) + self.assertEqual( + apply_process.returncode, + 0, + apply_process.stderr, + ) + applied = json.loads(apply_process.stdout) + + self.assertEqual(applied["status"], "applied") + self.assertEqual(applied["writes"], 1) + documents = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + self.assertEqual( + documents["HISTORY-2024.md"]["role"], + "historical", + ) + + def test_resolution_apply_refuses_drift_with_zero_writes(self): + """Stage 3B resolution apply compares the selected map's current + bytes to its preview binding and names drift without writing.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = resolution_preview(root, decisions) + map_path.write_text( + map_path.read_text(encoding="utf-8") + "\nOwner edit.\n", + encoding="utf-8", + newline="\n", + ) + drifted = map_path.read_bytes() + + refused = resolution_apply(root, decisions, preview["approval"]) + + self.assertEqual(refused["status"], "refused") + self.assertIn( + refused["reason"], + {"approval-code-not-current", "selected-map-drift"}, + ) + self.assertEqual(refused["writes"], 0) + self.assertEqual(map_path.read_bytes(), drifted) + + def test_a_superseding_resolution_preview_invalidates_the_older_code(self): + """A code bound to older decisions cannot approve a newer decision + mapping; exact recomputation makes the mismatch fail closed.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + first_decisions = {"HISTORY-2024.md": {"role": "historical"}} + second_decisions = {"HISTORY-2024.md": {"role": "preserved"}} + first = resolution_preview(root, first_decisions) + second = resolution_preview(root, second_decisions) + + refused = resolution_apply( + root, + second_decisions, + first["approval"], + ) + + self.assertEqual(refused["status"], "refused") + self.assertEqual(refused["writes"], 0) + self.assertEqual( + refused["reason"], + "approval-code-not-current", + ) + self.assertEqual( + resolution_apply( + root, + second_decisions, + second["approval"], + )["status"], + "applied", + ) + + def test_duplicate_resolution_selectors_fail_closed(self): + """Stage 3B consistency: a decisions mapping that emits the same + selector twice is rejected with one named error.""" + from _docs_checker.resolution import resolution_preview + + class DuplicateSelectors(Mapping): + def __getitem__(self, key): + return {"role": "historical"} + + def __iter__(self): + return iter(("HISTORY-2024.md",)) + + def __len__(self): + return 1 + + def items(self): + return ( + ("HISTORY-2024.md", {"role": "historical"}), + ("HISTORY-2024.md", {"role": "preserved"}), + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + with self.assertRaisesRegex( + ValueError, + "duplicate-resolution-selector", + ): + resolution_preview(root, DuplicateSelectors()) + + +class TreatmentEligibilityContracts(unittest.TestCase): + """Owner gates 3 and 4 and charter AC 11 and 12: write authority follows + role, status, confinement, and the user-owned never-edit list; navigation + treatments need a purpose, never a score.""" + + def test_never_edit_removes_write_eligibility_with_a_named_reason(self): + """PRODUCT.md 'Classification is three separate facts': a maintained + ``never-edit`` document keeps its role and readiness membership while + Doctor clearly reports that it will not propose edits to it. + Owned by PR 3 Stage 3.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + frozen = records["docs/frozen.md"] + + self.assertIs( + frozen.get("write_eligible"), + False, + "never-edit must remove write eligibility while the document " + "stays classified and scored", + ) + self.assertTrue(str(frozen.get("write_ineligible_reason", "")).strip()) + + def test_init_proposes_github_and_accepts_explicit_never_edit_entries(self): + """PRODUCT.md 'Commands and the jobs they do' plus owner gate 3: Init + proposes ``.github/`` for the initial never-edit list when it exists + and accepts explicit repository-relative entries; no conventional + filename is a hardcoded ban. Owned by PR 3 Stage 3.""" + from _docs_checker.init_adoption import adoption_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + git(root, "init", "--quiet") + git(root, "config", "user.email", "fixture@example.invalid") + git(root, "config", "user.name", "Fixture") + write(root, "docs/README.md", "# Documentation\n\n- [Guide](guide.md)\n") + write(root, "docs/guide.md", "# Guide\n\nShared guidance.\n") + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask.\n") + git(root, "add", "--", "docs", ".github") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + request, _preview = adoption_preview(root) + evidence = request["evidence"] + self.assertIn( + "never_edit", + evidence, + "Init's preview evidence carries no never-edit proposal; it " + "must propose .github/ as the initial entry when present", + ) + self.assertIn(".github/", evidence["never_edit"]) + + try: + explicit, _ = adoption_preview(root, never_edit=("docs/guide.md",)) + except TypeError: + self.fail( + "Init does not yet accept explicit repository-relative " + "never-edit entries" + ) + self.assertIn("docs/guide.md", explicit["evidence"]["never_edit"]) + + def test_an_explicit_empty_init_choice_declines_the_github_suggestion(self): + """PRODUCT.md 'Commands and the jobs they do' plus owner gate 3: the + ``.github/`` never-edit entry is a suggestion, not a mandate — an + explicit empty user choice declines it and the preview evidence + carries no never-edit entry at all. Owned by PR 3 Stage 3.""" + from _docs_checker.init_adoption import adoption_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write(root, "docs/README.md", "# Documentation\n\n- [Guide](guide.md)\n") + write(root, "docs/guide.md", "# Guide\n\nShared guidance.\n") + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask.\n") + git(root, "add", "--", "docs", ".github") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + try: + declined, _preview = adoption_preview(root, never_edit=()) + except TypeError: + self.fail( + "Init does not yet distinguish an explicit empty " + "never-edit choice from no choice at all" + ) + self.assertNotIn( + ".github/", + tuple(declined["evidence"].get("never_edit", ())), + "an explicit empty choice declines the suggested entry", + ) + + def test_approved_init_persists_each_never_edit_choice_into_the_map(self): + """Stage 3B review fix B1: Init's default suggestion and an explicit + choice survive exact approved apply in the canonical map, while an + explicit empty choice persists no never-edit rule.""" + from _docs_checker.documentation_policy import ( + load_documentation_policy, + never_edit_rule_for, + ) + from _docs_checker.init_adoption import adoption_apply, adoption_preview + + cases = ( + ("suggested", None, ".github/", ".github/SUPPORT.md"), + ("explicit", ("docs/guide.md",), "docs/guide.md", "docs/guide.md"), + ("declined", (), None, "docs/guide.md"), + ) + for name, choices, expected_rule, checked_path in cases: + with self.subTest(choice=name), tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask.\n") + if name == "suggested": + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", ".github", "docs/README.md") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "support", + ) + + arguments = {"completed_at": "2026-07-29T00:00:00Z"} + if choices is not None: + arguments["never_edit"] = choices + request, preview = adoption_preview(root, **arguments) + result = adoption_apply(root, request, preview["approval"]) + + self.assertEqual(result["status"], "applied") + policy = load_documentation_policy(root, "docs/README.md") + if expected_rule is None: + self.assertEqual(policy.never_edit_rules, ()) + else: + self.assertIsNotNone( + never_edit_rule_for( + policy, + checked_path, + ) + ) + if name == "suggested": + self.assertIsNotNone( + never_edit_rule_for(policy, "docs/STATE.md"), + "Init's suggestion must not erase an existing " + "user-owned never-edit entry", + ) + documents = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + self.assertIs( + documents[checked_path]["never_edit"], + expected_rule is not None, + ) + + def test_the_selected_map_stays_write_eligible_unless_user_never_edits_it(self): + """PRODUCT.md 'Classification is three separate facts' plus owner + gate 3: there are no hardcoded filename bans, so the selected map + stays write-eligible — Doctor's founding job is repairing it — unless + the user places it on the never-edit list, which flips exactly the + write fact and nothing else. Owned by PR 3 Stage 3.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + readme = records["docs/README.md"] + self.assertEqual(readme.get("role"), "maintained") + self.assertIs(readme.get("never_edit"), False) + self.assertIs( + readme.get("write_eligible"), + True, + "no hardcoded ban may make the selected map unwritable", + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "never-edit-map") + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + readme = records["docs/README.md"] + self.assertEqual(readme.get("role"), "maintained") + self.assertIs(readme.get("never_edit"), True) + self.assertIs(readme.get("write_eligible"), False) + + def test_sensitive_filenames_warn_but_are_not_refused(self): + """PRODUCT.md 'Who decides what' plus owner gate 3: sensitive + conventional filenames produce warnings, not refusals — a maintained + ``SECURITY.md`` carries a visible warning while staying fully + write-eligible after exact approval. Owned by PR 3 Stage 3.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n" + "- [Security policy](../SECURITY.md)\n" + "\n\n", + ) + write(root, "docs/guide.md", "# Guide\n\nShared guidance.\n") + write(root, "SECURITY.md", "# Security policy\n\nReport privately.\n") + git(root, "add", "--", "docs", "SECURITY.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "sensitive") + + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + security = records["SECURITY.md"] + + self.assertEqual(security.get("role"), "maintained") + self.assertIs( + security.get("write_eligible"), + True, + "a sensitive filename warns; it never becomes a hardcoded " + "refusal", + ) + self.assertTrue( + str(security.get("warning", "")).strip(), + "the sensitive-filename warning must be visible on the " + "document record", + ) + + def test_purpose_backed_links_are_eligible_and_score_only_links_refused(self): + """PRODUCT.md 'Target output shapes' plus owner gate 4: a proposed + navigation link must name its purpose, exact source, exact + destination, and wording; a link to non-maintained material earns no + readiness credit, and a proposal whose only effect is moving the + fraction is refused with a named reason. Owned by PR 3 Stage 3.""" + try: + treatment_policy = importlib.import_module( + "_docs_checker.treatment_policy" + ) + except ImportError: + self.fail( + "the engine has no treatment-policy unit yet; PR 3 must own " + "link-proposal eligibility deterministically" + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + manifest = build_manifest(root) + + purposeful = treatment_policy.evaluate_link_proposal( + manifest, + source="docs/README.md", + target="docs/archive/old/2019-review.md", + purpose="Readers auditing the 2019 decision need the record.", + wording="- [2019 review](archive/old/2019-review.md)", + ) + self.assertIs(purposeful.get("allowed"), True) + self.assertIs(purposeful.get("readiness_credit"), False) + self.assertEqual(purposeful.get("purpose_source"), "human") + + score_only = treatment_policy.evaluate_link_proposal( + manifest, + source="docs/README.md", + target="docs/lonely.md", + purpose="", + wording="- [Lonely](lonely.md)", + ) + self.assertIs(score_only.get("allowed"), False) + self.assertTrue(str(score_only.get("reason", "")).strip()) + + def test_an_existing_broken_confined_link_repair_stays_eligible(self): + """The engine authors the purpose fact for a repair to an existing + broken confined link, so no human purpose is required.""" + from _docs_checker.treatment_policy import evaluate_link_proposal + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + proposal = evaluate_link_proposal( + build_manifest(root), + source="docs/guide.md", + target="docs/STATE.md", + purpose="", + wording="[Current state](STATE.md)", + repair_existing=True, + ) + + self.assertIs(proposal["allowed"], True) + self.assertTrue(proposal["purpose"].strip()) + self.assertEqual(proposal["purpose_source"], "engine") + self.assertEqual( + proposal["reason"], + "existing-broken-confined-link-repair", + ) + + def test_unresolved_target_refusal_carries_its_classification_card(self): + """An unresolved target is not a bare refusal: one attached card + supplies the path, reason, choices, handle, and copy-paste reply needed + to resolve and unblock it.""" + from _docs_checker.treatment_policy import evaluate_link_proposal + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + + proposal = evaluate_link_proposal( + build_manifest(root), + source="docs/README.md", + target=".github/SUPPORT.md", + purpose="Readers need the support route.", + wording="- [Support](../.github/SUPPORT.md)", + ) + + self.assertIs(proposal["allowed"], False) + self.assertEqual(proposal["reason"], "target-role-unresolved") + card = proposal["classification_card"] + self.assertEqual(card["path"], ".github/SUPPORT.md") + self.assertTrue(card["reason"].strip()) + self.assertEqual(set(card["choices"]), set(RESOLUTION_CHOICES)) + self.assertTrue(card["handle"].strip()) + self.assertIn(card["handle"], card["reply"]) + + +class ExpectedFailureBookkeepingTests(unittest.TestCase): + def test_every_expected_failure_names_its_contract_and_owning_phase(self): + module = sys.modules[__name__] + marked = {} + for value in vars(module).values(): + if not isinstance(value, type) or not issubclass(value, unittest.TestCase): + continue + for name, member in vars(value).items(): + if getattr(member, "__unittest_expecting_failure__", False): + marked[name] = member + + self.assertEqual(sorted(marked), sorted(EXPECTED_FAILURE_OWNERS)) + for name, member in sorted(marked.items()): + with self.subTest(test=name): + docstring = member.__doc__ or "" + self.assertIn("Owned by ", docstring) + self.assertIn(EXPECTED_FAILURE_OWNERS[name], docstring) + self.assertTrue( + any( + source in docstring + for source in ("PRODUCT.md", "docs/STATE.md") + ), + "an expected failure must cite the contract it does not keep", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_adapters.py b/tools/build_adapters.py index f00cfdd..77668e9 100644 --- a/tools/build_adapters.py +++ b/tools/build_adapters.py @@ -44,7 +44,13 @@ "scripts/_docs_checker/discovery.py", "scripts/_docs_checker/scan.py", "scripts/_docs_checker/measurement.py", + "scripts/_docs_checker/documentation_policy.py", + "scripts/_docs_checker/document_classification.py", "scripts/_docs_checker/manifest.py", + "scripts/_docs_checker/readiness.py", + "scripts/_docs_checker/documentation_result.py", + "scripts/_docs_checker/resolution.py", + "scripts/_docs_checker/treatment_policy.py", "scripts/_docs_checker/selection.py", "scripts/_docs_checker/runtime.py", "scripts/_docs_checker/identity.py", From b43c26999edb908e1ed23ebd22e35fa9e415d183 Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 16:39:51 -0400 Subject: [PATCH 02/19] Stamp findings, expose resolution commands, and bind directory approvals --- BENCHMARK.md | 6 +- COMMANDS.md | 2 + PRODUCT.md | 2 + .../_docs_checker/documentation_result.py | 61 ++++++- .../docs/scripts/_docs_checker/readiness.py | 9 + .../docs/scripts/_docs_checker/resolution.py | 56 +++++++ adapters/claude/skills/docs/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../scripts/_docs_checker/readiness.py | 9 + .../scripts/_docs_checker/resolution.py | 56 +++++++ adapters/copilot/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../cursor/scripts/_docs_checker/readiness.py | 9 + .../scripts/_docs_checker/resolution.py | 56 +++++++ adapters/cursor/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../grok/scripts/_docs_checker/readiness.py | 9 + .../grok/scripts/_docs_checker/resolution.py | 56 +++++++ adapters/grok/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../docs/scripts/_docs_checker/readiness.py | 9 + .../docs/scripts/_docs_checker/resolution.py | 56 +++++++ adapters/plugin/skills/docs/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../docs/scripts/_docs_checker/readiness.py | 9 + .../docs/scripts/_docs_checker/resolution.py | 56 +++++++ .../skills/docs/scripts/check.py | 77 ++++++++- .../_docs_checker/documentation_result.py | 61 ++++++- .../docs/scripts/_docs_checker/readiness.py | 9 + .../docs/scripts/_docs_checker/resolution.py | 56 +++++++ skills/docs/scripts/check.py | 77 ++++++++- tests/test_docs_skill.py | 92 ++++++++--- tests/test_documentation_result_contract.py | 86 ++++++++++ tests/test_mdx_compatibility.py | 18 ++ tests/test_resolution_treatment_contract.py | 156 ++++++++++++++++++ tests/test_selection_policy.py | 16 +- 36 files changed, 1715 insertions(+), 84 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 867ad5b..446a1c6 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,12 +6,12 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,142 tests — 511 core, 452 lifecycle, and 179 trajectory. The complete Stage 4B local matrix passed; the 54 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted CI remains unclaimed before a later authorized push. | +| Windows deterministic groups | Current partition: 1,146 tests — 515 core, 452 lifecycle, and 179 trajectory. The complete Stage 4C local matrix passed; the 58 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. No hosted run is claimed for this uncommitted Stage 4C work. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | -| Repository documentation check | Exit 0 with repository-complete accounting: 93 documentation paths — 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stayed unread. Readiness v3 is 2 of 2 eligible documents; that 100% eligible fraction sits beside 59 nonignored unresolved documents and is not whole-repository health. Transitional rubric v2 remains separately visible at 89% with 27 findings until the beginner-interface phase removes it. | -| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,142-test snapshot has not run on Linux yet; fresh CI remains required before release. | +| Repository documentation check | Exit 0 with repository-complete accounting: 93 documentation paths — 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stayed unread. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 93, `nonignored_unresolved` is 59, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. All 27 published findings retain their prior count and now carry target role and write-eligibility facts. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | +| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,146-test snapshot has not run on Linux yet; fresh CI remains required before release. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | | Installed-package treatment canary | Pass: exact stateless preview → one-file apply → candidate and installed verification clean; zero control writes, no `.diataxis/`, unchanged real Git index, and only the intended disposable worktree file changed | diff --git a/COMMANDS.md b/COMMANDS.md index 99acfe5..74fba28 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -31,6 +31,8 @@ Doctor reports every finding in the evidenced scope, groups them into treatments `check` now returns readiness v3 as an unweighted ready/eligible fraction with every per-document failed fact. Trust coverage (are declared current-truth routes verified?), hash freshness, and byte telemetry stay separate. The envelope temporarily also carries rubric-v2 structural health for compatibility; the two percentages are different contracts and are never compared. A route counts as declared current truth when your documentation map marks it: append `` (or ``) to the end of the map line that links your current-state document. The engine reads that marker; it never infers a current-truth route from prose, and a map that declares none reports Trust as `unverified` rather than guessing. +The canonical checker exposes role-resolution writes through two machine-output flags. `--resolve-preview` reads one JSON decisions mapping from standard input and returns the exact zero-write map-policy diff plus its `DOCS-R-` approval. `--resolve-apply APPROVAL` reads the same mapping in a later process, recomputes the binding from the current snapshot, and applies only the matching current code; successful previews and applies exit 0, while refusals exit 2 with zero writes. + ## Boundaries These never move: diff --git a/PRODUCT.md b/PRODUCT.md index b977712..c9e1f33 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -126,6 +126,8 @@ Conventional names and locations are classification *evidence*, never an existen - A **filename alone is never sufficient.** `docs/2024-migration-notes.md` is a hint that a document is historical, not proof. - A **directory the project's own generator declares as its output is sufficient** for `generated`. The declaration is the evidence; the path is only where to look. A directory that merely *looks* like build output is not. +A link from the map proves a navigation path for readiness; it does not prove the linked document's purpose, so the link alone never assigns a role. + Letting an unrecognized name remove a document from the inventory is forbidden outright. A document whose evidence does not settle its role takes the role `unresolved` — never a guess, never silently dropped. Write eligibility is deterministic from the document's role, repository status, path confinement, and the user-owned `never-edit` policy. Conventional filenames may produce a warning but never a refusal. The selected map and README entry points remain eligible when otherwise maintained unless the human adds them to `never-edit`. diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/claude/skills/docs/scripts/check.py b/adapters/claude/skills/docs/scripts/check.py index 941d352..f4ef289 100644 --- a/adapters/claude/skills/docs/scripts/check.py +++ b/adapters/claude/skills/docs/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/adapters/copilot/scripts/_docs_checker/documentation_result.py b/adapters/copilot/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_result.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/adapters/copilot/scripts/_docs_checker/readiness.py b/adapters/copilot/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/adapters/copilot/scripts/_docs_checker/readiness.py +++ b/adapters/copilot/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/copilot/scripts/check.py b/adapters/copilot/scripts/check.py index 941d352..f4ef289 100644 --- a/adapters/copilot/scripts/check.py +++ b/adapters/copilot/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/adapters/cursor/scripts/_docs_checker/documentation_result.py b/adapters/cursor/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_result.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/adapters/cursor/scripts/_docs_checker/readiness.py b/adapters/cursor/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/adapters/cursor/scripts/_docs_checker/readiness.py +++ b/adapters/cursor/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/cursor/scripts/check.py b/adapters/cursor/scripts/check.py index 941d352..f4ef289 100644 --- a/adapters/cursor/scripts/check.py +++ b/adapters/cursor/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/adapters/grok/scripts/_docs_checker/documentation_result.py b/adapters/grok/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_result.py +++ b/adapters/grok/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/adapters/grok/scripts/_docs_checker/readiness.py b/adapters/grok/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/adapters/grok/scripts/_docs_checker/readiness.py +++ b/adapters/grok/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/grok/scripts/check.py b/adapters/grok/scripts/check.py index 941d352..f4ef289 100644 --- a/adapters/grok/scripts/check.py +++ b/adapters/grok/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/plugin/skills/docs/scripts/check.py b/adapters/plugin/skills/docs/scripts/check.py index 941d352..f4ef289 100644 --- a/adapters/plugin/skills/docs/scripts/check.py +++ b/adapters/plugin/skills/docs/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/plugins/diataxis-docs/skills/docs/scripts/check.py b/plugins/diataxis-docs/skills/docs/scripts/check.py index 941d352..f4ef289 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/check.py +++ b/plugins/diataxis-docs/skills/docs/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/skills/docs/scripts/_docs_checker/documentation_result.py b/skills/docs/scripts/_docs_checker/documentation_result.py index f1ea3a9..480951c 100644 --- a/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/skills/docs/scripts/_docs_checker/documentation_result.py @@ -3,7 +3,7 @@ from __future__ import annotations from .document_classification import COLD_ROLES -from .documentation_policy import documentation_policy_payload +from .documentation_policy import documentation_policy_payload, route_key from .manifest import manifest_payload from .readiness import calculate_readiness from .resolution import unresolved_cards @@ -72,7 +72,7 @@ def _published_findings(findings, manifest): for document in manifest.documents if document.role in COLD_ROLES } - return [ + published = [ finding for finding in findings if not ( @@ -80,6 +80,63 @@ def _published_findings(findings, manifest): and finding.get("path") in cold ) ] + documents = { + route_key(document.path): document + for document in manifest.documents + } + stamped = [] + for finding in published: + raw_routes = ( + [finding["path"]] + if isinstance(finding.get("path"), str) + else list(finding["paths"]) + if isinstance(finding.get("paths"), (list, tuple)) + else [finding["map"]] + if isinstance(finding.get("map"), str) + else [] + ) + routes = [] + seen = set() + for route in raw_routes: + if not isinstance(route, str): + continue + identity = route_key(route) + if identity in seen: + continue + seen.add(identity) + routes.append(route) + targets = [] + for route in routes: + document = documents.get(route_key(route)) + if document is None: + targets.append( + { + "path": route, + "role": None, + "write_eligible": False, + } + ) + continue + targets.append( + { + "path": document.path, + "role": document.role, + "write_eligible": document_write_facts(document)[ + "write_eligible" + ], + } + ) + roles = {target["role"] for target in targets} + stamped.append( + { + **finding, + "role": next(iter(roles)) if len(roles) == 1 else None, + "write_eligible": bool(targets) + and all(target["write_eligible"] for target in targets), + "target_documents": targets, + } + ) + return stamped def assemble_documentation_result(manifest, measurements, findings): diff --git a/skills/docs/scripts/_docs_checker/readiness.py b/skills/docs/scripts/_docs_checker/readiness.py index 026df3e..8efa11a 100644 --- a/skills/docs/scripts/_docs_checker/readiness.py +++ b/skills/docs/scripts/_docs_checker/readiness.py @@ -197,6 +197,12 @@ def calculate_readiness(manifest, measurements, *, complete): eligible_count = len(documents) ready_count = sum(row["ready"] for row in documents) + accounted_documents = manifest.coverage.accounted_documents + nonignored_unresolved = sum( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + for document in manifest.documents + ) valid_map = measurements.get("map_exists") is True percentage = ( round(100 * ready_count / eligible_count) @@ -208,6 +214,9 @@ def calculate_readiness(manifest, measurements, *, complete): "ready_count": ready_count, "eligible_count": eligible_count, "percentage": percentage, + "accounted_documents": accounted_documents, + "nonignored_unresolved": nonignored_unresolved, + "denominator_representative": nonignored_unresolved == 0, "measurement_complete": bool(complete), "selected_map_valid": valid_map, "documents": documents, diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 2c51236..4926564 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -31,6 +31,9 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) _GROUP_MINIMUM = 3 +_RESOLUTION_INPUT_ERROR = ( + "resolution decisions must be one JSON object on standard input" +) def _canonical_bytes(value): @@ -43,6 +46,30 @@ def _canonical_bytes(value): ).encode("utf-8") +def parse_resolution_decisions(value): + """Parse one fail-closed JSON decisions mapping from the CLI boundary.""" + def strict_object(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError(_RESOLUTION_INPUT_ERROR) + result[key] = item + return result + + if not isinstance(value, str): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + decisions = json.loads( + value, + object_pairs_hook=strict_object, + ) + except (UnicodeError, ValueError) as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if not isinstance(decisions, Mapping): + raise ValueError(_RESOLUTION_INPUT_ERROR) + return decisions + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -204,6 +231,30 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _directory_route_bindings(manifest, decisions): + """Bind each directory decision to the snapshot routes it can classify.""" + bindings = [] + for decision in decisions: + if not decision["directory"]: + continue + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) + bindings.append( + { + "selector": decision["selector"] + "/", + "routes": routes, + } + ) + return bindings + + def _proposed_rules(policy, decisions): role_rules = { _rule_key(rule.selector, rule.directory): rule @@ -320,6 +371,10 @@ def _resolution_plan(root, decisions): "source": source_text, "proposed": proposed_text, "decisions": normalized, + "directory_routes": _directory_route_bindings( + manifest, + normalized, + ), } digest = hashlib.sha256(_canonical_bytes(binding)).hexdigest() approval = f"DOCS-R-{digest[:12].upper()}" if map_diff else None @@ -420,6 +475,7 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", "RESOLUTION_CHOICES", + "parse_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/skills/docs/scripts/check.py b/skills/docs/scripts/check.py index 941d352..f4ef289 100644 --- a/skills/docs/scripts/check.py +++ b/skills/docs/scripts/check.py @@ -13,6 +13,9 @@ { "--agent requires --json", "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive", + "--resolve-preview and --resolve-apply are mutually exclusive", + "resolution cannot be combined with Init discovery, Doctor baseline, or Doctor recovery", + "resolution decisions must be one JSON object on standard input", "Doctor recovery cannot be combined with --init-discovery", "--init-discovery requires --json --agent", "--doctor-baseline requires --json --agent", @@ -148,6 +151,11 @@ validate_operational_state, ) from _docs_checker.measurement import measure_documentation +from _docs_checker.resolution import ( + parse_resolution_decisions, + resolution_apply, + resolution_preview, +) from _docs_checker.lifecycle import ( build_verified_event, prepare_dispositions, @@ -228,6 +236,8 @@ _PARSER.add_argument("--doctor-baseline", action="store_true") _PARSER.add_argument("--doctor-recovery-preview", action="store_true") _PARSER.add_argument("--doctor-recovery-apply", default=None, metavar="APPROVAL") +_PARSER.add_argument("--resolve-preview", action="store_true") +_PARSER.add_argument("--resolve-apply", default=None, metavar="APPROVAL") _PARSER.add_argument("--continuation", default=None) _PARSER.add_argument("--map", default=None) _PARSER.add_argument("--hot", default=None) @@ -287,6 +297,7 @@ def main(argv=None): "--scope", "--continuation", "--doctor-recovery-apply", + "--resolve-apply", } positional = [] skip = False @@ -306,6 +317,8 @@ def main(argv=None): "--json", "--doctor-recovery-preview", "--doctor-recovery-apply", + "--resolve-preview", + "--resolve-apply", "--doctor-baseline", ) ) @@ -342,6 +355,10 @@ def main(argv=None): namespace.doctor_recovery_preview or namespace.doctor_recovery_apply is not None ) + resolution_mode = ( + namespace.resolve_preview + or namespace.resolve_apply is not None + ) baseline_mode = namespace.doctor_baseline try: if namespace.agent and not namespace.json: @@ -353,6 +370,19 @@ def main(argv=None): raise ValueError( "--doctor-recovery-preview and --doctor-recovery-apply are mutually exclusive" ) + if namespace.resolve_preview and namespace.resolve_apply is not None: + raise ValueError( + "--resolve-preview and --resolve-apply are mutually exclusive" + ) + if resolution_mode and ( + recovery_mode + or namespace.init_discovery + or baseline_mode + ): + raise ValueError( + "resolution cannot be combined with Init discovery, " + "Doctor baseline, or Doctor recovery" + ) if recovery_mode and namespace.init_discovery: raise ValueError("Doctor recovery cannot be combined with --init-discovery") if baseline_mode and (namespace.init_discovery or recovery_mode): @@ -389,7 +419,27 @@ def main(argv=None): # validate_root; the façade must not preflight ancestors through a # platform-specific Path operation before that boundary. raw = Path(os.path.abspath(os.path.expanduser(os.fspath(namespace.root)))) - if recovery_mode: + if resolution_mode: + _assert_no_reparse_components(raw) + if _is_reparse(raw) or not os.path.isdir(raw): + raise ValueError("root must be a real directory") + root = safe_path(raw, raw) + decisions = parse_resolution_decisions(sys.stdin.read()) + try: + resolution_response = ( + resolution_preview(root, decisions) + if namespace.resolve_preview + else resolution_apply( + root, + decisions, + namespace.resolve_apply, + ) + ) + except TypeError as exc: + raise ValueError( + "resolution decisions must be one JSON object on standard input" + ) from exc + elif recovery_mode: recovery_preview = preview_state_conflict_recovery(raw) recovery_response = ( recovery_preview @@ -468,7 +518,7 @@ def main(argv=None): ) except InvalidDocumentationPolicy as exc: policy_response = invalid_policy_envelope(exc, doctor=baseline_mode) - if namespace.json: + if namespace.json or resolution_mode: print( json.dumps( policy_response, @@ -482,7 +532,7 @@ def main(argv=None): print("error: invalid-policy") return 2 except NavigationBoundary as exc: - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -512,7 +562,7 @@ def main(argv=None): # A declared but unreadable ``.git`` is an ordinary environment # failure, not a defect to show as a traceback. It carries no errno, # so the generic environmental policy below cannot recognize it. - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -529,7 +579,7 @@ def main(argv=None): except OSError as exc: if not is_expected_environmental_error(exc): raise - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -558,7 +608,7 @@ def main(argv=None): if raw_detail in _SAFE_PUBLIC_CLI_ERRORS else "invalid command input" ) - if namespace.json or recovery_mode: + if namespace.json or recovery_mode or resolution_mode: print( json.dumps( { @@ -572,6 +622,21 @@ def main(argv=None): else: print(f"error: {detail}") return 2 + if resolution_mode: + print( + json.dumps( + resolution_response, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + ) + return 0 if resolution_response.get("status") in { + "approval-required", + "no-change", + "applied", + } else 2 if recovery_mode: print( json.dumps( diff --git a/tests/test_docs_skill.py b/tests/test_docs_skill.py index 766eeb7..9aba638 100644 --- a/tests/test_docs_skill.py +++ b/tests/test_docs_skill.py @@ -19,6 +19,31 @@ import build_adapters +def stamped_finding( + kind, + path, + *, + role=None, + write_eligible=False, + **details, +): + """Return the exact additive manifest treatment stamp for one target.""" + return { + "kind": kind, + "path": path, + **details, + "role": role, + "write_eligible": write_eligible, + "target_documents": [ + { + "path": path, + "role": role, + "write_eligible": write_eligible, + } + ], + } + + class DocsSkillContractTests(unittest.TestCase): @staticmethod def _init_text(): @@ -2265,7 +2290,12 @@ def test_root_scope_prunes_standard_non_documentation_trees_and_reports_jurisdic self.assertNotIn("vendor/linked.md", json.dumps(payload["findings"])) self.assertNotIn("vendor\\\\linked.md", json.dumps(payload["findings"])) self.assertIn( - {"kind": "missing-link", "path": "docs/STATE.md", "target": "missing.md"}, + stamped_finding( + "missing-link", + "docs/STATE.md", + role="unresolved", + target="missing.md", + ), payload["findings"], ) @@ -2296,7 +2326,7 @@ def test_checker_reports_in_scope_reparse_path_consistently(self): first = self._agent_payload(root, scope="docs") second = self._agent_payload(root, scope="docs") - expected = {"kind": "symlink", "path": "docs/linked"} + expected = stamped_finding("symlink", "docs/linked") self.assertIn(expected, first["findings"]) self.assertEqual(first["findings"], second["findings"]) @@ -2349,7 +2379,7 @@ def test_selected_reparse_dedup_uses_filesystem_case_semantics(self): ] self.assertEqual( symlink_findings, - [{"kind": "symlink", "path": "docs/linked"}], + [stamped_finding("symlink", "docs/linked")], ) def test_nested_documentation_named_build_is_not_globally_pruned(self): @@ -2367,11 +2397,12 @@ def test_nested_documentation_named_build_is_not_globally_pruned(self): payload = self._agent_payload(root, scope="docs") self.assertIn( - { - "kind": "missing-link", - "path": "docs/build/guide.md", - "target": "missing.md", - }, + stamped_finding( + "missing-link", + "docs/build/guide.md", + role="unresolved", + target="missing.md", + ), payload["findings"], ) @@ -2452,19 +2483,21 @@ def test_prune_json_distinguishes_policy_from_applied_paths(self): self.assertNotIn("docs/build", prunes["applied_paths"]) self.assertNotIn("docs/vendor", prunes["applied_paths"]) self.assertIn( - { - "kind": "missing-link", - "path": "docs/build/guide.md", - "target": "missing-build.md", - }, + stamped_finding( + "missing-link", + "docs/build/guide.md", + role="unresolved", + target="missing-build.md", + ), payload["findings"], ) self.assertIn( - { - "kind": "missing-link", - "path": "docs/vendor/reference.md", - "target": "missing-vendor.md", - }, + stamped_finding( + "missing-link", + "docs/vendor/reference.md", + role="unresolved", + target="missing-vendor.md", + ), payload["findings"], ) self.assertNotIn("missing-cache.md", json.dumps(payload)) @@ -2521,7 +2554,7 @@ def test_selected_reparse_map_is_a_finding_not_an_execution_error(self): self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) payload = json.loads(proc.stdout) self.assertIn( - {"kind": "symlink", "path": "docs/README.md"}, + stamped_finding("symlink", "docs/README.md"), payload["findings"], ) self.assertNotIn("outside-missing.md", proc.stdout) @@ -2561,7 +2594,7 @@ def test_selected_reparse_hot_path_is_a_finding_not_an_execution_error(self): self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) payload = json.loads(proc.stdout) self.assertIn( - {"kind": "symlink", "path": "docs/STATE.md"}, + stamped_finding("symlink", "docs/STATE.md"), payload["findings"], ) self.assertNotIn("outside-missing.md", proc.stdout) @@ -2606,7 +2639,7 @@ def test_selected_descendant_of_in_scope_reparse_preserves_ancestor_finding(self self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) payload = json.loads(proc.stdout) - expected = {"kind": "symlink", "path": "docs/linked"} + expected = stamped_finding("symlink", "docs/linked") self.assertEqual(payload["findings"].count(expected), 1) self.assertNotIn("outside-missing.md", proc.stdout) @@ -2638,7 +2671,11 @@ def test_map_outside_scope_is_read_directly_without_walking_its_parent(self): self.assertEqual(payload["map"], "vendor/README.md") self.assertIn( - {"kind": "missing-link", "path": "vendor/README.md", "target": "missing.md"}, + stamped_finding( + "missing-link", + "vendor/README.md", + target="missing.md", + ), payload["findings"], ) self.assertNotIn("vendor/internal.md", json.dumps(payload)) @@ -2658,7 +2695,11 @@ def test_hot_path_outside_scope_is_read_directly_without_walking_its_parent(self payload = self._agent_payload(root, scope=".", hot="build/STATE.md") self.assertIn( - {"kind": "missing-link", "path": "build/STATE.md", "target": "missing.md"}, + stamped_finding( + "missing-link", + "build/STATE.md", + target="missing.md", + ), payload["findings"], ) self.assertNotIn("build/internal.md", json.dumps(payload)) @@ -2914,7 +2955,10 @@ def test_internal_junction_is_reported_but_not_read(self): self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) payload = json.loads(proc.stdout) self.assertNotIn(sentinel, proc.stdout) - self.assertIn({"kind": "symlink", "path": "docs/linked"}, payload["findings"]) + self.assertIn( + stamped_finding("symlink", "docs/linked"), + payload["findings"], + ) def test_cross_scope_anchor_and_root_scope(self): with tempfile.TemporaryDirectory() as td: diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 1b90461..3e38668 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -882,6 +882,47 @@ def test_readiness_is_an_unweighted_fraction_with_visible_counts(self): "machinery", ) + def test_readiness_reports_whether_its_denominator_represents_all_accounted_docs(self): + """Stage 4C correction 3: readiness publishes the accounted-document + count, the exact nonignored unresolved count, and the engine-owned + representative judgment. The real repository remains visibly + nonrepresentative with 59 unresolved documents, while a fully + classified fixture is representative.""" + repository_readiness = section(run_checker(ROOT), "readiness") + + self.assertEqual(repository_readiness.get("accounted_documents"), 93) + self.assertEqual(repository_readiness.get("nonignored_unresolved"), 59) + self.assertIs( + repository_readiness.get("denominator_representative"), + False, + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write(root, "docs/guide.md", "# Guide\n\nComplete guidance.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "fixture") + + classified = section(run_checker(root), "readiness") + + self.assertEqual(classified.get("accounted_documents"), 2) + self.assertEqual(classified.get("nonignored_unresolved"), 0) + self.assertIs( + classified.get("denominator_representative"), + True, + ) + def test_every_eligible_document_exposes_all_four_facts(self): """Every eligible document row exposes its reachability, internal-link/anchor, title, and confinement facts individually, so @@ -1281,6 +1322,51 @@ class ColdMaterialContracts(unittest.TestCase): """Charter AC 8 and 12, read-result half: classified cold material leaves readiness with a named reason instead of surfacing as repair findings.""" + def test_published_findings_are_stamped_with_target_treatment_facts(self): + """Stage 4C correction 1: every published finding carries manifest + target facts, and a single-target unreachable finding against an + unresolved document says plainly that the target is not write + eligible. Stamping is additive: the repository's 27 findings remain + 27 and keep their existing numbering and content.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + + unresolved = [ + finding + for finding in run_checker(root)["findings"] + if finding.get("kind") == "unreachable" + ] + + self.assertTrue(unresolved) + for finding in unresolved: + with self.subTest(path=finding.get("path")): + self.assertEqual(finding.get("role"), "unresolved") + self.assertIs(finding.get("write_eligible"), False) + self.assertEqual( + finding.get("target_documents"), + [ + { + "path": finding["path"], + "role": "unresolved", + "write_eligible": False, + } + ], + ) + + repository_findings = run_checker(ROOT)["findings"] + self.assertEqual(len(repository_findings), 27) + for finding in repository_findings: + with self.subTest( + kind=finding.get("kind"), + path=finding.get("path"), + paths=finding.get("paths"), + ): + self.assertIn("role", finding) + self.assertIn("write_eligible", finding) + self.assertTrue(finding.get("target_documents")) + def test_classified_cold_documents_are_excluded_with_reasons_not_findings(self): """Policy-classified preserved, generated, and historical documents stop surfacing as ``unreachable`` repair findings; each leaves the diff --git a/tests/test_mdx_compatibility.py b/tests/test_mdx_compatibility.py index 36a76f0..3a9731b 100644 --- a/tests/test_mdx_compatibility.py +++ b/tests/test_mdx_compatibility.py @@ -693,6 +693,15 @@ def test_mintlify_links_and_exact_redirects_resolve_provider_routes(self): "kind": "missing-anchor", "path": "docs/getting-started/installing-cline.mdx", "target": "../cline-overview#missing-anchor", + "role": "maintained", + "write_eligible": True, + "target_documents": [ + { + "path": "docs/getting-started/installing-cline.mdx", + "role": "maintained", + "write_eligible": True, + } + ], } ], ) @@ -1016,6 +1025,15 @@ def test_missing_navigation_page_is_a_deterministic_provider_finding(self): "path": "docs/missing", "route": "missing", "context": ["Docs", "Start"], + "role": None, + "write_eligible": False, + "target_documents": [ + { + "path": "docs/missing", + "role": None, + "write_eligible": False, + } + ], } ], ) diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index bd7f9ab..ffcafae 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -27,6 +27,7 @@ ROOT = Path(__file__).parents[1] SCRIPTS = ROOT / "skills" / "docs" / "scripts" +CHECKER = SCRIPTS / "check.py" sys.path.insert(0, str(SCRIPTS)) from tests.test_documentation_result_contract import ( # noqa: E402 @@ -83,6 +84,25 @@ def run_resolution_process(root, action, decisions, approval=None): ) +def run_resolution_cli(root, action, decisions, approval=None): + arguments = [ + sys.executable, + "-B", + str(CHECKER), + str(root), + "--resolve-preview" if action == "preview" else "--resolve-apply", + ] + if action == "apply": + arguments.append("" if approval is None else approval) + return subprocess.run( + arguments, + input=json.dumps(decisions, sort_keys=True), + capture_output=True, + text=True, + cwd=str(ROOT), + ) + + class UnresolvedResolutionContracts(unittest.TestCase): """Decision 13 and charter AC 10: unresolved asks the human plainly and persists nothing before one approved write.""" @@ -195,6 +215,41 @@ def test_directory_groups_resolve_in_one_reply_with_exact_exceptions(self): self.assertIn("role preserved research/", map_diff) self.assertIn("role historical research/note-02.md", map_diff) + def test_directory_resolution_approval_binds_the_snapshot_route_list(self): + """Stage 4C correction 4: a directory decision's DOCS-R approval + includes the exact manifest routes it resolves, so adding a document + to that directory invalidates the old code with zero writes.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + for index in range(1, 4): + write( + root, + f"research/note-{index:02d}.md", + f"# Note {index:02d}\n\nRecorded result.\n", + ) + git(root, "add", "--", "research") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "research") + decisions = {"research/": {"role": "preserved"}} + preview = resolution_preview(root, decisions) + map_path = root / "docs" / "README.md" + map_before = map_path.read_bytes() + + write( + root, + "research/note-04.md", + "# Note 04\n\nAdded after approval.\n", + ) + refused = resolution_apply(root, decisions, preview["approval"]) + + self.assertEqual(refused["status"], "refused") + self.assertEqual(refused["reason"], "approval-code-not-current") + self.assertEqual(refused["writes"], 0) + self.assertEqual(map_path.read_bytes(), map_before) + def test_resolving_a_role_previews_an_exact_map_diff_with_a_short_code(self): """PRODUCT.md 'Consent and approval': a role resolution produces an exact proposed map-policy diff and a short engine-owned approval code, @@ -362,6 +417,107 @@ def test_resolution_receipt_survives_a_process_boundary(self): "historical", ) + def test_checker_resolution_cli_is_cross_process_exact_and_zero_write_on_refusal(self): + """Stage 4C correction 2: check.py exposes the DOCS-R preview/apply + pair. JSON decisions enter on standard input; preview and apply may + run in separate processes, while stale, mismatched, and reused codes + all refuse through the CLI without writing.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + decisions = {"HISTORY-2024.md": {"role": "historical"}} + map_path = root / "docs" / "README.md" + original = map_path.read_bytes() + + preview_process = run_resolution_cli(root, "preview", decisions) + self.assertEqual( + preview_process.returncode, + 0, + preview_process.stderr, + ) + preview = json.loads(preview_process.stdout) + self.assertEqual(preview["status"], "approval-required") + self.assertEqual(preview["writes"], 0) + self.assertEqual(map_path.read_bytes(), original) + + mismatched = run_resolution_cli( + root, + "apply", + {"HISTORY-2024.md": {"role": "preserved"}}, + preview["approval"], + ) + self.assertEqual(mismatched.returncode, 2, mismatched.stderr) + mismatch_result = json.loads(mismatched.stdout) + self.assertEqual(mismatch_result["status"], "refused") + self.assertEqual(mismatch_result["writes"], 0) + self.assertEqual(map_path.read_bytes(), original) + + apply_process = run_resolution_cli( + root, + "apply", + decisions, + preview["approval"], + ) + self.assertEqual( + apply_process.returncode, + 0, + apply_process.stderr, + ) + applied = json.loads(apply_process.stdout) + self.assertEqual(applied["status"], "applied") + self.assertEqual(applied["writes"], 1) + self.assertIn( + "role historical HISTORY-2024.md", + map_path.read_text(encoding="utf-8"), + ) + installed = map_path.read_bytes() + + reused = run_resolution_cli( + root, + "apply", + decisions, + preview["approval"], + ) + self.assertEqual(reused.returncode, 2, reused.stderr) + reused_result = json.loads(reused.stdout) + self.assertEqual(reused_result["status"], "refused") + self.assertEqual(reused_result["writes"], 0) + self.assertEqual(map_path.read_bytes(), installed) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = json.loads( + run_resolution_cli(root, "preview", decisions).stdout + ) + map_path = root / "docs" / "README.md" + write( + root, + "docs/README.md", + map_path.read_text(encoding="utf-8") + "\nOwner edit.\n", + ) + drifted = map_path.read_bytes() + + stale = run_resolution_cli( + root, + "apply", + decisions, + preview["approval"], + ) + + self.assertEqual(stale.returncode, 2, stale.stderr) + stale_result = json.loads(stale.stdout) + self.assertEqual(stale_result["status"], "refused") + self.assertIn( + stale_result["reason"], + {"approval-code-not-current", "selected-map-drift"}, + ) + self.assertEqual(stale_result["writes"], 0) + self.assertEqual(map_path.read_bytes(), drifted) + def test_resolution_apply_refuses_drift_with_zero_writes(self): """Stage 3B resolution apply compares the selected map's current bytes to its preview binding and names drift without writing.""" diff --git a/tests/test_selection_policy.py b/tests/test_selection_policy.py index 627d526..57a5aeb 100644 --- a/tests/test_selection_policy.py +++ b/tests/test_selection_policy.py @@ -919,7 +919,21 @@ def test_an_empty_repository_reports_the_missing_map_finding(self): self.assertTrue(payload["has_findings"]) self.assertEqual( payload["findings"], - [{"kind": "missing-map", "map": "docs/README.md"}], + [ + { + "kind": "missing-map", + "map": "docs/README.md", + "role": None, + "write_eligible": False, + "target_documents": [ + { + "path": "docs/README.md", + "role": None, + "write_eligible": False, + } + ], + } + ], ) def test_untracked_only_documentation_is_visible_to_map_and_doctor(self): From c8db3e92770fda17fa9f9f6d64f47a37893001e2 Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 17:23:48 -0400 Subject: [PATCH 03/19] Make the readiness contract clone-portable --- BENCHMARK.md | 2 +- tests/test_documentation_result_contract.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 446a1c6..53fc803 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -10,7 +10,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | -| Repository documentation check | Exit 0 with repository-complete accounting: 93 documentation paths — 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stayed unread. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 93, `nonignored_unresolved` is 59, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. All 27 published findings retain their prior count and now carry target role and write-eligibility facts. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | +| Repository documentation check | A fresh checkout exits 0 with repository-complete accounting for 61 documentation paths — 2 maintained and 59 unresolved. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 61, `nonignored_unresolved` is 59, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. Local working copies with ignored drafts account for those additional paths by name and count while leaving their bodies unread. All 27 published findings retain their prior count and carry target role and write-eligibility facts. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | | Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,146-test snapshot has not run on Linux yet; fresh CI remains required before release. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 3e38668..31b3ddb 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -889,8 +889,12 @@ def test_readiness_reports_whether_its_denominator_represents_all_accounted_docs nonrepresentative with 59 unresolved documents, while a fully classified fixture is representative.""" repository_readiness = section(run_checker(ROOT), "readiness") + repository_manifest = manifest_payload(build_manifest(ROOT)) - self.assertEqual(repository_readiness.get("accounted_documents"), 93) + self.assertEqual( + repository_readiness.get("accounted_documents"), + len(repository_manifest["documents"]), + ) self.assertEqual(repository_readiness.get("nonignored_unresolved"), 59) self.assertIs( repository_readiness.get("denominator_representative"), From 9ebf01c2830309edcb57deae31a64ac12f41b8cc Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 17:43:55 -0400 Subject: [PATCH 04/19] Record hosted validation and WSL clone guidance --- BENCHMARK.md | 2 +- TESTING.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 53fc803..eea5eed 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -11,7 +11,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | | Repository documentation check | A fresh checkout exits 0 with repository-complete accounting for 61 documentation paths — 2 maintained and 59 unresolved. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 61, `nonignored_unresolved` is 59, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. Local working copies with ignored drafts account for those additional paths by name and count while leaving their bodies unread. All 27 published findings retain their prior count and carry target role and write-eligibility facts. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | -| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,146-test snapshot has not run on Linux yet; fresh CI remains required before release. | +| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,146-test snapshot passed on hosted Ubuntu and Windows in PR #30's `validate` run #295. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | | Installed-package treatment canary | Pass: exact stateless preview → one-file apply → candidate and installed verification clean; zero control writes, no `.diataxis/`, unchanged real Git index, and only the intended disposable worktree file changed | diff --git a/TESTING.md b/TESTING.md index a137efc..4eb5382 100644 --- a/TESTING.md +++ b/TESTING.md @@ -74,6 +74,8 @@ Pull-request CI runs the complete suite once. Windows lifecycle tests are split Run the Ubuntu proof from a Linux-native checkout under `$HOME`, not directly from `/mnt/c`, `/mnt/d`, or another Windows-mounted path. The Windows bridge is correct but makes metadata-heavy lifecycle tests much slower; the backing WSL virtual disk can still live on the chosen Windows drive. Keep the Windows checkout authoritative, copy or clone it into a clearly named disposable Linux directory for verification, and remove only that verified copy afterward. +On `/mnt/d`, that metadata cost is high enough to trip the engine's five-second Git inventory timeout, so Linux test runs on a Windows machine should use a clone inside the Linux filesystem. + ## Proof order 1. Add or run the smallest regression that proves the changed behavior. From 176e5c97c07c260f8db4e4a72a272bcee5a24ba5 Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 19:16:52 -0400 Subject: [PATCH 05/19] Bind policy provenance, generator evidence, and exact opt-outs --- BENCHMARK.md | 6 +- PRODUCT.md | 2 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../docs/scripts/_docs_checker/manifest.py | 9 +- .../docs/scripts/_docs_checker/resolution.py | 33 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 1 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../copilot/scripts/_docs_checker/manifest.py | 9 +- .../scripts/_docs_checker/resolution.py | 33 ++++- .../copilot/scripts/_docs_checker/scan.py | 1 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../cursor/scripts/_docs_checker/manifest.py | 9 +- .../scripts/_docs_checker/resolution.py | 33 ++++- adapters/cursor/scripts/_docs_checker/scan.py | 1 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../grok/scripts/_docs_checker/manifest.py | 9 +- .../grok/scripts/_docs_checker/resolution.py | 33 ++++- adapters/grok/scripts/_docs_checker/scan.py | 1 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../docs/scripts/_docs_checker/manifest.py | 9 +- .../docs/scripts/_docs_checker/resolution.py | 33 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 1 + docs/STATE.md | 6 +- .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ .../docs/scripts/_docs_checker/manifest.py | 9 +- .../docs/scripts/_docs_checker/resolution.py | 33 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 1 + .../_docs_checker/document_classification.py | 8 ++ .../_docs_checker/documentation_policy.py | 67 +++++++--- .../_docs_checker/generator_evidence.py | 71 +++++++++++ skills/docs/scripts/_docs_checker/manifest.py | 9 +- .../docs/scripts/_docs_checker/resolution.py | 33 ++++- skills/docs/scripts/_docs_checker/scan.py | 1 + tests/test_adapters.py | 1 + tests/test_documentation_result_contract.py | 61 ++++++++- tests/test_resolution_treatment_contract.py | 116 ++++++++++++++++++ tools/build_adapters.py | 1 + 49 files changed, 1358 insertions(+), 158 deletions(-) create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/generator_evidence.py create mode 100644 adapters/copilot/scripts/_docs_checker/generator_evidence.py create mode 100644 adapters/cursor/scripts/_docs_checker/generator_evidence.py create mode 100644 adapters/grok/scripts/_docs_checker/generator_evidence.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/generator_evidence.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/generator_evidence.py create mode 100644 skills/docs/scripts/_docs_checker/generator_evidence.py diff --git a/BENCHMARK.md b/BENCHMARK.md index eea5eed..ada79bd 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,12 +6,12 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,146 tests — 515 core, 452 lifecycle, and 179 trajectory. The complete Stage 4C local matrix passed; the 58 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. No hosted run is claimed for this uncommitted Stage 4C work. | +| Windows deterministic groups | Current partition: 1,149 tests — 518 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 61 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | -| Repository documentation check | A fresh checkout exits 0 with repository-complete accounting for 61 documentation paths — 2 maintained and 59 unresolved. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 61, `nonignored_unresolved` is 59, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. Local working copies with ignored drafts account for those additional paths by name and count while leaving their bodies unread. All 27 published findings retain their prior count and carry target role and write-eligibility facts. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | -| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The current 1,146-test snapshot passed on hosted Ubuntu and Windows in PR #30's `validate` run #295. | +| Repository documentation check | A fresh checkout exits 0 with repository-complete accounting for 61 documentation paths — 2 maintained, 20 generator-declared outputs, and 39 unresolved. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 61, `nonignored_unresolved` is 39, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. Local working copies with ignored drafts account for those additional paths by name and count while leaving their bodies unread. The 7 remaining published findings carry target role and write-eligibility facts; generated cold documents no longer surface as repair findings. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | +| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The 1,146-test snapshot passed on hosted Ubuntu and Windows in PR #30's `validate` run #295; hosted coverage for the current 1,149-test snapshot is reported by that PR's latest validate run. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | | Installed-package treatment canary | Pass: exact stateless preview → one-file apply → candidate and installed verification clean; zero control writes, no `.diataxis/`, unchanged real Git index, and only the intended disposable worktree file changed | diff --git a/PRODUCT.md b/PRODUCT.md index c9e1f33..5de566b 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -151,6 +151,8 @@ DOC-7A4C2E9F leave unresolved Role and `never-edit` are different answers to different questions, so one policy preview may set a role, a `never-edit` choice, or both. Several handles may be resolved in one reply. Large unresolved sets receive deterministic directory-group handles with exact counts and complete route lists; an exact-file decision overrides its group. Ignored documentation receives no classification card because its body was not read. Map writes nothing; a resolution persists only through a separate exact `DOCS-R-` approval. The code is recomputed from current map bytes and supplied decisions, so it survives a process boundary while changed bytes or decisions invalidate it. +When a navigation provider separates its machine-readable authority from its readable entry, the entry is the policy source: the parsed bytes, reported source route, Git durability, preview, and apply all name that same file. An exact `never_edit: false` choice is persisted as a `never-edit false ` exception and wins over a containing directory rule. + ## Readiness always shows its facts and denominator The readiness denominator is exactly: diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py b/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/generator_evidence.py b/adapters/claude/skills/docs/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/adapters/copilot/scripts/_docs_checker/document_classification.py b/adapters/copilot/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/adapters/copilot/scripts/_docs_checker/document_classification.py +++ b/adapters/copilot/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_policy.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/adapters/copilot/scripts/_docs_checker/generator_evidence.py b/adapters/copilot/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/adapters/copilot/scripts/_docs_checker/manifest.py b/adapters/copilot/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/adapters/copilot/scripts/_docs_checker/manifest.py +++ b/adapters/copilot/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/adapters/cursor/scripts/_docs_checker/document_classification.py b/adapters/cursor/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/adapters/cursor/scripts/_docs_checker/document_classification.py +++ b/adapters/cursor/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_policy.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/adapters/cursor/scripts/_docs_checker/generator_evidence.py b/adapters/cursor/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/adapters/cursor/scripts/_docs_checker/manifest.py b/adapters/cursor/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/adapters/cursor/scripts/_docs_checker/manifest.py +++ b/adapters/cursor/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/adapters/grok/scripts/_docs_checker/document_classification.py b/adapters/grok/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/adapters/grok/scripts/_docs_checker/document_classification.py +++ b/adapters/grok/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_policy.py +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/adapters/grok/scripts/_docs_checker/generator_evidence.py b/adapters/grok/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/adapters/grok/scripts/_docs_checker/manifest.py b/adapters/grok/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/adapters/grok/scripts/_docs_checker/manifest.py +++ b/adapters/grok/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py b/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/generator_evidence.py b/adapters/plugin/skills/docs/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/docs/STATE.md b/docs/STATE.md index 2af3df1..26fcf4d 100644 --- a/docs/STATE.md +++ b/docs/STATE.md @@ -11,7 +11,7 @@ Default command routes used to measure different documentation. Reproduced on th | Route | Command | Result then | Result now | | --- | --- | --- | --- | -| Checker default | `check.py . --json --agent` | boundary `docs` — 2 maintained documents, 100%, `clean`, 0 findings | boundary `.` — legacy rubric v2: 61 measured documents, 89%, `needs-attention`, 27 findings | +| Checker default | `check.py . --json --agent` | boundary `docs` — 2 maintained documents, 100%, `clean`, 0 findings | boundary `.` — legacy rubric v2: 61 measured documents, 89%, `needs-attention`, 7 findings | | Human override | `check.py . --json --agent --scope .` | boundary `.` — 89%, `needs-attention`, 27 findings | unchanged boundary, now identical to the default | | Doctor first-contact baseline | `check.py . --json --agent --doctor-baseline` | `unavailable` — two candidate roots, no boundary, no score | boundary `.` — same documentation/readiness result; provisional entry candidate with no treatment authority | @@ -37,7 +37,7 @@ The shared result now carries the ordered documentation ledger, independent role - An additive engine-owned documentation manifest accounts for every in-policy repository document exactly once — tracked, untracked, and ignored or excluded — with ignored bodies unread and named exclusions. Its identity binds normalized content for inspected bodies, so a real documentation change moves it while line-ending, Unicode-composition, or path-case spelling differences do not, and it survives a fresh clone. - One engine-owned `documentation` result publishes that manifest's ordered path ledger. Every document has an evidence-bounded role, independent user-owned `never_edit` fact, independent repository status, write eligibility, and the reason for any exclusion. The selected map and declared current-truth routes are maintained evidence; ordinary links and filename or folder hints cannot silently assign a role. Invalid or conflicting `docs:policy` input fails closed. -- The selected map's current working-tree `docs:policy` bytes are authoritative for the current snapshot. The result separately reports whether those bytes match Git and are therefore durable; a non-Git repository receives the honest no-saved-version notice. Exact-file policy rules override directory rules, and rendering preserves the existing line order while touching only edited entries. +- The selected readable entry's current working-tree `docs:policy` bytes are authoritative for the current snapshot. When a provider uses a separate machine-readable authority, policy provenance and approved writes stay bound to the entry whose bytes were parsed. The result separately reports whether those bytes match Git and are therefore durable; a non-Git repository receives the honest no-saved-version notice. Exact-file policy rules override directory rules, including explicit `never-edit` opt-outs, and rendering preserves the existing line order while touching only edited entries. - Readiness v3 is the unweighted fraction of eligible maintained documents whose reachability, internal links and anchors, title identity, and confinement all pass. It exists only for a complete measurement with a valid selected map, names every failed fact, and never consumes Trust or compares itself with stored rubric-v2 state. - Unresolved nonignored documents receive deterministic individual or directory-group cards. A resolution previews one exact map-policy diff behind a single-use stateless `DOCS-R-` approval; preview and apply can cross a process boundary, while changed map bytes, changed supplied decisions, or reuse after a successful apply refuses without writing. Init persists approved `never-edit` choices into the map, and treatment policy keeps `never-edit`, ignored, unresolved, and non-maintained reasons separate from readiness. - One Windows-visible behavior change came with it. Repository-relative path validation rejected a rooted path such as `/docs/README.md` on POSIX but accepted it on Windows, where a rooted path carries no drive letter. It is now rejected on both. A rooted `--map` or `--hot` argument therefore returns one clean input error on Windows instead of the previous wrong answer — a `missing-map` finding for a file that exists, or a silently discarded hot path. A rooted `--scope` was already refused and is unchanged. @@ -46,7 +46,7 @@ The shared result now carries the ordered documentation ledger, independent role Each item above is proven for the route that owns it. -On this worktree, the repository checker accounts for 93 documentation paths: 2 maintained and 91 unresolved, including 32 ignored or excluded paths whose bodies stay unread. Readiness v3 reports 2 of 2 eligible maintained documents ready. That 100% is only the eligible fraction beside 59 nonignored unresolved documents; it is not whole-repository health. The transitional legacy rubric remains separately visible until the presentation work removes it. +On this worktree, the repository checker accounts for 93 documentation paths: 2 maintained, 20 generator-declared outputs, and 71 unresolved, including 32 ignored or excluded paths whose bodies stay unread. Readiness v3 reports 2 of 2 eligible maintained documents ready. That 100% is only the eligible fraction beside 39 nonignored unresolved documents; it is not whole-repository health. The transitional legacy rubric remains separately visible until the presentation work removes it. The same ownership gap also affects `context`, `write`, and `update`: the current routes do not have matching deterministic engine entrypoints for evidence selection or mutation planning. Until those entrypoints exist, the model can be asked to choose work that the product constitution assigns to the engine. diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/generator_evidence.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/skills/docs/scripts/_docs_checker/document_classification.py b/skills/docs/scripts/_docs_checker/document_classification.py index 5a13534..6724d67 100644 --- a/skills/docs/scripts/_docs_checker/document_classification.py +++ b/skills/docs/scripts/_docs_checker/document_classification.py @@ -61,6 +61,7 @@ def classify_documents( map_route, map_exists, current_truth_routes, + generated_evidence=None, ): """Classify every accounted route once from explicit evidence only.""" authoritative = _authoritative_routes( @@ -79,6 +80,10 @@ def classify_documents( conflict_route=route, ) + generated_by_key = { + route_key(route): evidence + for route, evidence in (generated_evidence or {}).items() + } result = {} for route in routes: status = repository_status[route] @@ -91,6 +96,9 @@ def classify_documents( if role_rule is not None: role = role_rule.value evidence = f"working-tree-policy:{role_rule.selector}" + elif route_key(route) in generated_by_key: + role = "generated" + evidence = generated_by_key[route_key(route)] elif status == "ignored or excluded": role = "unresolved" evidence = "ignored-body-unread" diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py index 212415c..e079ae6 100644 --- a/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -18,6 +18,8 @@ POLICY_ROLES = ("maintained", "preserved", "generated", "historical") +NEVER_EDIT_VALUE = "never-edit" +NEVER_EDIT_EXCEPTION_VALUE = "edit-allowed" _POLICY_START = "[ \t]*(?=\r?$)", @@ -170,6 +172,28 @@ def _provenance(root, map_route, *, git_available=None): } +def _never_edit_parts(line): + parts = line.split(maxsplit=2) + if len(parts) == 2 and parts[0] == "never-edit": + return NEVER_EDIT_VALUE, parts[1] + if ( + len(parts) == 3 + and parts[0] == "never-edit" + and parts[1] == "false" + ): + return NEVER_EDIT_EXCEPTION_VALUE, parts[2] + raise ValueError("malformed never-edit directive") + + +def _never_edit_directive(rule): + selector = rule.selector + ("/" if rule.directory else "") + if rule.value == NEVER_EDIT_VALUE: + return f"never-edit {selector}" + if rule.value == NEVER_EDIT_EXCEPTION_VALUE: + return f"never-edit false {selector}" + raise ValueError("never-edit rule has an invalid value") + + def _selector(root, raw, map_route): directory = raw.endswith("/") if "\\" in raw or any(character in raw for character in _GLOB_CHARACTERS): @@ -257,12 +281,20 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules[key] = PolicyRule(selector, directory, role) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, raw_selector = _never_edit_parts(line) + except ValueError: _invalid("malformed-never-edit-directive", map_route) - selector, directory = _selector(root, parts[1], map_route) + selector, directory = _selector(root, raw_selector, map_route) key = (route_key(selector), directory) - never_edit_rules[key] = PolicyRule(selector, directory, "never-edit") + previous = never_edit_rules.get(key) + if previous is not None and previous.value != value: + _invalid( + "conflicting-never-edit-rules", + map_route, + conflict_route=selector, + ) + never_edit_rules[key] = PolicyRule(selector, directory, value) continue _invalid("unknown-directive", map_route) @@ -350,12 +382,13 @@ def render_documentation_policy(role_rules, never_edit_rules, *, newline="\n"): ) ) lines.extend( - f"never-edit {rule.selector}{'/' if rule.directory else ''}" + _never_edit_directive(rule) for rule in sorted( never_edit_rules, key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -431,25 +464,22 @@ def replace_documentation_policy( ) continue if line.startswith("never-edit"): - parts = line.split(maxsplit=1) - if len(parts) != 2 or parts[0] != "never-edit": + try: + value, selector = _never_edit_parts(line) + except ValueError: raise ValueError( "policy source contains an invalid never-edit directive" ) - selector = parts[1] directory = selector.endswith("/") key = (route_key(selector[:-1] if directory else selector), directory) desired = desired_never_edit.get(key) if desired is None: continue seen_never_edit.add(key) - expected = ( - f"never-edit {desired.selector}" - f"{'/' if desired.directory else ''}" - ) + expected = _never_edit_directive(desired) retained.append( raw_line - if line == expected + if line == expected and value == desired.value else expected + ( "\r\n" if raw_line.endswith("\r\n") @@ -483,6 +513,7 @@ def replace_documentation_policy( key=lambda rule: ( route_key(rule.selector), not rule.directory, + rule.value, rule.selector, ), ) @@ -492,8 +523,7 @@ def replace_documentation_policy( for rule in missing_roles ] additions.extend( - f"never-edit {rule.selector}" - f"{'/' if rule.directory else ''}{newline}" + f"{_never_edit_directive(rule)}{newline}" for rule in missing_never_edit ) content = "".join(retained) @@ -543,7 +573,10 @@ def role_rule_for(policy, route): def never_edit_rule_for(policy, route): - return _matching_rule(policy.never_edit_rules, route) + rule = _matching_rule(policy.never_edit_rules, route) + if rule is None or rule.value != NEVER_EDIT_VALUE: + return None + return rule def invalid_policy_envelope(error, *, doctor=False): @@ -582,6 +615,8 @@ def invalid_policy_envelope(error, *, doctor=False): __all__ = ( + "NEVER_EDIT_EXCEPTION_VALUE", + "NEVER_EDIT_VALUE", "POLICY_ROLES", "DocumentationPolicy", "InvalidDocumentationPolicy", diff --git a/skills/docs/scripts/_docs_checker/generator_evidence.py b/skills/docs/scripts/_docs_checker/generator_evidence.py new file mode 100644 index 0000000..672f581 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/generator_evidence.py @@ -0,0 +1,71 @@ +"""Confined, bounded evidence for generator-declared documentation outputs.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .paths import normalize_repo_relative, safe_path + + +# These files are emitted into the output root by this project's canonical +# generator. Their exact bytes are the declaration; the surrounding directory +# name is not evidence by itself. +_OUTPUT_MARKERS = { + ".statusnone-generated-plugin": b"statusnone-codex-plugin-v1\n", +} + + +def _ancestor_directories(route): + parent = PurePosixPath(route).parent + while True: + relative = parent.as_posix() + yield relative + if relative == ".": + return + parent = parent.parent + + +def declared_generator_evidence(root, routes): + """Map accounted routes to the exact marker that declares their output. + + Only ancestors of already-accounted documents are probed, each marker is + checked at most once, and a marker is read only when its size matches the + supported declaration. Document bodies are never opened here. + """ + root = Path(root).absolute() + safe_path(root, root) + normalized_routes = tuple( + normalize_repo_relative(route, "generated document route") + for route in routes + ) + marker_state = {} + + def declared(marker_route, expected): + if marker_route in marker_state: + return marker_state[marker_route] + try: + marker = safe_path(root / marker_route, root) + if not marker.is_file(): + matches = False + else: + with marker.open("rb") as stream: + matches = stream.read(len(expected) + 1) == expected + except (OSError, ValueError): + matches = False + marker_state[marker_route] = matches + return matches + + evidence = {} + for route in normalized_routes: + for directory in _ancestor_directories(route): + for name, expected in _OUTPUT_MARKERS.items(): + marker_route = name if directory == "." else f"{directory}/{name}" + if declared(marker_route, expected): + evidence[route] = f"declared-generator:{marker_route}" + break + if route in evidence: + break + return evidence + + +__all__ = ("declared_generator_evidence",) diff --git a/skills/docs/scripts/_docs_checker/manifest.py b/skills/docs/scripts/_docs_checker/manifest.py index 349e1c4..da58b77 100644 --- a/skills/docs/scripts/_docs_checker/manifest.py +++ b/skills/docs/scripts/_docs_checker/manifest.py @@ -83,6 +83,7 @@ from .discovery_policy import prune_reason from .document_classification import classify_documents from .documentation_policy import DocumentationPolicy, load_documentation_policy +from .generator_evidence import declared_generator_evidence from .health import normalized_content_digest from .paths import ( ANYWHERE_PRUNE_DIRS, @@ -589,7 +590,12 @@ def build_documentation_manifest( if "_map_source_bytes" in measurements: policy_arguments["source_bytes"] = measurements["_map_source_bytes"] policy_arguments["git_available"] = status_source == "git" - policy = load_documentation_policy(root, map_route, **policy_arguments) + policy_route = measurements.get("_map_source_route", map_route) + if not isinstance(policy_route, str): + raise ValueError("policy source route must be a string") + policy_route = normalize_repo_relative(policy_route, "policy source route") + policy = load_documentation_policy(root, policy_route, **policy_arguments) + generated_evidence = declared_generator_evidence(root, accounted) classifications = classify_documents( accounted, status_of, @@ -600,6 +606,7 @@ def build_documentation_manifest( {"route": row.route, "marker": row.marker} for row in current_truth ], + generated_evidence=generated_evidence, ) maintained_set = frozenset( route diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 4926564..089ee29 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -10,12 +10,16 @@ import tempfile from collections import defaultdict from collections.abc import Mapping +from dataclasses import replace from pathlib import Path, PurePosixPath from .document_classification import classify_documents from .documentation_policy import ( + NEVER_EDIT_EXCEPTION_VALUE, + NEVER_EDIT_VALUE, POLICY_ROLES, PolicyRule, + never_edit_rule_for, parse_documentation_policy, replace_documentation_policy, route_key, @@ -278,10 +282,26 @@ def _proposed_rules(policy, decisions): never_edit_rules[key] = PolicyRule( decision["selector"], decision["directory"], - "never-edit", + NEVER_EDIT_VALUE, ) elif never_edit is False: never_edit_rules.pop(key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), + ) + if ( + never_edit_rule_for( + candidate_policy, + decision["selector"], + ) + is not None + ): + never_edit_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -291,7 +311,11 @@ def _selected_snapshot(root): from .measurement import measure_documentation from .selection import select_documentation - inventory = repository_document_inventory(root, ".") + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) selection = select_documentation( root, requested_boundary=None, @@ -325,7 +349,8 @@ def _resolution_plan(root, decisions): """Recompute one exact policy edit and its stateless approval binding.""" root = Path(root).absolute() manifest, source_bytes = _selected_snapshot(root) - map_route = manifest.boundary.map_route + authority_route = manifest.boundary.map_route + map_route = manifest.policy.source_route if source_bytes is None: raise ValueError("selected map is unavailable") try: @@ -351,7 +376,7 @@ def _resolution_plan(root, decisions): for document in manifest.documents }, proposed_policy, - map_route=map_route, + map_route=authority_route, map_exists=True, current_truth_routes=[ {"route": row.route, "marker": row.marker} diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 963d51d..2146686 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -823,6 +823,7 @@ def shared_markdown_route(relative): # Internal only: policy parsing consumes the exact selected-map bytes # this scan measured instead of reopening a potentially changed file. "_map_source_bytes": map_source_bytes, + "_map_source_route": map_norm, "cold_paths": sorted(_relative_posix(path, root) for path in cold_paths), "prunes": prune_summary(applied_prunes), } diff --git a/tests/test_adapters.py b/tests/test_adapters.py index b045652..678addd 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -125,6 +125,7 @@ def test_canonical_checker_registry_covers_package_and_generated_bundles(self): "measurement.py", "documentation_policy.py", "document_classification.py", + "generator_evidence.py", "manifest.py", "readiness.py", "documentation_result.py", diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 31b3ddb..0538360 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -398,6 +398,50 @@ def test_hints_are_recorded_beside_unresolved_roles_never_as_roles(self): "hint separately from its role", ) + def test_declared_generator_output_is_classified_before_unresolved(self): + """A generator-owned marker is explicit role evidence for documents + beneath that output root; it is not a filename or directory guess.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + write( + root, + "plugins/generated-output/.statusnone-generated-plugin", + "statusnone-codex-plugin-v1\n", + ) + write( + root, + "plugins/generated-output/skills/example/SKILL.md", + "# Generated skill\n\nBuilt from canonical source.\n", + ) + git(root, "add", "--", "plugins/generated-output") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "generated-output", + ) + + documentation = section(run_checker(root), "documentation") + generated = next( + document + for document in documentation["documents"] + if document["path"] + == "plugins/generated-output/skills/example/SKILL.md" + ) + cards = json.dumps(documentation["unresolved"], sort_keys=True) + + self.assertEqual(generated["role"], "generated") + self.assertEqual( + generated["role_evidence"], + "declared-generator:" + "plugins/generated-output/.statusnone-generated-plugin", + ) + self.assertNotIn(generated["path"], cards) + class CommittedPolicyContracts(unittest.TestCase): """Charter AC 3: one committed, deterministic, fail-closed policy block.""" @@ -645,6 +689,12 @@ def test_invalid_policy_fails_closed_as_the_invalid_policy_classification(self): "role historical docs/archive/\n" "-->\n" ), + "conflicting-never-edit-duplicate": ( + "\n" + ), "second-policy-block": second_block + second_block, } for name, block in sorted(invalid_blocks.items()): @@ -886,7 +936,7 @@ def test_readiness_reports_whether_its_denominator_represents_all_accounted_docs """Stage 4C correction 3: readiness publishes the accounted-document count, the exact nonignored unresolved count, and the engine-owned representative judgment. The real repository remains visibly - nonrepresentative with 59 unresolved documents, while a fully + nonrepresentative with 39 unresolved documents, while a fully classified fixture is representative.""" repository_readiness = section(run_checker(ROOT), "readiness") repository_manifest = manifest_payload(build_manifest(ROOT)) @@ -895,7 +945,7 @@ def test_readiness_reports_whether_its_denominator_represents_all_accounted_docs repository_readiness.get("accounted_documents"), len(repository_manifest["documents"]), ) - self.assertEqual(repository_readiness.get("nonignored_unresolved"), 59) + self.assertEqual(repository_readiness.get("nonignored_unresolved"), 39) self.assertIs( repository_readiness.get("denominator_representative"), False, @@ -1330,8 +1380,9 @@ def test_published_findings_are_stamped_with_target_treatment_facts(self): """Stage 4C correction 1: every published finding carries manifest target facts, and a single-target unreachable finding against an unresolved document says plainly that the target is not write - eligible. Stamping is additive: the repository's 27 findings remain - 27 and keep their existing numbering and content.""" + eligible. The repository's seven remaining findings keep their + existing numbering and content after declared generated outputs stop + surfacing as cold-document findings.""" with tempfile.TemporaryDirectory() as td: root = Path(td) / "repo" root.mkdir() @@ -1360,7 +1411,7 @@ def test_published_findings_are_stamped_with_target_treatment_facts(self): ) repository_findings = run_checker(ROOT)["findings"] - self.assertEqual(len(repository_findings), 27) + self.assertEqual(len(repository_findings), 7) for finding in repository_findings: with self.subTest( kind=finding.get("kind"), diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index ffcafae..56a54e7 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -369,6 +369,122 @@ def test_resolution_apply_is_exact_single_use_and_visible_on_a_fresh_run(self): self.assertEqual(reused["writes"], 0) self.assertTrue(str(reused.get("reason", "")).strip()) + def test_mintlify_policy_provenance_follows_the_entry_bytes_it_parses(self): + """A provider authority and its readable entry are different routes: + resolution writes the entry, and the next read reports that same route + as the policy source with honest working-tree durability.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/docs.json", + json.dumps( + { + "$schema": "https://mintlify.com/docs.json", + "navigation": {"pages": ["index"]}, + } + ) + + "\n", + ) + write(root, "docs/index.md", "# Index\n\nWelcome.\n") + write(root, "docs/extra.md", "# Extra\n\nUnclassified.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "mintlify") + + decisions = {"docs/extra.md": {"role": "preserved"}} + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + documentation = section(run_checker(root), "documentation") + records = { + document["path"]: document + for document in documentation["documents"] + } + + self.assertEqual(preview["map"], "docs/index.md") + self.assertEqual(applied["map"], "docs/index.md") + self.assertEqual( + documentation["policy"]["source_route"], + "docs/index.md", + ) + self.assertEqual(documentation["policy"]["git"], "uncommitted") + self.assertIs(documentation["policy"]["durable"], False) + self.assertEqual( + documentation["policy"]["notice"], + "not-yet-durable", + ) + self.assertEqual(records["docs/extra.md"]["role"], "preserved") + + git(root, "add", "--", "docs/index.md") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "persist-policy", + ) + clone = Path(td) / "clone" + git(Path(td), "clone", "--quiet", str(root), str(clone)) + cloned = section(run_checker(clone), "documentation") + cloned_records = { + document["path"]: document + for document in cloned["documents"] + } + + self.assertEqual(cloned["policy"]["source_route"], "docs/index.md") + self.assertEqual(cloned["policy"]["git"], "committed") + self.assertIs(cloned["policy"]["durable"], True) + self.assertEqual( + cloned_records["docs/extra.md"]["role"], + "preserved", + ) + + def test_exact_never_edit_false_overrides_its_directory_decision(self): + """An exact-file false choice is a durable exception to the containing + directory's true choice, so the file remains write-eligible.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + for index in range(1, 4): + write( + root, + f"research/note-{index:02d}.md", + f"# Note {index:02d}\n\nRecorded result.\n", + ) + git(root, "add", "--", "research") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "research") + + decisions = { + "research/": {"never_edit": True}, + "research/note-01.md": {"never_edit": False}, + } + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + records = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + + self.assertIn("never-edit research/", preview["map_diff"]) + self.assertIn( + "never-edit false research/note-01.md", + preview["map_diff"], + ) + self.assertEqual(applied["status"], "applied") + self.assertIs(records["research/note-01.md"]["never_edit"], False) + self.assertIs(records["research/note-02.md"]["never_edit"], True) + self.assertIs(records["research/note-03.md"]["never_edit"], True) + def test_resolution_receipt_survives_a_process_boundary(self): """P2-1: a DOCS-R receipt is a stateless digest binding, so preview and apply may run in separate Python processes without local memory.""" diff --git a/tools/build_adapters.py b/tools/build_adapters.py index 77668e9..29e7443 100644 --- a/tools/build_adapters.py +++ b/tools/build_adapters.py @@ -46,6 +46,7 @@ "scripts/_docs_checker/measurement.py", "scripts/_docs_checker/documentation_policy.py", "scripts/_docs_checker/document_classification.py", + "scripts/_docs_checker/generator_evidence.py", "scripts/_docs_checker/manifest.py", "scripts/_docs_checker/readiness.py", "scripts/_docs_checker/documentation_result.py", From e74dd42df8651f9c82ad50b05a7ad85c9da7c802 Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 20:09:14 -0400 Subject: [PATCH 06/19] Preserve Init map bytes and bound resolution input --- BENCHMARK.md | 4 +- COMMANDS.md | 2 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../docs/scripts/_docs_checker/resolution.py | 21 +++ adapters/claude/skills/docs/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../scripts/_docs_checker/resolution.py | 21 +++ adapters/copilot/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../scripts/_docs_checker/resolution.py | 21 +++ adapters/cursor/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../grok/scripts/_docs_checker/resolution.py | 21 +++ adapters/grok/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../docs/scripts/_docs_checker/resolution.py | 21 +++ adapters/plugin/skills/docs/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../docs/scripts/_docs_checker/resolution.py | 21 +++ .../skills/docs/scripts/check.py | 4 +- .../scripts/_docs_checker/init_adoption.py | 77 +++++++- .../docs/scripts/_docs_checker/resolution.py | 21 +++ skills/docs/scripts/check.py | 4 +- tests/test_resolution_treatment_contract.py | 166 ++++++++++++++++++ 24 files changed, 855 insertions(+), 31 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index ada79bd..0aeee01 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,12 +6,12 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,149 tests — 518 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 61 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,151 tests — 520 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 63 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | | Repository documentation check | A fresh checkout exits 0 with repository-complete accounting for 61 documentation paths — 2 maintained, 20 generator-declared outputs, and 39 unresolved. Readiness v3 is 2 of 2 eligible documents; `accounted_documents` is 61, `nonignored_unresolved` is 39, and `denominator_representative` is false, so that 100% eligible fraction is not whole-repository health. Local working copies with ignored drafts account for those additional paths by name and count while leaving their bodies unread. The 7 remaining published findings carry target role and write-eligibility facts; generated cold documents no longer surface as repair findings. Transitional rubric v2 remains separately visible at 89% until the beginner-interface phase removes it. | -| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The 1,146-test snapshot passed on hosted Ubuntu and Windows in PR #30's `validate` run #295; hosted coverage for the current 1,149-test snapshot is reported by that PR's latest validate run. | +| Full Linux rerun for this branch | The pre-guidance 906-test snapshot passed on Ubuntu/Python 3.14.4 in 94 s. The 1,149-test snapshot passed on hosted Ubuntu and Windows in PR #30's `validate` run #297; that PR's checks are authoritative for later snapshots. | | Installed-package no-state retry | Pass: 49/49 links valid, a stateless approval receipt was produced, and the repository remained unchanged without `.diataxis/` | | Installed-package guided recovery | Pass outside the source tree: foreign-only, tracked-missing, merge-conflicted, and absent-state fixtures selected the expected four outcomes with zero writes and no private sentinel exposure | | Installed-package treatment canary | Pass: exact stateless preview → one-file apply → candidate and installed verification clean; zero control writes, no `.diataxis/`, unchanged real Git index, and only the intended disposable worktree file changed | diff --git a/COMMANDS.md b/COMMANDS.md index 74fba28..f16eefe 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -31,7 +31,7 @@ Doctor reports every finding in the evidenced scope, groups them into treatments `check` now returns readiness v3 as an unweighted ready/eligible fraction with every per-document failed fact. Trust coverage (are declared current-truth routes verified?), hash freshness, and byte telemetry stay separate. The envelope temporarily also carries rubric-v2 structural health for compatibility; the two percentages are different contracts and are never compared. A route counts as declared current truth when your documentation map marks it: append `` (or ``) to the end of the map line that links your current-state document. The engine reads that marker; it never infers a current-truth route from prose, and a map that declares none reports Trust as `unverified` rather than guessing. -The canonical checker exposes role-resolution writes through two machine-output flags. `--resolve-preview` reads one JSON decisions mapping from standard input and returns the exact zero-write map-policy diff plus its `DOCS-R-` approval. `--resolve-apply APPROVAL` reads the same mapping in a later process, recomputes the binding from the current snapshot, and applies only the matching current code; successful previews and applies exit 0, while refusals exit 2 with zero writes. +The canonical checker exposes role-resolution writes through two machine-output flags. `--resolve-preview` reads one UTF-8 JSON decisions mapping from standard input, bounded to 1 MiB before parsing, and returns the exact zero-write map-policy diff plus its `DOCS-R-` approval. `--resolve-apply APPROVAL` reads the same bounded mapping in a later process, recomputes the binding from the current snapshot, and applies only the matching current code; oversized or malformed input uses the existing input-error envelope, successful previews and applies exit 0, and refusals exit 2 with zero writes. ## Boundaries diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/claude/skills/docs/scripts/check.py b/adapters/claude/skills/docs/scripts/check.py index f4ef289..5780b31 100644 --- a/adapters/claude/skills/docs/scripts/check.py +++ b/adapters/claude/skills/docs/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/adapters/copilot/scripts/_docs_checker/init_adoption.py b/adapters/copilot/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/adapters/copilot/scripts/_docs_checker/init_adoption.py +++ b/adapters/copilot/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/copilot/scripts/check.py b/adapters/copilot/scripts/check.py index f4ef289..5780b31 100644 --- a/adapters/copilot/scripts/check.py +++ b/adapters/copilot/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/adapters/cursor/scripts/_docs_checker/init_adoption.py b/adapters/cursor/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/adapters/cursor/scripts/_docs_checker/init_adoption.py +++ b/adapters/cursor/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/cursor/scripts/check.py b/adapters/cursor/scripts/check.py index f4ef289..5780b31 100644 --- a/adapters/cursor/scripts/check.py +++ b/adapters/cursor/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/adapters/grok/scripts/_docs_checker/init_adoption.py b/adapters/grok/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/adapters/grok/scripts/_docs_checker/init_adoption.py +++ b/adapters/grok/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/grok/scripts/check.py b/adapters/grok/scripts/check.py index f4ef289..5780b31 100644 --- a/adapters/grok/scripts/check.py +++ b/adapters/grok/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/adapters/plugin/skills/docs/scripts/check.py b/adapters/plugin/skills/docs/scripts/check.py index f4ef289..5780b31 100644 --- a/adapters/plugin/skills/docs/scripts/check.py +++ b/adapters/plugin/skills/docs/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/plugins/diataxis-docs/skills/docs/scripts/check.py b/plugins/diataxis-docs/skills/docs/scripts/check.py index f4ef289..5780b31 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/check.py +++ b/plugins/diataxis-docs/skills/docs/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/skills/docs/scripts/_docs_checker/init_adoption.py b/skills/docs/scripts/_docs_checker/init_adoption.py index 997c2a0..686de9b 100644 --- a/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/skills/docs/scripts/_docs_checker/init_adoption.py @@ -6,7 +6,7 @@ import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import posixpath import tempfile from urllib.parse import quote @@ -158,6 +158,50 @@ def _map_git_recovery(root, map_path, source_bytes): } +def _map_archive_recovery(root, map_path, source_bytes): + """Plan one confined archive containing the exact pre-Init map bytes.""" + digest = hashlib.sha256(source_bytes).hexdigest() + parent = PurePosixPath(map_path).parent + archive_path = ( + parent / "archive" / f"map-before-init-{digest}.md" + ).as_posix() + try: + target = safe_path(Path(root) / archive_path, root) + if os.path.lexists(target): + raise InitCloseoutError( + "waiting", + "map-policy-recovery-collision", + "adoption-policy", + ) + except InitCloseoutError: + raise + except (OSError, ValueError) as exc: + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) from exc + return { + "kind": "archive", + "mode": "planned", + "path": archive_path, + "digest": "sha256:" + digest, + } + + +def _map_recovery(root, map_path, source_bytes, policy_git): + """Use committed Git bytes when exact; otherwise preserve working bytes.""" + if policy_git == "committed": + return _map_git_recovery(root, map_path, source_bytes) + if policy_git in {"uncommitted", "unavailable"}: + return _map_archive_recovery(root, map_path, source_bytes) + raise InitCloseoutError( + "waiting", + "map-policy-recovery-unavailable", + "adoption-policy", + ) + + def measure_adoption_repository( root, measure, @@ -544,6 +588,7 @@ def _request_from_context( } for relative in paths ] + map_recovery = None if existing_map_policy_edit: map_item_id = f"{map_path}#" map_disposition = next( @@ -560,6 +605,12 @@ def _request_from_context( "map-policy-source-unavailable", "adoption-policy", ) + map_recovery = _map_recovery( + root, + map_path, + map_source, + setup_measurements["manifest"].policy.git, + ) map_disposition.update( { "disposition": "MIGRATED", @@ -568,7 +619,7 @@ def _request_from_context( "canonical documentation map." ), "target": map_path, - "recovery": _map_git_recovery(root, map_path, map_source), + "recovery": map_recovery, } ) document_changes = [] @@ -593,6 +644,21 @@ def _request_from_context( selected_scope, ) elif existing_map_policy_edit: + if ( + map_recovery["kind"] == "archive" + and map_recovery["mode"] == "planned" + ): + document_changes.append( + { + "operation": "CREATE", + "path": map_recovery["path"], + "reason": ( + "Preserve the exact pre-Init map bytes for recovery." + ), + "source_item_ids": [f"{map_path}#"], + "content_base64": base64.b64encode(map_source).decode("ascii"), + } + ) document_changes.append( { "operation": "REPLACE", @@ -604,6 +670,13 @@ def _request_from_context( "content_base64": base64.b64encode(proposed_map).decode("ascii"), } ) + document_changes.sort( + key=lambda item: ( + item["path"].casefold(), + item["path"], + item["operation"], + ) + ) declared_routes = [] seen_routes = set() for route in (map_path, *current_truth_routes): diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 089ee29..701adca 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -34,6 +34,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) +MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -74,6 +75,24 @@ def strict_object(pairs): return decisions +def read_resolution_decisions(stream): + """Read and parse one UTF-8 decisions mapping within the public capacity.""" + try: + raw = stream.read(MAX_RESOLUTION_REQUEST_BYTES + 1) + except OSError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + if ( + not isinstance(raw, bytes) + or len(raw) > MAX_RESOLUTION_REQUEST_BYTES + ): + raise ValueError(_RESOLUTION_INPUT_ERROR) + try: + text = raw.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + return parse_resolution_decisions(text) + + def _handle(kind, path, routes): digest = hashlib.sha256( _canonical_bytes( @@ -499,8 +518,10 @@ def resolution_apply(root, decisions, approval): __all__ = ( "LEAVE_UNRESOLVED", + "MAX_RESOLUTION_REQUEST_BYTES", "RESOLUTION_CHOICES", "parse_resolution_decisions", + "read_resolution_decisions", "resolution_apply", "resolution_preview", "unresolved_cards", diff --git a/skills/docs/scripts/check.py b/skills/docs/scripts/check.py index f4ef289..5780b31 100644 --- a/skills/docs/scripts/check.py +++ b/skills/docs/scripts/check.py @@ -152,7 +152,7 @@ ) from _docs_checker.measurement import measure_documentation from _docs_checker.resolution import ( - parse_resolution_decisions, + read_resolution_decisions, resolution_apply, resolution_preview, ) @@ -424,7 +424,7 @@ def main(argv=None): if _is_reparse(raw) or not os.path.isdir(raw): raise ValueError("root must be a real directory") root = safe_path(raw, raw) - decisions = parse_resolution_decisions(sys.stdin.read()) + decisions = read_resolution_decisions(sys.stdin.buffer) try: resolution_response = ( resolution_preview(root, decisions) diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 56a54e7..762e056 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -634,6 +634,78 @@ def test_checker_resolution_cli_is_cross_process_exact_and_zero_write_on_refusal self.assertEqual(stale_result["writes"], 0) self.assertEqual(map_path.read_bytes(), drifted) + def test_checker_bounds_resolution_stdin_before_json_parsing(self): + """The public resolution boundary reads at most its documented byte + capacity plus one before JSON parsing and returns the existing safe + input error with zero repository writes when that capacity is crossed.""" + from _docs_checker.resolution import ( + MAX_RESOLUTION_REQUEST_BYTES, + read_resolution_decisions, + ) + + class OversizedInput: + def __init__(self): + self.read_sizes = [] + + def read(self, size=-1): + self.read_sizes.append(size) + return b"{" + (b"x" * MAX_RESOLUTION_REQUEST_BYTES) + + source = OversizedInput() + with self.assertRaisesRegex( + ValueError, + "^resolution decisions must be one JSON object on standard input$", + ): + read_resolution_decisions(source) + self.assertEqual( + source.read_sizes, + [MAX_RESOLUTION_REQUEST_BYTES + 1], + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + before = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + payload = json.dumps( + { + "HISTORY-2024.md": { + "role": "historical", + "padding": "x" * MAX_RESOLUTION_REQUEST_BYTES, + } + } + ) + + completed = subprocess.run( + [ + sys.executable, + "-B", + str(CHECKER), + str(root), + "--resolve-preview", + ], + input=payload, + capture_output=True, + text=True, + cwd=str(ROOT), + ) + after = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + + self.assertEqual(completed.returncode, 2, completed.stderr) + self.assertEqual( + json.loads(completed.stdout)["error"], + "resolution decisions must be one JSON object on standard input", + ) + self.assertEqual(after, before) + def test_resolution_apply_refuses_drift_with_zero_writes(self): """Stage 3B resolution apply compares the selected map's current bytes to its preview binding and names drift without writing.""" @@ -905,6 +977,100 @@ def test_approved_init_persists_each_never_edit_choice_into_the_map(self): expected_rule is not None, ) + def test_init_policy_edits_recover_exact_non_git_and_dirty_map_bytes(self): + """Init previews and applies its approved never-edit policy for both a + non-Git repository and a Git repository whose selected map has owner + edits, using the existing archive recovery path for those exact bytes.""" + from _docs_checker.documentation_policy import ( + load_documentation_policy, + never_edit_rule_for, + ) + from _docs_checker.init_adoption import adoption_apply, adoption_preview + + for name, git_repository in ( + ("non-git", False), + ("dirty-git", True), + ): + with self.subTest(shape=name), tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + if git_repository: + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n- [Guide](guide.md)\n", + ) + write(root, "docs/guide.md", "# Guide\n\nShared guidance.\n") + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask.\n") + if git_repository: + git(root, "add", "--", "docs", ".github") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "fixture", + ) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "Owner working-tree note.\n", + ) + + map_path = root / "docs" / "README.md" + source_bytes = map_path.read_bytes() + before = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(root).parts + } + + request, preview = adoption_preview( + root, + completed_at="2026-07-29T00:00:00Z", + ) + repeated_request, repeated_preview = adoption_preview( + root, + completed_at="2026-07-29T00:00:00Z", + ) + after_preview = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(root).parts + } + map_disposition = next( + item + for item in request["evidence"]["dispositions"] + if item["path"] == "docs/README.md" + ) + recovery = map_disposition["recovery"] + archive_path = root / recovery["path"] + + self.assertEqual(preview["status"], "approval-required") + self.assertEqual(after_preview, before) + self.assertEqual(recovery["kind"], "archive") + self.assertEqual(recovery["mode"], "planned") + self.assertTrue(recovery["path"].startswith("docs/archive/")) + self.assertFalse(archive_path.exists()) + self.assertEqual(repeated_request, request) + self.assertEqual(repeated_preview["approval"], preview["approval"]) + + applied = adoption_apply(root, request, preview["approval"]) + + self.assertEqual(applied["status"], "applied") + self.assertEqual(archive_path.read_bytes(), source_bytes) + self.assertIn(source_bytes, map_path.read_bytes()) + policy = load_documentation_policy(root, "docs/README.md") + self.assertIsNotNone( + never_edit_rule_for(policy, ".github/SUPPORT.md") + ) + def test_the_selected_map_stays_write_eligible_unless_user_never_edits_it(self): """PRODUCT.md 'Classification is three separate facts' plus owner gate 3: there are no hardcoded filename bans, so the selected map From 629fc77b6c0e9e20b9999d3da9662f5163488571 Mon Sep 17 00:00:00 2001 From: Statusnone420 Date: Wed, 29 Jul 2026 21:02:38 -0400 Subject: [PATCH 07/19] Honor exact unresolved choices and frontmatter titles --- BENCHMARK.md | 2 +- .../docs/scripts/_docs_checker/readiness.py | 18 ++++--- .../docs/scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- .../skills/docs/scripts/_docs_checker/scan.py | 13 ++++- .../scripts/_docs_checker/readiness.py | 18 ++++--- .../scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- .../copilot/scripts/_docs_checker/scan.py | 13 ++++- .../cursor/scripts/_docs_checker/readiness.py | 18 ++++--- .../scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- adapters/cursor/scripts/_docs_checker/scan.py | 13 ++++- .../grok/scripts/_docs_checker/readiness.py | 18 ++++--- .../grok/scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- adapters/grok/scripts/_docs_checker/scan.py | 13 ++++- .../docs/scripts/_docs_checker/readiness.py | 18 ++++--- .../docs/scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- .../skills/docs/scripts/_docs_checker/scan.py | 13 ++++- .../docs/scripts/_docs_checker/readiness.py | 18 ++++--- .../docs/scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- .../skills/docs/scripts/_docs_checker/scan.py | 13 ++++- .../docs/scripts/_docs_checker/readiness.py | 18 ++++--- .../docs/scripts/_docs_checker/resolution.py | 48 ++++++++++++++++--- skills/docs/scripts/_docs_checker/scan.py | 13 ++++- tests/test_documentation_result_contract.py | 32 +++++++++++++ tests/test_resolution_treatment_contract.py | 40 ++++++++++++++++ 24 files changed, 528 insertions(+), 99 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 0aeee01..ae7b6fa 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,151 tests — 520 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 63 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,153 tests — 522 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 65 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/adapters/copilot/scripts/_docs_checker/readiness.py b/adapters/copilot/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/adapters/copilot/scripts/_docs_checker/readiness.py +++ b/adapters/copilot/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/adapters/cursor/scripts/_docs_checker/readiness.py b/adapters/cursor/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/adapters/cursor/scripts/_docs_checker/readiness.py +++ b/adapters/cursor/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/adapters/grok/scripts/_docs_checker/readiness.py b/adapters/grok/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/adapters/grok/scripts/_docs_checker/readiness.py +++ b/adapters/grok/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/skills/docs/scripts/_docs_checker/readiness.py b/skills/docs/scripts/_docs_checker/readiness.py index 8efa11a..b7967ea 100644 --- a/skills/docs/scripts/_docs_checker/readiness.py +++ b/skills/docs/scripts/_docs_checker/readiness.py @@ -50,28 +50,32 @@ def _title_facts(rows, evidence): for value in observed.get("h1s", ()) if isinstance(value, str) and value.strip() ] + frontmatter_title = observed.get("frontmatter_title") + if not isinstance(frontmatter_title, str) or not frontmatter_title.strip(): + frontmatter_title = None + title = h1s[0] if len(h1s) == 1 else frontmatter_title if not observed: row["facts"]["title"] = _fact( False, classification="title-unmeasured", ) - elif len(h1s) == 0: - row["facts"]["title"] = _fact( - False, - classification="missing-title", - ) elif len(h1s) > 1: row["facts"]["title"] = _fact( False, classification="multiple-titles", titles=h1s, ) + elif title is None: + row["facts"]["title"] = _fact( + False, + classification="missing-title", + ) else: - normalized = _normalized_text(h1s[0]) + normalized = _normalized_text(title) row["facts"]["title"] = _fact( True, classification="unique-title", - title=h1s[0], + title=title, ) duplicate_candidates.setdefault(normalized, []).append(path) diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 701adca..d3b3154 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -278,7 +278,7 @@ def _directory_route_bindings(manifest, decisions): return bindings -def _proposed_rules(policy, decisions): +def _proposed_rules(policy, decisions, manifest): role_rules = { _rule_key(rule.selector, rule.directory): rule for rule in policy.role_rules @@ -287,15 +287,47 @@ def _proposed_rules(policy, decisions): _rule_key(rule.selector, rule.directory): rule for rule in policy.never_edit_rules } + exact_unresolved = { + route_key(decision["selector"]) + for decision in decisions + if ( + not decision["directory"] + and decision["role"] == LEAVE_UNRESOLVED + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - role_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - role, + prefix = route_key(decision["selector"]) + "/" + unresolved_exceptions = ( + { + route + for route in exact_unresolved + if route.startswith(prefix) + } + if decision["directory"] + else set() ) + if unresolved_exceptions: + role_rules.pop(key, None) + for document in manifest.documents: + document_key = route_key(document.path) + if ( + document_key.startswith(prefix) + and document_key not in unresolved_exceptions + ): + exact_key = _rule_key(document.path, False) + role_rules.setdefault( + exact_key, + PolicyRule(document.path, False, role), + ) + else: + role_rules[key] = PolicyRule( + decision["selector"], + decision["directory"], + role, + ) never_edit = decision["never_edit"] if never_edit is True: never_edit_rules[key] = PolicyRule( @@ -377,7 +409,11 @@ def _resolution_plan(root, decisions): except UnicodeDecodeError as exc: raise ValueError("selected map must be UTF-8") from exc normalized = _decisions(root, decisions) - role_rules, never_edit_rules = _proposed_rules(manifest.policy, normalized) + role_rules, never_edit_rules = _proposed_rules( + manifest.policy, + normalized, + manifest, + ) proposed_text = replace_documentation_policy( source_text, role_rules, diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 2146686..ff8ab97 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -341,6 +341,7 @@ def shared_markdown_route(relative): titles = {} first_h1 = {} h1_values = {} + frontmatter_titles = {} texts = {} map_source_bytes = None for path in files: @@ -365,7 +366,16 @@ def shared_markdown_route(relative): ] first_h1[path] = next(iter(h1_values[path]), None) metadata = parse_frontmatter_scalars(raw_text) - title = first_h1[path] or metadata["values"].get("title") + frontmatter_title = metadata["values"].get("title") + if ( + isinstance(frontmatter_title, str) + and frontmatter_title.strip() + and "title" not in metadata.get("unresolved", ()) + ): + frontmatter_titles[path] = frontmatter_title.strip() + else: + frontmatter_titles[path] = None + title = first_h1[path] or frontmatter_title if isinstance(title, str) and title.strip(): titles.setdefault(title.strip().lower(), []).append( _relative_posix(path, root) @@ -782,6 +792,7 @@ def shared_markdown_route(relative): "path": route, "reachable": path in reachable, "h1s": list(h1_values.get(path, ())), + "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( map_link_labels.get(route, ()), key=lambda value: (value.casefold(), value), diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 0538360..abed073 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -1014,6 +1014,38 @@ def test_every_unready_document_names_each_failed_fact(self): self.assertIs(lonely.get("ready"), False) self.assertIn("reachability", json.dumps(lonely.get("failed_facts"))) + def test_supported_frontmatter_title_counts_for_readiness_identity(self): + """A supported scalar frontmatter title is the document's title + identity when the maintained Markdown or MDX body has no literal H1.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Frontmatter guide](guide.mdx)\n" + "\n\n", + ) + write( + root, + "docs/guide.mdx", + "---\ntitle: Frontmatter Guide\n---\n\n## Start\n\nBody.\n", + ) + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "frontmatter") + + readiness = section(run_checker(root), "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + title = rows["docs/guide.mdx"]["facts"]["title"] + + self.assertEqual(readiness["ready_count"], 2) + self.assertEqual(readiness["eligible_count"], 2) + self.assertEqual(readiness["percentage"], 100) + self.assertIs(title["passed"], True) + self.assertEqual(title["title"], "Frontmatter Guide") + def test_a_repository_without_a_selected_map_reports_no_percentage(self): """No selected map means no readiness percentage — an honest no-map state, never a misleading low number.""" diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 762e056..277793c 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -215,6 +215,46 @@ def test_directory_groups_resolve_in_one_reply_with_exact_exceptions(self): self.assertIn("role preserved research/", map_diff) self.assertIn("role historical research/note-02.md", map_diff) + def test_exact_leave_unresolved_overrides_its_directory_group(self): + """An exact leave-unresolved reply wins over a role chosen for its + containing directory, without persisting unresolved as a policy role.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + for index in range(1, 4): + write( + root, + f"research/note-{index:02d}.md", + f"# Note {index:02d}\n\nRecorded result.\n", + ) + git(root, "add", "--", "research") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "research") + decisions = { + "research/": {"role": "preserved"}, + "research/note-02.md": { + "role": LEAVE_UNRESOLVED_CHOICE, + }, + } + + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + records = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + + self.assertEqual(applied["status"], "applied") + self.assertNotIn("role unresolved", preview["map_diff"]) + self.assertEqual(records["research/note-01.md"]["role"], "preserved") + self.assertEqual(records["research/note-02.md"]["role"], "unresolved") + self.assertEqual(records["research/note-03.md"]["role"], "preserved") + def test_directory_resolution_approval_binds_the_snapshot_route_list(self): """Stage 4C correction 4: a directory decision's DOCS-R approval includes the exact manifest routes it resolves, so adding a document From a134a8d975b66ce0d9ca9374e277dc2bbf5b5644 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:01:28 -0400 Subject: [PATCH 08/19] Honor explicit Init never-edit choices --- BENCHMARK.md | 2 +- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- .../scripts/_docs_checker/init_adoption.py | 7 +-- tests/test_resolution_treatment_contract.py | 62 +++++++++++++++++++ 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index ae7b6fa..5c14175 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,153 tests — 522 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 65 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,154 tests — 523 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 66 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/adapters/copilot/scripts/_docs_checker/init_adoption.py b/adapters/copilot/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/adapters/copilot/scripts/_docs_checker/init_adoption.py +++ b/adapters/copilot/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/adapters/cursor/scripts/_docs_checker/init_adoption.py b/adapters/cursor/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/adapters/cursor/scripts/_docs_checker/init_adoption.py +++ b/adapters/cursor/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/adapters/grok/scripts/_docs_checker/init_adoption.py b/adapters/grok/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/adapters/grok/scripts/_docs_checker/init_adoption.py +++ b/adapters/grok/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/skills/docs/scripts/_docs_checker/init_adoption.py b/skills/docs/scripts/_docs_checker/init_adoption.py index 686de9b..1f33e6a 100644 --- a/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/skills/docs/scripts/_docs_checker/init_adoption.py @@ -124,10 +124,9 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): for rule in policy.never_edit_rules } for rule in _never_edit_rules(never_edit): - never_edit_rules.setdefault( - (_path_identity(rule.selector), rule.directory), - rule, - ) + never_edit_rules[ + (_path_identity(rule.selector), rule.directory) + ] = rule proposed = replace_documentation_policy( source_text, policy.role_rules, diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 277793c..138622a 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -1017,6 +1017,68 @@ def test_approved_init_persists_each_never_edit_choice_into_the_map(self): expected_rule is not None, ) + def test_init_explicit_never_edit_replaces_same_path_edit_allowed_exception(self): + """An explicit Init protection choice replaces an existing exact + edit-allowed exception for the same selector in the canonical map.""" + from _docs_checker.documentation_policy import ( + NEVER_EDIT_VALUE, + load_documentation_policy, + never_edit_rule_for, + ) + from _docs_checker.init_adoption import adoption_apply, adoption_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8") + + "\n\n", + encoding="utf-8", + newline="\n", + ) + git(root, "add", "--", "docs/README.md") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "edit-allowed", + ) + + request, preview = adoption_preview( + root, + never_edit=("docs/guide.md",), + completed_at="2026-07-30T00:00:00Z", + ) + result = adoption_apply(root, request, preview["approval"]) + policy = load_documentation_policy(root, "docs/README.md") + matching = [ + rule + for rule in policy.never_edit_rules + if rule.selector == "docs/guide.md" and not rule.directory + ] + documents = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + + self.assertEqual(result["status"], "applied") + self.assertEqual(len(matching), 1) + self.assertEqual(matching[0].value, NEVER_EDIT_VALUE) + self.assertIsNotNone( + never_edit_rule_for(policy, "docs/guide.md") + ) + self.assertIs(documents["docs/guide.md"]["never_edit"], True) + self.assertIs(documents["docs/guide.md"]["write_eligible"], False) + def test_init_policy_edits_recover_exact_non_git_and_dirty_map_bytes(self): """Init previews and applies its approved never-edit policy for both a non-Git repository and a Git repository whose selected map has owner From f0b41270bdabb7737347bce8be716d4c067a3a2f Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:31:11 -0400 Subject: [PATCH 09/19] Scope directory resolutions and honor Mintlify labels --- BENCHMARK.md | 2 +- .../docs/scripts/_docs_checker/resolution.py | 77 +++++++++++-------- .../skills/docs/scripts/_docs_checker/scan.py | 21 ++++- .../scripts/_docs_checker/resolution.py | 77 +++++++++++-------- .../copilot/scripts/_docs_checker/scan.py | 21 ++++- .../scripts/_docs_checker/resolution.py | 77 +++++++++++-------- adapters/cursor/scripts/_docs_checker/scan.py | 21 ++++- .../grok/scripts/_docs_checker/resolution.py | 77 +++++++++++-------- adapters/grok/scripts/_docs_checker/scan.py | 21 ++++- .../docs/scripts/_docs_checker/resolution.py | 77 +++++++++++-------- .../skills/docs/scripts/_docs_checker/scan.py | 21 ++++- .../docs/scripts/_docs_checker/resolution.py | 77 +++++++++++-------- .../skills/docs/scripts/_docs_checker/scan.py | 21 ++++- .../docs/scripts/_docs_checker/resolution.py | 77 +++++++++++-------- skills/docs/scripts/_docs_checker/scan.py | 21 ++++- tests/test_documentation_result_contract.py | 57 ++++++++++++++ tests/test_resolution_treatment_contract.py | 67 +++++++++++++++- 17 files changed, 586 insertions(+), 226 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 5c14175..d422b51 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,154 tests — 523 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 66 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,156 tests — 525 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 68 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index d3b3154..0486c0c 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -254,21 +254,46 @@ def _rule_key(selector, directory): return route_key(selector), directory +def _advertised_directory_routes(manifest, selector): + """Return the nonignored unresolved routes named by one group card.""" + selector_key = route_key(selector) + return sorted( + ( + document.path + for document in manifest.documents + if ( + document.role == "unresolved" + and document.repository_status != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) == selector_key + ) + ), + key=lambda value: (value.casefold(), value), + ) + + def _directory_route_bindings(manifest, decisions): """Bind each directory decision to the snapshot routes it can classify.""" bindings = [] for decision in decisions: if not decision["directory"]: continue - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + if decision["role"] is not None: + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) + else: + prefix = route_key(decision["selector"]) + "/" + routes = sorted( + ( + document.path + for document in manifest.documents + if route_key(document.path).startswith(prefix) + ), + key=lambda value: (value.casefold(), value), + ) bindings.append( { "selector": decision["selector"] + "/", @@ -299,29 +324,19 @@ def _proposed_rules(policy, decisions, manifest): key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] if role is not None and role != LEAVE_UNRESOLVED: - prefix = route_key(decision["selector"]) + "/" - unresolved_exceptions = ( - { - route - for route in exact_unresolved - if route.startswith(prefix) - } - if decision["directory"] - else set() - ) - if unresolved_exceptions: + if decision["directory"]: role_rules.pop(key, None) - for document in manifest.documents: - document_key = route_key(document.path) - if ( - document_key.startswith(prefix) - and document_key not in unresolved_exceptions - ): - exact_key = _rule_key(document.path, False) - role_rules.setdefault( - exact_key, - PolicyRule(document.path, False, role), - ) + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ): + if route_key(route) in exact_unresolved: + continue + exact_key = _rule_key(route, False) + role_rules.setdefault( + exact_key, + PolicyRule(route, False, role), + ) else: role_rules[key] = PolicyRule( decision["selector"], diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index ff8ab97..6f52351 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -771,6 +771,22 @@ def shared_markdown_route(relative): if key in finding } link_failures.setdefault(route, []).append(failure) + navigation_labels = {} + if navigation and navigation.get("provider") == "mintlify": + for route, rows in navigation.get("contexts", {}).items(): + labels = { + " / ".join( + str(part).strip() + for part in row.get("breadcrumb", ()) + ) + for row in rows + if ( + row.get("hidden") is not True + and row.get("breadcrumb") + ) + } + if labels: + navigation_labels[route] = labels document_evidence = [] for path in sorted( files, @@ -794,7 +810,10 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - map_link_labels.get(route, ()), + ( + set(map_link_labels.get(route, ())) + | navigation_labels.get(route, set()) + ), key=lambda value: (value.casefold(), value), ), "link_failures": failures, diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index abed073..36abd34 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -1046,6 +1046,63 @@ def test_supported_frontmatter_title_counts_for_readiness_identity(self): self.assertIs(title["passed"], True) self.assertEqual(title["title"], "Frontmatter Guide") + def test_mintlify_breadcrumbs_distinguish_duplicate_title_identity(self): + """Validated visible Mintlify breadcrumbs provide the distinct labels + readiness needs when separate navigation groups reuse one page title.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/docs.json", + json.dumps( + { + "$schema": "https://mintlify.com/docs.json", + "navigation": { + "groups": [ + { + "group": "Alpha", + "pages": ["alpha/guide"], + }, + { + "group": "Beta", + "pages": ["beta/guide"], + }, + ] + }, + } + ), + ) + write(root, "docs/alpha/guide.mdx", "# Guide\n\nAlpha body.\n") + write(root, "docs/beta/guide.mdx", "# Guide\n\nBeta body.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "mintlify") + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + self.assertNotIn( + "duplicate-title", + [finding.get("kind") for finding in payload["findings"]], + ) + self.assertEqual(readiness["ready_count"], 2) + self.assertEqual(readiness["eligible_count"], 2) + self.assertEqual(readiness["percentage"], 100) + for route, label in ( + ("docs/alpha/guide.mdx", "Alpha"), + ("docs/beta/guide.mdx", "Beta"), + ): + with self.subTest(route=route): + title = rows[route]["facts"]["title"] + self.assertIs(title["passed"], True) + self.assertEqual( + title["classification"], + "distinct-map-label", + ) + self.assertEqual(title["map_labels"], [label.casefold()]) + def test_a_repository_without_a_selected_map_reports_no_percentage(self): """No selected map means no readiness percentage — an honest no-map state, never a misleading low number.""" diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 138622a..97166bf 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -212,9 +212,74 @@ def test_directory_groups_resolve_in_one_reply_with_exact_exceptions(self): }, ) map_diff = str(preview.get("map_diff", "")) - self.assertIn("role preserved research/", map_diff) + self.assertNotIn( + "+role preserved research/", + map_diff.splitlines(), + ) + self.assertIn("role preserved research/note-01.md", map_diff) + self.assertIn("role preserved research/note-03.md", map_diff) self.assertIn("role historical research/note-02.md", map_diff) + def test_directory_role_resolution_changes_only_the_advertised_routes(self): + """A directory group role applies only to the unresolved routes named + by its card, never maintained siblings or the selected map.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + write(root, "docs/alpha.md", "# Alpha\n\nFirst unresolved guide.\n") + write(root, "docs/beta.md", "# Beta\n\nSecond unresolved guide.\n") + git(root, "add", "--", "docs/alpha.md", "docs/beta.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "docs group") + + documentation = section(run_checker(root), "documentation") + group = next( + card + for card in documentation["unresolved"] + if card.get("path") == "docs/" + ) + advertised = sorted(group["routes"]) + decisions = {"docs/": {"role": "preserved"}} + + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + records = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + + self.assertEqual( + advertised, + ["docs/alpha.md", "docs/beta.md", "docs/guide.md"], + ) + self.assertEqual(applied["status"], "applied") + self.assertNotIn( + "+role preserved docs/", + preview["map_diff"].splitlines(), + ) + self.assertNotIn( + "role preserved docs/README.md", + preview["map_diff"], + ) + self.assertNotIn( + "role preserved docs/STATE.md", + preview["map_diff"], + ) + for route in advertised: + with self.subTest(route=route): + self.assertIn( + f"role preserved {route}", + preview["map_diff"], + ) + self.assertEqual(records[route]["role"], "preserved") + self.assertEqual(records["docs/README.md"]["role"], "maintained") + self.assertEqual(records["docs/STATE.md"]["role"], "maintained") + def test_exact_leave_unresolved_overrides_its_directory_group(self): """An exact leave-unresolved reply wins over a role chosen for its containing directory, without persisting unresolved as a policy role.""" From aae578c051a0a7642798e7e87beefd4d2931e8fc Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 04:14:31 -0400 Subject: [PATCH 10/19] Preserve scoped resolution and Init recovery semantics --- BENCHMARK.md | 2 +- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../docs/scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../grok/scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../docs/scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../docs/scripts/_docs_checker/resolution.py | 143 +++++++++++++----- .../scripts/_docs_checker/init_adoption.py | 18 +++ .../docs/scripts/_docs_checker/resolution.py | 143 +++++++++++++----- tests/test_resolution_treatment_contract.py | 133 ++++++++++++++++ 16 files changed, 1002 insertions(+), 260 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index d422b51..bab0595 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,156 tests — 525 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 68 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,159 tests — 528 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 71 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/copilot/scripts/_docs_checker/init_adoption.py b/adapters/copilot/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/adapters/copilot/scripts/_docs_checker/init_adoption.py +++ b/adapters/copilot/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/cursor/scripts/_docs_checker/init_adoption.py b/adapters/cursor/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/adapters/cursor/scripts/_docs_checker/init_adoption.py +++ b/adapters/cursor/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/grok/scripts/_docs_checker/init_adoption.py b/adapters/grok/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/adapters/grok/scripts/_docs_checker/init_adoption.py +++ b/adapters/grok/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/skills/docs/scripts/_docs_checker/init_adoption.py b/skills/docs/scripts/_docs_checker/init_adoption.py index 1f33e6a..ca4b037 100644 --- a/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/skills/docs/scripts/_docs_checker/init_adoption.py @@ -610,6 +610,24 @@ def _request_from_context( map_source, setup_measurements["manifest"].policy.git, ) + if map_recovery["kind"] == "archive": + projected_policy = parse_documentation_policy( + proposed_map.decode("utf-8", "strict"), + root=root, + map_route=map_path, + ) + proposed_map = replace_documentation_policy( + proposed_map.decode("utf-8", "strict"), + ( + *projected_policy.role_rules, + PolicyRule( + map_recovery["path"], + False, + "historical", + ), + ), + projected_policy.never_edit_rules, + ).encode("utf-8") map_disposition.update( { "disposition": "MIGRATED", diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 0486c0c..662b8a7 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -279,21 +279,10 @@ def _directory_route_bindings(manifest, decisions): for decision in decisions: if not decision["directory"]: continue - if decision["role"] is not None: - routes = _advertised_directory_routes( - manifest, - decision["selector"], - ) - else: - prefix = route_key(decision["selector"]) + "/" - routes = sorted( - ( - document.path - for document in manifest.documents - if route_key(document.path).startswith(prefix) - ), - key=lambda value: (value.casefold(), value), - ) + routes = _advertised_directory_routes( + manifest, + decision["selector"], + ) bindings.append( { "selector": decision["selector"] + "/", @@ -320,6 +309,15 @@ def _proposed_rules(policy, decisions, manifest): and decision["role"] == LEAVE_UNRESOLVED ) } + group_never_edit_routes = { + route_key(route) + for decision in decisions + if decision["directory"] and decision["never_edit"] is True + for route in _advertised_directory_routes( + manifest, + decision["selector"], + ) + } for decision in decisions: key = _rule_key(decision["selector"], decision["directory"]) role = decision["role"] @@ -344,30 +342,39 @@ def _proposed_rules(policy, decisions, manifest): role, ) never_edit = decision["never_edit"] - if never_edit is True: - never_edit_rules[key] = PolicyRule( + never_edit_routes = ( + _advertised_directory_routes( + manifest, decision["selector"], - decision["directory"], - NEVER_EDIT_VALUE, - ) - elif never_edit is False: - never_edit_rules.pop(key, None) - candidate_policy = replace( - policy, - never_edit_rules=tuple(never_edit_rules.values()), ) - if ( - never_edit_rule_for( - candidate_policy, - decision["selector"], + if decision["directory"] + else (decision["selector"],) + ) + if never_edit is True: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_VALUE, ) - is not None - ): - never_edit_rules[key] = PolicyRule( - decision["selector"], - decision["directory"], - NEVER_EDIT_EXCEPTION_VALUE, + elif never_edit is False: + for route in never_edit_routes: + exact_key = _rule_key(route, False) + never_edit_rules.pop(exact_key, None) + candidate_policy = replace( + policy, + never_edit_rules=tuple(never_edit_rules.values()), ) + if ( + route_key(route) in group_never_edit_routes + or never_edit_rule_for(candidate_policy, route) is not None + ): + never_edit_rules[exact_key] = PolicyRule( + route, + False, + NEVER_EDIT_EXCEPTION_VALUE, + ) return tuple(role_rules.values()), tuple(never_edit_rules.values()) @@ -494,6 +501,63 @@ def resolution_preview(root, decisions): } +def _install_if_unchanged(target, staged, expected_bytes): + """Preserve a concurrent save while installing only over expected bytes.""" + descriptor, displaced_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-original-", + suffix=".tmp", + dir=target.parent, + ) + os.close(descriptor) + displaced = Path(displaced_name) + displaced.unlink() + moved = False + try: + os.replace(target, displaced) + moved = True + if displaced.is_symlink() or not displaced.is_file(): + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + if displaced.read_bytes() != expected_bytes: + if os.path.lexists(target): + return "selected-map-drift" + os.replace(displaced, target) + moved = False + return "selected-map-drift" + try: + os.link(staged, target) + except FileExistsError: + if displaced.read_bytes() == expected_bytes: + displaced.unlink() + moved = False + return "selected-map-drift" + except OSError: + if not os.path.lexists(target): + os.replace(displaced, target) + moved = False + return "selected-map-write-failed" + if displaced.read_bytes() != expected_bytes: + if target.read_bytes() == staged.read_bytes(): + os.replace(displaced, target) + moved = False + return "selected-map-drift" + return "selected-map-write-failed" + displaced.unlink() + moved = False + return None + finally: + if moved and os.path.lexists(displaced): + try: + if not os.path.lexists(target): + os.replace(displaced, target) + elif displaced.read_bytes() == expected_bytes: + displaced.unlink() + except OSError: + pass + + def resolution_apply(root, decisions, approval): """Apply only an exact stateless preview binding recomputed from snapshot.""" root = Path(root).absolute() @@ -537,8 +601,13 @@ def resolution_apply(root, decisions, approval): os.chmod(staged, target_mode) if target.read_bytes() != plan["source_bytes"]: return _refusal("selected-map-drift", map_route=map_route) - os.replace(staged, target) - staged = None + refusal = _install_if_unchanged( + target, + staged, + plan["source_bytes"], + ) + if refusal is not None: + return _refusal(refusal, map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 97166bf..1d6a8d7 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -280,6 +280,56 @@ def test_directory_role_resolution_changes_only_the_advertised_routes(self): self.assertEqual(records["docs/README.md"]["role"], "maintained") self.assertEqual(records["docs/STATE.md"]["role"], "maintained") + def test_directory_never_edit_changes_only_the_advertised_routes(self): + """A directory card's never-edit choice protects only its unresolved + routes, never maintained siblings or the selected map.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + write(root, "docs/alpha.md", "# Alpha\n\nFirst unresolved guide.\n") + write(root, "docs/beta.md", "# Beta\n\nSecond unresolved guide.\n") + git(root, "add", "--", "docs/alpha.md", "docs/beta.md") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "docs group") + + documentation = section(run_checker(root), "documentation") + group = next( + card + for card in documentation["unresolved"] + if card.get("path") == "docs/" + ) + advertised = sorted(group["routes"]) + decisions = {"docs/": {"never_edit": True}} + + preview = resolution_preview(root, decisions) + applied = resolution_apply(root, decisions, preview["approval"]) + records = { + document["path"]: document + for document in section( + run_checker(root), + "documentation", + )["documents"] + } + + self.assertEqual(applied["status"], "applied") + self.assertNotIn( + "+never-edit docs/", + preview["map_diff"].splitlines(), + ) + for route in advertised: + with self.subTest(route=route): + self.assertIn( + f"never-edit {route}", + preview["map_diff"], + ) + self.assertIs(records[route]["never_edit"], True) + self.assertIs(records["docs/README.md"]["never_edit"], False) + self.assertIs(records["docs/README.md"]["write_eligible"], True) + self.assertIs(records["docs/STATE.md"]["never_edit"], False) + self.assertIs(records["docs/STATE.md"]["write_eligible"], True) + def test_exact_leave_unresolved_overrides_its_directory_group(self): """An exact leave-unresolved reply wins over a role chosen for its containing directory, without persisting unresolved as a policy role.""" @@ -840,6 +890,44 @@ def test_resolution_apply_refuses_drift_with_zero_writes(self): self.assertEqual(refused["writes"], 0) self.assertEqual(map_path.read_bytes(), drifted) + def test_resolution_apply_preserves_a_save_during_compare_and_install(self): + """A save after the byte comparison but at the replacement boundary is + preserved and reported as drift, never overwritten by approved bytes.""" + import _docs_checker.resolution as resolution + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = resolution.resolution_preview(root, decisions) + owner_bytes = map_path.read_bytes() + b"\nConcurrent owner save.\n" + real_replace = resolution.os.replace + injected = False + + def save_then_replace(source, destination): + nonlocal injected + if not injected: + injected = True + map_path.write_bytes(owner_bytes) + return real_replace(source, destination) + + resolution.os.replace = save_then_replace + try: + refused = resolution.resolution_apply( + root, + decisions, + preview["approval"], + ) + finally: + resolution.os.replace = real_replace + + self.assertEqual(refused["status"], "refused") + self.assertEqual(refused["reason"], "selected-map-drift") + self.assertEqual(refused["writes"], 0) + self.assertEqual(map_path.read_bytes(), owner_bytes) + def test_a_superseding_resolution_preview_invalidates_the_older_code(self): """A code bound to older decisions cannot approve a newer decision mapping; exact recomputation makes the mismatch fail closed.""" @@ -1238,6 +1326,51 @@ def test_init_policy_edits_recover_exact_non_git_and_dirty_map_bytes(self): never_edit_rule_for(policy, ".github/SUPPORT.md") ) + def test_init_recovery_archive_is_historical_not_unresolved(self): + """Init's own Markdown recovery archive is classified explicitly and + never creates a follow-up unresolved card.""" + from _docs_checker.init_adoption import adoption_apply, adoption_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + write( + root, + "docs/README.md", + "# Documentation\n\n- [Guide](guide.md)\n", + ) + write(root, "docs/guide.md", "# Guide\n\nShared guidance.\n") + write(root, ".github/SUPPORT.md", "# Support\n\nWhere to ask.\n") + + request, preview = adoption_preview( + root, + completed_at="2026-07-30T00:00:00Z", + ) + recovery = next( + item["recovery"] + for item in request["evidence"]["dispositions"] + if item["path"] == "docs/README.md" + ) + applied = adoption_apply(root, request, preview["approval"]) + documentation = section(run_checker(root), "documentation") + documents = { + document["path"]: document + for document in documentation["documents"] + } + unresolved_paths = { + route + for card in documentation["unresolved"] + for route in card.get("routes", (card["path"],)) + } + + self.assertEqual(applied["status"], "applied") + self.assertEqual(recovery["kind"], "archive") + self.assertEqual( + documents[recovery["path"]]["role"], + "historical", + ) + self.assertNotIn(recovery["path"], unresolved_paths) + def test_the_selected_map_stays_write_eligible_unless_user_never_edits_it(self): """PRODUCT.md 'Classification is three separate facts' plus owner gate 3: there are no hardcoded filename bans, so the selected map From 3a4726f2cf85cd62ad4734d2e40a9f22f4ee86f6 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 04:39:19 -0400 Subject: [PATCH 11/19] Ignore inert policy examples and validate reference links --- BENCHMARK.md | 2 +- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- .../skills/docs/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- .../copilot/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- adapters/cursor/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- adapters/grok/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- .../skills/docs/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- .../skills/docs/scripts/_docs_checker/scan.py | 60 ++++++++- .../_docs_checker/documentation_policy.py | 98 ++++++++++++++- skills/docs/scripts/_docs_checker/scan.py | 60 ++++++++- tests/test_documentation_result_contract.py | 118 ++++++++++++++++++ 16 files changed, 1176 insertions(+), 50 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index bab0595..e147bf5 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,159 tests — 528 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 71 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,161 tests — 530 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 73 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index e079ae6..be81c43 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -25,6 +25,7 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) +_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -71,6 +72,87 @@ def _invalid(reason, map_route, *, conflict_route=None): ) +def _mask_characters(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _unescaped_tick_runs(text): + runs = [] + index = 0 + while index < len(text): + if text[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and text[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(text) and text[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + return runs + + +def _policy_markdown_view(text): + """Mask fenced and inline code while preserving source byte positions.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = _FENCE_LINE.match(line) + if fence is not None: + _mask_characters(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + marker = match.group(2) + suffix = match.group(3) + if marker[0] == "~" or "`" not in suffix: + fence = (marker[0], len(marker)) + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + offset += len(raw_line) + + visible = "".join(characters) + runs = _unescaped_tick_runs(visible) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask_characters(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _policy_matches(text): + visible = _policy_markdown_view(text) + return visible, list(_POLICY_BLOCK.finditer(visible)) + + _UNSHARED_SOURCE = object() @@ -240,7 +322,8 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - starts = text.count(_POLICY_START) + visible, matches = _policy_matches(text) + starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( map_route, @@ -249,7 +332,6 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): False, **provenance, ) - matches = list(_POLICY_BLOCK.finditer(text)) if starts != 1 or len(matches) != 1: _invalid( "multiple-policy-blocks" if starts > 1 else "malformed-policy-block", @@ -258,7 +340,9 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): role_rules = {} never_edit_rules = {} - for raw_line in matches[0].group(1).splitlines(): + for raw_line in text[ + matches[0].start(1) : matches[0].end(1) + ].splitlines(): line = raw_line.strip() if not line: continue @@ -412,12 +496,12 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - matches = list(_POLICY_BLOCK.finditer(text)) + visible, matches = _policy_matches(text) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" if matches: - if len(matches) != 1 or text.count(_POLICY_START) != 1: + if len(matches) != 1 or visible.count(_POLICY_START) != 1: raise ValueError("policy source must contain one policy block") match = matches[0] desired_roles = { @@ -431,7 +515,9 @@ def replace_documentation_policy( seen_roles = set() seen_never_edit = set() retained = [] - for raw_line in match.group(1).splitlines(keepends=True): + for raw_line in text[ + match.start(1) : match.end(1) + ].splitlines(keepends=True): line = raw_line.strip() if not line: retained.append(raw_line) diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 6f52351..16f1793 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -27,6 +27,18 @@ LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?\n" + "```\n" + ), + "tilde-fence": ( + "~~~markdown\n" + "\n" + "~~~\n" + ), + "multiline-code-span": ( + "``\n" + ), + } + for name, example in sorted(inert_examples.items()): + with self.subTest(example=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + + example + + "\n\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + name, + ) + + payload, returncode = run_checker_process(root) + documentation = section(payload, "documentation") + roles = { + document["path"]: document["role"] + for document in documentation["documents"] + } + + self.assertEqual(returncode, 0) + self.assertEqual(roles["docs/README.md"], "maintained") + self.assertEqual(roles["docs/guide.md"], "maintained") + def test_directory_rules_exact_exceptions_and_deepest_rule_win(self): """Directory rules classify groups, the deepest directory rule wins, and an exact path overrides its containing directory.""" @@ -1014,6 +1076,62 @@ def test_every_unready_document_names_each_failed_fact(self): self.assertIs(lonely.get("ready"), False) self.assertIn("reachability", json.dumps(lonely.get("failed_facts"))) + def test_reference_style_internal_links_feed_readiness(self): + """Standard reference definitions and usages contribute the same link + and reachability evidence as inline Markdown links. An unused + definition is not itself a link.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide][guide]\n\n" + "[guide]: guide.md\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n" + "See [missing page][missing].\n\n" + "[missing]: missing.md\n" + "[unused]: missing-unused.md\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "reference-links", + ) + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + guide = rows["docs/guide.md"] + missing_findings = [ + finding + for finding in payload["findings"] + if finding.get("kind") == "missing-link" + and finding.get("path") == "docs/guide.md" + ] + + self.assertIs(guide["facts"]["reachability"]["passed"], True) + self.assertIs(guide["facts"]["links"]["passed"], False) + self.assertIn("missing.md", json.dumps(guide)) + self.assertEqual( + [finding.get("target") for finding in missing_findings], + ["missing.md"], + ) + self.assertNotIn("missing-unused.md", json.dumps(payload)) + def test_supported_frontmatter_title_counts_for_readiness_identity(self): """A supported scalar frontmatter title is the document's title identity when the maintained Markdown or MDX body has no literal H1.""" From 96fc156eb79cfa144052f0330e1fad09cece1793 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:00:13 -0400 Subject: [PATCH 12/19] Mask indented policy examples and flag undefined references --- BENCHMARK.md | 2 +- .../_docs_checker/documentation_policy.py | 16 +++ .../skills/docs/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ .../copilot/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ adapters/cursor/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ adapters/grok/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ .../skills/docs/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ .../skills/docs/scripts/_docs_checker/scan.py | 40 +++++-- .../_docs_checker/documentation_policy.py | 16 +++ skills/docs/scripts/_docs_checker/scan.py | 40 +++++-- tests/test_documentation_result_contract.py | 104 ++++++++++++++++++ 16 files changed, 427 insertions(+), 71 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index e147bf5..7211de3 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,161 tests — 530 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 73 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,163 tests — 532 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 75 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_policy.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_policy.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_policy.py +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py index be81c43..cac9206 100644 --- a/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -78,6 +78,18 @@ def _mask_characters(characters, start, end): characters[index] = " " +def _indent_columns(line): + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + return columns + + def _unescaped_tick_runs(text): runs = [] index = 0 @@ -117,6 +129,10 @@ def _policy_markdown_view(text): fence = None offset += len(raw_line) continue + if line.strip(" \t") and _indent_columns(line) >= 4: + _mask_characters(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue if match is not None: marker = match.group(2) suffix = match.group(3) diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 16f1793..3877322 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -189,16 +189,21 @@ def _overlaps(span, occupied): def _labeled_markdown_links(text): - """Return inline and used reference-style links in source order.""" + """Return inline, used reference, and undefined explicit links in order.""" links = [ - (match.start(), match.group(1), match.group(2)) + (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): key = _reference_label(match.group(1)) - definitions.setdefault(key, match.group(2) or match.group(3)) + target = ( + match.group(2) + if match.group(2) is not None + else match.group(3) + ) + definitions.setdefault(key, target) occupied.append(match.span()) for match in _FULL_REFERENCE_LINK.finditer(text): @@ -207,8 +212,9 @@ def _labeled_markdown_links(text): key = _reference_label(match.group(2) or match.group(1)) target = definitions.get(key) if target is None: - continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), key, True)) + else: + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) for match in _SHORTCUT_REFERENCE_LINK.finditer(text): @@ -217,12 +223,15 @@ def _labeled_markdown_links(text): target = definitions.get(_reference_label(match.group(1))) if target is None: continue - links.append((match.start(), match.group(1), target)) + links.append((match.start(), match.group(1), target, False)) occupied.append(match.span()) return [ - (label, target) - for _position, label, target in sorted(links, key=lambda row: row[0]) + (label, target, undefined) + for _position, label, target, undefined in sorted( + links, + key=lambda row: row[0], + ) ] @@ -449,9 +458,19 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target in _labeled_markdown_links(text): - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + for link_label, raw_target, undefined in _labeled_markdown_links(text): source_relative = _relative_posix(path, root) + if undefined: + checked_links += 1 + findings.append( + { + "kind": "undefined-reference", + "path": source_relative, + "target": raw_target, + } + ) + continue + raw_target = _unwrap_angle_link_destination(unquote(raw_target)) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -816,6 +835,7 @@ def shared_markdown_route(relative): "missing-link", "missing-anchor", "outside-link", + "undefined-reference", "unsupported-navigation-link", } link_failures = {} diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index d8e7869..c0ad07f 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -508,6 +508,46 @@ def test_policy_examples_in_fences_and_code_spans_are_inert(self): self.assertEqual(roles["docs/README.md"], "maintained") self.assertEqual(roles["docs/guide.md"], "maintained") + def test_policy_examples_in_indented_code_blocks_are_inert(self): + """A four-space-indented CommonMark code example cannot become policy + authority or conflict with the one live block outside the example.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + " \n\n" + "\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "indented-policy-example", + ) + + payload, returncode = run_checker_process(root) + documentation = section(payload, "documentation") + roles = { + document["path"]: document["role"] + for document in documentation["documents"] + } + + self.assertEqual(returncode, 0) + self.assertEqual(roles["docs/README.md"], "maintained") + self.assertEqual(roles["docs/guide.md"], "maintained") + def test_directory_rules_exact_exceptions_and_deepest_rule_win(self): """Directory rules classify groups, the deepest directory rule wins, and an exact path overrides its containing directory.""" @@ -1132,6 +1172,70 @@ def test_reference_style_internal_links_feed_readiness(self): ) self.assertNotIn("missing-unused.md", json.dumps(payload)) + def test_undefined_reference_link_fails_readiness(self): + """A reference-style link without a matching definition is named as + invalid link evidence instead of silently passing readiness.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\nSee [Guide][missing].\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "undefined-reference", + ) + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + guide = rows["docs/guide.md"] + failures = [ + finding + for finding in payload["findings"] + if finding.get("path") == "docs/guide.md" + and finding.get("kind") == "undefined-reference" + ] + + self.assertIs(guide["facts"]["links"]["passed"], False) + self.assertEqual(len(failures), 1) + self.assertEqual( + { + key: failures[0].get(key) + for key in ( + "kind", + "path", + "target", + "role", + "write_eligible", + ) + }, + { + "kind": "undefined-reference", + "path": "docs/guide.md", + "target": "missing", + "role": "maintained", + "write_eligible": True, + }, + ) + def test_supported_frontmatter_title_counts_for_readiness_identity(self): """A supported scalar frontmatter title is the document's title identity when the maintained Markdown or MDX body has no literal H1.""" From 0ceae14feba058ea3f67749153d359793f391f0c Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:39:18 -0400 Subject: [PATCH 13/19] Validate resolution selection, policy durability, and Mintlify labels --- BENCHMARK.md | 2 +- .../_docs_checker/documentation_policy.py | 52 +++++-- .../skills/docs/scripts/_docs_checker/scan.py | 6 +- adapters/claude/skills/docs/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- .../copilot/scripts/_docs_checker/scan.py | 6 +- adapters/copilot/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- adapters/cursor/scripts/_docs_checker/scan.py | 6 +- adapters/cursor/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- adapters/grok/scripts/_docs_checker/scan.py | 6 +- adapters/grok/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- .../skills/docs/scripts/_docs_checker/scan.py | 6 +- adapters/plugin/skills/docs/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- .../skills/docs/scripts/_docs_checker/scan.py | 6 +- .../skills/docs/scripts/check.py | 6 + .../_docs_checker/documentation_policy.py | 52 +++++-- skills/docs/scripts/_docs_checker/scan.py | 6 +- skills/docs/scripts/check.py | 6 + tests/test_documentation_result_contract.py | 142 ++++++++++++++++++ tests/test_resolution_treatment_contract.py | 56 +++++++ 24 files changed, 549 insertions(+), 99 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 7211de3..5277e11 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,163 tests — 532 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 75 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,166 tests — 535 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 78 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/adapters/claude/skills/docs/scripts/check.py b/adapters/claude/skills/docs/scripts/check.py index 5780b31..7bca499 100644 --- a/adapters/claude/skills/docs/scripts/check.py +++ b/adapters/claude/skills/docs/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_policy.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/adapters/copilot/scripts/check.py b/adapters/copilot/scripts/check.py index 5780b31..7bca499 100644 --- a/adapters/copilot/scripts/check.py +++ b/adapters/copilot/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_policy.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/adapters/cursor/scripts/check.py b/adapters/cursor/scripts/check.py index 5780b31..7bca499 100644 --- a/adapters/cursor/scripts/check.py +++ b/adapters/cursor/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_policy.py +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/adapters/grok/scripts/check.py b/adapters/grok/scripts/check.py index 5780b31..7bca499 100644 --- a/adapters/grok/scripts/check.py +++ b/adapters/grok/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/adapters/plugin/skills/docs/scripts/check.py b/adapters/plugin/skills/docs/scripts/check.py index 5780b31..7bca499 100644 --- a/adapters/plugin/skills/docs/scripts/check.py +++ b/adapters/plugin/skills/docs/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/plugins/diataxis-docs/skills/docs/scripts/check.py b/plugins/diataxis-docs/skills/docs/scripts/check.py index 5780b31..7bca499 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/check.py +++ b/plugins/diataxis-docs/skills/docs/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py index cac9206..bdf8c50 100644 --- a/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -172,7 +172,13 @@ def _policy_matches(text): _UNSHARED_SOURCE = object() -def _git_map_state(root, map_route, *, git_available=None): +def _git_map_state( + root, + map_route, + source_bytes, + *, + git_available=None, +): """Return whether Git considers the current map saved in ``HEAD``.""" if git_available is False: return "unavailable" @@ -188,31 +194,47 @@ def _git_map_state(root, map_route, *, git_available=None): return "unavailable" if repository.returncode or repository.stdout.strip() != b"true": return "unavailable" + if source_bytes is None: + return "uncommitted" try: saved = subprocess.run( - ["git", "-C", str(root), "cat-file", "-e", f"HEAD:{map_route}"], - stdout=subprocess.DEVNULL, + [ + "git", + "-C", + str(root), + "ls-tree", + "-z", + "HEAD", + "--", + map_route, + ], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) - status = subprocess.run( + working = subprocess.run( [ "git", "-C", str(root), - "status", - "--porcelain=v1", - "--untracked-files=all", - "--", - map_route, + "hash-object", + f"--path={map_route}", + "--stdin", ], + input=source_bytes, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, ) except OSError: return "unavailable" - if not saved.returncode and not status.returncode and not status.stdout: + saved_fields = saved.stdout.split(b"\t", 1)[0].split() + saved_oid = saved_fields[2] if len(saved_fields) == 3 else None + if ( + not saved.returncode + and not working.returncode + and saved_oid == working.stdout.strip() + ): return "committed" return "uncommitted" @@ -242,10 +264,17 @@ def _policy_text(source_bytes, map_route): return "" -def _provenance(root, map_route, *, git_available=None): +def _provenance( + root, + map_route, + source_bytes, + *, + git_available=None, +): git_state = _git_map_state( root, map_route, + source_bytes, git_available=git_available, ) if git_state == "unavailable": @@ -437,6 +466,7 @@ def load_documentation_policy( provenance = _provenance( root, map_route, + source_bytes, git_available=git_available, ) return parse_documentation_policy( diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 3877322..54c3046 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -888,9 +888,9 @@ def shared_markdown_route(relative): "h1s": list(h1_values.get(path, ())), "frontmatter_title": frontmatter_titles.get(path), "map_labels": sorted( - ( - set(map_link_labels.get(route, ())) - | navigation_labels.get(route, set()) + navigation_labels.get( + route, + set(map_link_labels.get(route, ())), ), key=lambda value: (value.casefold(), value), ), diff --git a/skills/docs/scripts/check.py b/skills/docs/scripts/check.py index 5780b31..7bca499 100644 --- a/skills/docs/scripts/check.py +++ b/skills/docs/scripts/check.py @@ -374,6 +374,12 @@ def main(argv=None): raise ValueError( "--resolve-preview and --resolve-apply are mutually exclusive" ) + if resolution_mode and ( + namespace.scope is not None or namespace.map is not None + ): + raise ValueError( + "resolution does not accept --scope or --map" + ) if resolution_mode and ( recovery_mode or namespace.init_discovery diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index c0ad07f..0901dbf 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -719,6 +719,82 @@ def test_an_uncommitted_policy_edit_applies_and_is_not_yet_durable(self): "not-yet-durable", ) + def test_policy_durability_compares_head_despite_hidden_index_changes(self): + """Git index hints cannot make edited working policy bytes durable; + both assume-unchanged and skip-worktree still require exact HEAD + content equality.""" + for index_flag in ("assume-unchanged", "skip-worktree"): + with self.subTest(index_flag=index_flag): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + index_flag, + ) + git( + root, + "update-index", + f"--{index_flag}", + "--", + "docs/README.md", + ) + map_path = root / "docs" / "README.md" + map_path.write_text( + map_path.read_text(encoding="utf-8").replace( + "role maintained docs/", + "role historical docs/guide.md", + ), + encoding="utf-8", + newline="\n", + ) + self.assertEqual( + git( + root, + "status", + "--porcelain=v1", + "--", + "docs/README.md", + ).stdout, + "", + ) + + documentation = section( + run_checker(root), + "documentation", + ) + guide = next( + document + for document in documentation["documents"] + if document["path"] == "docs/guide.md" + ) + + self.assertEqual(guide["role"], "historical") + self.assertEqual( + documentation["policy"]["git"], + "uncommitted", + ) + self.assertFalse(documentation["policy"]["durable"]) + self.assertEqual( + documentation["policy"]["notice"], + "not-yet-durable", + ) + def test_a_non_git_repository_applies_policy_with_no_saved_version_notice(self): """Without Git, current map bytes still decide policy and provenance honestly says there is no saved version to restore.""" @@ -1325,6 +1401,72 @@ def test_mintlify_breadcrumbs_distinguish_duplicate_title_identity(self): ) self.assertEqual(title["map_labels"], [label.casefold()]) + def test_mintlify_breadcrumbs_override_entry_link_labels(self): + """An ordinary link from the readable entry cannot contaminate the + provider breadcrumbs that already disambiguate duplicate titles.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/docs.json", + json.dumps( + { + "$schema": "https://mintlify.com/docs.json", + "navigation": { + "groups": [ + { + "group": "Alpha", + "pages": ["alpha/guide"], + }, + { + "group": "Beta", + "pages": ["beta/guide"], + }, + ] + }, + } + ), + ) + write( + root, + "docs/alpha/guide.mdx", + "# Guide\n\n" + "Alpha body.\n\n" + "[Beta guide](../beta/guide)\n", + ) + write(root, "docs/beta/guide.mdx", "# Guide\n\nBeta body.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "mintlify-entry-link", + ) + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + + self.assertEqual(readiness["ready_count"], 2) + self.assertEqual(readiness["eligible_count"], 2) + self.assertEqual(readiness["percentage"], 100) + for route, label in ( + ("docs/alpha/guide.mdx", "Alpha"), + ("docs/beta/guide.mdx", "Beta"), + ): + with self.subTest(route=route): + title = rows[route]["facts"]["title"] + self.assertIs(title["passed"], True) + self.assertEqual( + title["classification"], + "distinct-map-label", + ) + self.assertEqual(title["map_labels"], [label.casefold()]) + def test_a_repository_without_a_selected_map_reports_no_percentage(self): """No selected map means no readiness percentage — an honest no-map state, never a misleading low number.""" diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 1d6a8d7..445e552 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -861,6 +861,62 @@ def read(self, size=-1): ) self.assertEqual(after, before) + def test_resolution_cli_rejects_selection_flags_it_does_not_consume(self): + """Resolution may not accept an explicit scope or map and then apply + the default repository selection to a different map.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + write( + root, + "component/README.md", + "# Component\n\n[Note](note.md)\n", + ) + write(root, "component/note.md", "# Note\n\nBody.\n") + decisions = {"HISTORY-2024.md": {"role": "historical"}} + original = (root / "docs" / "README.md").read_bytes() + cases = ( + ("scope", ["--scope", "component"]), + ("map", ["--map", "component/README.md"]), + ( + "scope-and-map", + [ + "--scope", + "component", + "--map", + "component/README.md", + ], + ), + ) + + for name, selection in cases: + with self.subTest(selection=name): + completed = subprocess.run( + [ + sys.executable, + "-B", + str(CHECKER), + str(root), + *selection, + "--resolve-preview", + ], + input=json.dumps(decisions), + capture_output=True, + text=True, + cwd=str(ROOT), + ) + + self.assertEqual(completed.returncode, 2, completed.stderr) + self.assertEqual( + json.loads(completed.stdout)["error"], + "invalid command input", + ) + self.assertEqual( + (root / "docs" / "README.md").read_bytes(), + original, + ) + def test_resolution_apply_refuses_drift_with_zero_writes(self): """Stage 3B resolution apply compares the selected map's current bytes to its preview binding and names drift without writing.""" From cd50cfbbb2ab8f7f7361972e16d6b096b86e30eb Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:05:55 -0400 Subject: [PATCH 14/19] Mask code examples and validate HTML links --- BENCHMARK.md | 2 +- .../skills/docs/scripts/_docs_checker/scan.py | 76 ++++++++++++++- .../copilot/scripts/_docs_checker/scan.py | 76 ++++++++++++++- adapters/cursor/scripts/_docs_checker/scan.py | 76 ++++++++++++++- adapters/grok/scripts/_docs_checker/scan.py | 76 ++++++++++++++- .../skills/docs/scripts/_docs_checker/scan.py | 76 ++++++++++++++- .../skills/docs/scripts/_docs_checker/scan.py | 76 ++++++++++++++- skills/docs/scripts/_docs_checker/scan.py | 76 ++++++++++++++- tests/test_documentation_result_contract.py | 93 +++++++++++++++++++ 9 files changed, 619 insertions(+), 8 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 5277e11..9d7aa33 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,166 tests — 535 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 78 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,168 tests — 537 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 80 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 54c3046..193dd2a 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -39,6 +39,11 @@ _SHORTCUT_REFERENCE_LINK = re.compile( r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) @@ -188,12 +193,81 @@ def _overlaps(span, occupied): return any(span[0] < end and start < span[1] for start, end in occupied) +def _link_markdown_view(text): + """Mask inline and indented code before collecting rendered links.""" + characters = list(text) + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + for index in range(offset, offset + len(raw_line)): + if characters[index] not in "\r\n": + characters[index] = " " + offset += len(raw_line) + + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`": + index += 1 + continue + slashes = 0 + cursor = index - 1 + while cursor >= 0 and visible[cursor] == "\\": + slashes += 1 + cursor -= 1 + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + if slashes % 2 == 0: + runs.append((start, index, index - start)) + + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + for cursor in range(start, runs[closing][1]): + if characters[cursor] not in "\r\n": + characters[cursor] = " " + index = closing + 1 + return "".join(characters) + + def _labeled_markdown_links(text): - """Return inline, used reference, and undefined explicit links in order.""" + """Return rendered Markdown and HTML links in source order.""" + text = _link_markdown_view(text) links = [ (match.start(), match.group(1), match.group(2), False) for match in LABELED_LINK.finditer(text) ] + links.extend( + ( + match.start(), + "", + next(group for group in match.groups() if group is not None), + False, + ) + for match in _HTML_ANCHOR.finditer(text) + ) definitions = {} occupied = [match.span() for match in LABELED_LINK.finditer(text)] for match in _REFERENCE_DEFINITION.finditer(text): diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 0901dbf..134b4f7 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -1312,6 +1312,99 @@ def test_undefined_reference_link_fails_readiness(self): }, ) + def test_reference_examples_in_code_do_not_fail_readiness(self): + """Inline and indented code examples remain inert when they + demonstrate undefined reference-link syntax.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n" + "Inline example: `[label][inline-example]`.\n\n" + " [label][indented-example]\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "reference-code-examples", + ) + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + failures = [ + finding + for finding in payload["findings"] + if finding.get("path") == "docs/guide.md" + and finding.get("kind") == "undefined-reference" + ] + + self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], True) + self.assertEqual(failures, []) + + def test_raw_html_internal_links_feed_readiness(self): + """A raw HTML anchor destination contributes the same internal link + evidence as Markdown links before readiness is published.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + '# Guide\n\nMissing guide\n', + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "raw-html-link", + ) + + payload = run_checker(root) + readiness = section(payload, "readiness") + rows = {row["path"]: row for row in readiness["documents"]} + failures = [ + finding + for finding in payload["findings"] + if finding.get("path") == "docs/guide.md" + and finding.get("kind") == "missing-link" + ] + + self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], False) + self.assertEqual( + [finding.get("target") for finding in failures], + ["missing.md"], + ) + def test_supported_frontmatter_title_counts_for_readiness_identity(self): """A supported scalar frontmatter title is the document's title identity when the maintained Markdown or MDX body has no literal H1.""" From a65c2925df868e70c623753d124c0b8f4984cc92 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 07:17:10 -0400 Subject: [PATCH 15/19] Harden Markdown evidence and resolution approvals --- BENCHMARK.md | 2 +- PRODUCT.md | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../docs/scripts/_docs_checker/resolution.py | 98 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ adapters/claude/skills/docs/scripts/check.py | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../scripts/_docs_checker/resolution.py | 98 ++++- .../copilot/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ adapters/copilot/scripts/check.py | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../scripts/_docs_checker/resolution.py | 98 ++++- adapters/cursor/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ adapters/cursor/scripts/check.py | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../grok/scripts/_docs_checker/resolution.py | 98 ++++- adapters/grok/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ adapters/grok/scripts/check.py | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../docs/scripts/_docs_checker/resolution.py | 98 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ adapters/plugin/skills/docs/scripts/check.py | 6 +- docs/STATE.md | 2 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../docs/scripts/_docs_checker/resolution.py | 98 ++++- .../skills/docs/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ .../skills/docs/scripts/check.py | 6 +- .../_docs_checker/documentation_result.py | 43 ++- .../_docs_checker/rendered_markdown.py | 339 ++++++++++++++++++ .../docs/scripts/_docs_checker/resolution.py | 98 ++++- skills/docs/scripts/_docs_checker/scan.py | 168 ++------- .../scripts/_docs_checker/treatment_policy.py | 50 +++ skills/docs/scripts/check.py | 6 +- tests/test_adapters.py | 1 + tests/test_docs_checker_architecture.py | 14 +- tests/test_documentation_result_contract.py | 120 ++++++- tests/test_resolution_treatment_contract.py | 215 ++++++++++- tools/build_adapters.py | 1 + 50 files changed, 4192 insertions(+), 1097 deletions(-) create mode 100644 adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py create mode 100644 adapters/copilot/scripts/_docs_checker/rendered_markdown.py create mode 100644 adapters/cursor/scripts/_docs_checker/rendered_markdown.py create mode 100644 adapters/grok/scripts/_docs_checker/rendered_markdown.py create mode 100644 adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py create mode 100644 plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py create mode 100644 skills/docs/scripts/_docs_checker/rendered_markdown.py diff --git a/BENCHMARK.md b/BENCHMARK.md index 9d7aa33..e678b67 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,168 tests — 537 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 80 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,175 tests — 544 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 87 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/PRODUCT.md b/PRODUCT.md index 5de566b..a5e6d50 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -183,7 +183,7 @@ Consequential approval belongs at the write boundary, and there is one gate: - The engine returns a short handle such as `PLAN-3B9T`. Humans never copy raw SHA-256 fingerprints for routine work. - The short handle is **not** the security control. Immediately before writing, the engine resolves it to the full internal receipt and revalidates the repository root, evidence, approved files, and proposed result. - The model never generates, interprets, or substitutes an approval identifier. -- A code is single-use. The engine recomputes its full internal digest from current evidence and supplied decisions, so changed evidence or decisions invalidate it. Codes have no arbitrary clock or process-lifetime expiry. +- The engine states each workflow's consumption rule explicitly. Stateless `DOCS-R-` approvals are **snapshot-bound**: the engine recomputes the full internal digest from current evidence and supplied decisions, so changed evidence or decisions invalidate the code and a successful apply makes immediate reuse fail. If those exact bytes and decisions are deliberately restored, the same stateless code is current for that restored snapshot. Codes have no arbitrary clock or process-lifetime expiry. - Full hashes stay internal; the normal approval surface carries only the short code and exact proposed diff. Two identities do two jobs and may not grow into a general identity chain: a **normalized** manifest identity, so equivalent documentation is not treated as a different repository merely because Windows and Linux differ on line endings or path case; and an **exact raw-byte** binding on every approved target, so any local byte change after approval refuses the write. @@ -355,7 +355,7 @@ A treatment that moves, renames, or removes a file is the highest-consequence ca docs/how-to/README.md. Nothing is discarded without a disposition. ``` -Unresolved cards use engine-owned document and directory handles; the separate map-policy approval uses `DOCS-R-`. Doctor treatments and whole proposed change sets keep their own receipt namespaces. Codes from different workflows are unrelated, and every approval code must be single-use and unambiguous within its proposal. +Unresolved cards use engine-owned document and directory handles; the separate map-policy approval uses `DOCS-R-`. Doctor treatments and whole proposed change sets keep their own receipt namespaces. Codes from different workflows are unrelated, and every approval code must be unambiguous within its proposal. A workflow backed by durable lifecycle state may consume a code permanently; stateless `DOCS-R-` remains snapshot-bound as described above. **Init** — a read-only plan first, then the per-file diff shown above, then one short-code approval, then the applied-and-verified result. Nothing changes before that approval. @@ -369,7 +369,7 @@ The contract above is the target. The current 0.1.8 candidate does not meet all 4. ~~**Root-document recognition uses a fixed 12-name allowlist as an existence gate.**~~ **Closed by the manifest and selection steps.** Every normal nonignored Markdown document is inventoried regardless of its filename or Git staging state, so `GLOSSARY.md` and an uncommitted `TROUBLESHOOTING.md` cannot disappear. Conventional names may help select an entry from the complete inventory; they never decide whether a document exists. 5. ~~**The engine does not produce one maintained numerator and denominator.**~~ **Closed by readiness v3.** The unweighted result names ready and eligible counts and every per-document failed fact, and is shared across Map, Check, Doctor, Audit, and Init. The envelope temporarily still carries the legacy rubric-v2 percentage beside readiness v3; the beginner-interface step must remove that second presented percentage. 6. ~~**The three-field classification model is not implemented.**~~ **Closed by the classification/readiness step.** Every accounted document now carries independent role, user-owned `never-edit`, and repository status facts. Names and folders may supply hints but cannot silently assign a role. -7. **The installed command presentation does not yet consume all resolution and treatment facts.** The engine now emits unresolved document and directory-group cards, previews and exactly applies one map-policy edit through a single-use `DOCS-R-` code, reports write eligibility, and evaluates purpose-backed navigation. Final plain-English ordering, the single-percentage display, and the installed Map tree remain assigned to the beginner-interface step. +7. **The installed command presentation does not yet consume all resolution and treatment facts.** The engine now emits unresolved document and directory-group cards, previews and exactly applies one map-policy edit through a snapshot-bound `DOCS-R-` code, reports write eligibility, and evaluates purpose-backed navigation. Final plain-English ordering, the single-percentage display, and the installed Map tree remain assigned to the beginner-interface step. 8. **The shipped command surface is larger than the set above**, and absorbed verbs still exist as separate routes. 9. ~~**The shipped prompt still asserts the disproven claim.**~~ **Closed by the selection-policy step.** `skills/docs/SKILL.md`, its `references/` contracts, and the generated adapter copies now state the selection the engine keeps, and the prompt no longer asks the host to supply a boundary, map, or current-truth route. The historical `0.1.3` entry in [CHANGELOG.md](CHANGELOG.md) still records the claim as it was believed then; a dated changelog entry is not rewritten. diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/adapters/claude/skills/docs/scripts/check.py b/adapters/claude/skills/docs/scripts/check.py index 7bca499..94b632f 100644 --- a/adapters/claude/skills/docs/scripts/check.py +++ b/adapters/claude/skills/docs/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/adapters/copilot/scripts/_docs_checker/documentation_result.py b/adapters/copilot/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_result.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/adapters/copilot/scripts/_docs_checker/treatment_policy.py b/adapters/copilot/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/adapters/copilot/scripts/_docs_checker/treatment_policy.py +++ b/adapters/copilot/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/adapters/copilot/scripts/check.py b/adapters/copilot/scripts/check.py index 7bca499..94b632f 100644 --- a/adapters/copilot/scripts/check.py +++ b/adapters/copilot/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/adapters/cursor/scripts/_docs_checker/documentation_result.py b/adapters/cursor/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_result.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/adapters/cursor/scripts/_docs_checker/treatment_policy.py b/adapters/cursor/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/adapters/cursor/scripts/_docs_checker/treatment_policy.py +++ b/adapters/cursor/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/adapters/cursor/scripts/check.py b/adapters/cursor/scripts/check.py index 7bca499..94b632f 100644 --- a/adapters/cursor/scripts/check.py +++ b/adapters/cursor/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/adapters/grok/scripts/_docs_checker/documentation_result.py b/adapters/grok/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_result.py +++ b/adapters/grok/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/adapters/grok/scripts/_docs_checker/rendered_markdown.py b/adapters/grok/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/adapters/grok/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/adapters/grok/scripts/_docs_checker/treatment_policy.py b/adapters/grok/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/adapters/grok/scripts/_docs_checker/treatment_policy.py +++ b/adapters/grok/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/adapters/grok/scripts/check.py b/adapters/grok/scripts/check.py index 7bca499..94b632f 100644 --- a/adapters/grok/scripts/check.py +++ b/adapters/grok/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/adapters/plugin/skills/docs/scripts/check.py b/adapters/plugin/skills/docs/scripts/check.py index 7bca499..94b632f 100644 --- a/adapters/plugin/skills/docs/scripts/check.py +++ b/adapters/plugin/skills/docs/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/docs/STATE.md b/docs/STATE.md index 26fcf4d..f5850c8 100644 --- a/docs/STATE.md +++ b/docs/STATE.md @@ -39,7 +39,7 @@ The shared result now carries the ordered documentation ledger, independent role - One engine-owned `documentation` result publishes that manifest's ordered path ledger. Every document has an evidence-bounded role, independent user-owned `never_edit` fact, independent repository status, write eligibility, and the reason for any exclusion. The selected map and declared current-truth routes are maintained evidence; ordinary links and filename or folder hints cannot silently assign a role. Invalid or conflicting `docs:policy` input fails closed. - The selected readable entry's current working-tree `docs:policy` bytes are authoritative for the current snapshot. When a provider uses a separate machine-readable authority, policy provenance and approved writes stay bound to the entry whose bytes were parsed. The result separately reports whether those bytes match Git and are therefore durable; a non-Git repository receives the honest no-saved-version notice. Exact-file policy rules override directory rules, including explicit `never-edit` opt-outs, and rendering preserves the existing line order while touching only edited entries. - Readiness v3 is the unweighted fraction of eligible maintained documents whose reachability, internal links and anchors, title identity, and confinement all pass. It exists only for a complete measurement with a valid selected map, names every failed fact, and never consumes Trust or compares itself with stored rubric-v2 state. -- Unresolved nonignored documents receive deterministic individual or directory-group cards. A resolution previews one exact map-policy diff behind a single-use stateless `DOCS-R-` approval; preview and apply can cross a process boundary, while changed map bytes, changed supplied decisions, or reuse after a successful apply refuses without writing. Init persists approved `never-edit` choices into the map, and treatment policy keeps `never-edit`, ignored, unresolved, and non-maintained reasons separate from readiness. +- Unresolved nonignored documents receive deterministic individual or directory-group cards. A resolution previews one exact map-policy diff behind a stateless, snapshot-bound `DOCS-R-` approval; preview and apply can cross a process boundary, while changed map bytes, changed supplied decisions, or immediate reuse after a successful apply refuses without writing. If those exact bytes and decisions are deliberately restored, the same code is current for that restored snapshot. Init persists approved `never-edit` choices into the map, and treatment policy keeps `never-edit`, ignored, unresolved, and non-maintained reasons separate from readiness. - One Windows-visible behavior change came with it. Repository-relative path validation rejected a rooted path such as `/docs/README.md` on POSIX but accepted it on Windows, where a rooted path carries no drive letter. It is now rejected on both. A rooted `--map` or `--hot` argument therefore returns one clean input error on Windows instead of the previous wrong answer — a `missing-map` finding for a file that exists, or a silently discarded hot path. A rooted `--scope` was already refused and is unchanged. - Map, Check, Doctor, Audit, and Init consume one engine-owned selection. On one unchanged snapshot the default checker route, the first-contact Doctor baseline, and an initialized Doctor produce the same measured boundary and the same manifest identity, and a component request is reported as partial. Selection reads directory listings and the one repository inventory the manifest already needs, so a command inventories the repository once rather than twice. diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/plugins/diataxis-docs/skills/docs/scripts/check.py b/plugins/diataxis-docs/skills/docs/scripts/check.py index 7bca499..94b632f 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/check.py +++ b/plugins/diataxis-docs/skills/docs/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/skills/docs/scripts/_docs_checker/documentation_result.py b/skills/docs/scripts/_docs_checker/documentation_result.py index 480951c..910ec02 100644 --- a/skills/docs/scripts/_docs_checker/documentation_result.py +++ b/skills/docs/scripts/_docs_checker/documentation_result.py @@ -65,13 +65,38 @@ def _groups(manifest): ] -def _published_findings(findings, manifest): - """Suppress only cold-role reachability repairs owned by no denominator.""" +def _published_findings(findings, manifest, readiness): + """Reconcile legacy findings with engine-owned role and title facts.""" cold = { document.path for document in manifest.documents if document.role in COLD_ROLES } + disambiguated_titles = { + route_key(row["path"]) + for row in readiness.get("documents", ()) + if isinstance(row, dict) + and isinstance(row.get("path"), str) + and isinstance(row.get("facts"), dict) + and isinstance(row["facts"].get("title"), dict) + and row["facts"]["title"].get("passed") is True + and row["facts"]["title"].get("classification") == "distinct-map-label" + } + + def duplicate_title_is_disambiguated(finding): + if finding.get("kind") != "duplicate-title": + return False + routes = finding.get("paths") + return ( + isinstance(routes, (list, tuple)) + and bool(routes) + and all( + isinstance(route, str) + and route_key(route) in disambiguated_titles + for route in routes + ) + ) + published = [ finding for finding in findings @@ -79,6 +104,7 @@ def _published_findings(findings, manifest): finding.get("kind") == "unreachable" and finding.get("path") in cold ) + and not duplicate_title_is_disambiguated(finding) ] documents = { route_key(document.path): document @@ -157,14 +183,15 @@ def assemble_documentation_result(manifest, measurements, findings): documentation["groups"] = _groups(manifest) documentation["unresolved"] = unresolved_cards(manifest) complete = documentation["coverage"]["complete"] + readiness = calculate_readiness( + manifest, + measurements, + complete=complete, + ) return { "documentation": documentation, - "readiness": calculate_readiness( - manifest, - measurements, - complete=complete, - ), - "findings": _published_findings(findings, manifest), + "readiness": readiness, + "findings": _published_findings(findings, manifest, readiness), } diff --git a/skills/docs/scripts/_docs_checker/rendered_markdown.py b/skills/docs/scripts/_docs_checker/rendered_markdown.py new file mode 100644 index 0000000..4c13eb9 --- /dev/null +++ b/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -0,0 +1,339 @@ +"""Rendered Markdown link evidence shared by scanning and treatment policy.""" + +from __future__ import annotations + +import re +import unicodedata + + +_REFERENCE_DEFINITION = re.compile( + r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" + r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", + re.M, +) +_FULL_REFERENCE_LINK = re.compile( + r"(?]*?\bhref[ \t]*=[ \t]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", + re.I, +) +_CURRENT_MARKER = re.compile( + r"[ \t]*[ \t]*(?:\r?\n|$)" +) + + +def _escaped(text, index): + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _mask(characters, start, end): + for index in range(start, end): + if characters[index] not in "\r\n": + characters[index] = " " + + +def _block_code_masked_view(text): + """Mask fenced and indented code while preserving offsets.""" + characters = list(text) + fence = None + offset = 0 + for raw_line in text.splitlines(keepends=True): + line = raw_line.rstrip("\r\n") + match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + if fence is not None: + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + if match is not None: + fence = (match.group(2)[0], len(match.group(2))) + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + continue + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if line.strip(" \t") and columns >= 4: + _mask(characters, offset, offset + len(raw_line)) + offset += len(raw_line) + return "".join(characters) + + +def _code_masked_view(text): + """Also mask inline code while preserving offsets.""" + characters = list(_block_code_masked_view(text)) + visible = "".join(characters) + runs = [] + index = 0 + while index < len(visible): + if visible[index] != "`" or _escaped(visible, index): + index += 1 + continue + start = index + while index < len(visible) and visible[index] == "`": + index += 1 + runs.append((start, index, index - start)) + index = 0 + while index < len(runs): + start, _end, length = runs[index] + closing = next( + ( + candidate + for candidate in range(index + 1, len(runs)) + if runs[candidate][2] == length + ), + None, + ) + if closing is None: + index += 1 + continue + _mask(characters, start, runs[closing][1]) + index = closing + 1 + return "".join(characters) + + +def _comments_masked_view(text): + """Mask HTML comments without changing other Markdown syntax.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("", start + 4) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _rendered_view(text): + """Mask every non-rendered surface while preserving source offsets.""" + return _comments_masked_view(_code_masked_view(text)) + + +def _reference_label(value): + return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + + +def _overlaps(span, occupied): + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def _closing_bracket(text, start): + index = start + while index < len(text): + if text[index] == "]" and not _escaped(text, index): + return index + if text[index] in "\r\n": + return None + index += 1 + return None + + +def _inline_link_at(text, start): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + return None + index = label_end + 2 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text): + return None + if text[index] == "<": + destination_start = index + 1 + index += 1 + while index < len(text) and not ( + text[index] == ">" and not _escaped(text, index) + ): + if text[index] in "\r\n": + return None + index += 1 + if index >= len(text): + return None + destination = text[destination_start:index] + index += 1 + else: + destination_start = index + depth = 0 + while index < len(text): + character = text[index] + if character == "\\": + index += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + break + depth -= 1 + elif character in " \t\r\n" and depth == 0: + break + index += 1 + destination = text[destination_start:index] + whitespace = index + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + index += 1 + while index < len(text) and not ( + text[index] == quote and not _escaped(text, index) + ): + index += 1 + if index >= len(text): + return None + index += 1 + while index < len(text) and text[index] in " \t\r\n": + index += 1 + elif index < len(text) and text[index] == "(" and index > whitespace: + depth = 1 + index += 1 + while index < len(text) and depth: + if text[index] == "\\": + index += 2 + continue + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + index += 1 + if depth: + return None + while index < len(text) and text[index] in " \t\r\n": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return { + "start": start, + "end": index + 1, + "label": text[start + 1 : label_end], + "target": destination, + "undefined": False, + "kind": "inline", + } + + +def rendered_markdown_evidence(text): + """Return live rendered links and exact current-truth link markers.""" + code_view = _code_masked_view(text) + view = _rendered_view(text) + metadata_view = _comments_masked_view(_block_code_masked_view(text)) + links = [] + occupied = [] + index = 0 + while index < len(view): + start = view.find("[", index) + if start < 0: + break + link = _inline_link_at(view, start) + if link is None: + index = start + 1 + continue + links.append(link) + occupied.append((link["start"], link["end"])) + index = link["end"] + + for match in _HTML_ANCHOR.finditer(view): + if _overlaps(match.span(), occupied): + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": "", + "target": next(group for group in match.groups() if group is not None), + "undefined": False, + "kind": "html", + } + ) + occupied.append(match.span()) + + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + for match in _FULL_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + key = _reference_label(match.group(2) or match.group(1)) + target = definitions.get(key) + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + ) + occupied.append(match.span()) + for match in _SHORTCUT_REFERENCE_LINK.finditer(view): + if _overlaps(match.span(), occupied): + continue + target = definitions.get(_reference_label(match.group(1))) + if target is None: + continue + links.append( + { + "start": match.start(), + "end": match.end(), + "label": match.group(1), + "target": target, + "undefined": False, + "kind": "reference", + } + ) + occupied.append(match.span()) + + links.sort(key=lambda item: item["start"]) + current = [] + for link in links: + marker = _CURRENT_MARKER.match(code_view, link["end"]) + if marker is not None: + current.append( + { + "target": link["target"], + "marker": marker.group(1), + } + ) + return { + "text": view, + "metadata_text": metadata_view, + "links": links, + "current": current, + } + + +__all__ = ("rendered_markdown_evidence",) diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 662b8a7..5f42b92 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -35,6 +35,7 @@ LEAVE_UNRESOLVED = "leave unresolved" RESOLUTION_CHOICES = (*POLICY_ROLES, LEAVE_UNRESOLVED) MAX_RESOLUTION_REQUEST_BYTES = 1024 * 1024 +MAX_RESOLUTION_JSON_DEPTH = 128 _GROUP_MINIMUM = 3 _RESOLUTION_INPUT_ERROR = ( "resolution decisions must be one JSON object on standard input" @@ -68,8 +69,17 @@ def strict_object(pairs): value, object_pairs_hook=strict_object, ) - except (UnicodeError, ValueError) as exc: + except (UnicodeError, ValueError, RecursionError, OverflowError) as exc: raise ValueError(_RESOLUTION_INPUT_ERROR) from exc + stack = [(decisions, 1)] + while stack: + current, depth = stack.pop() + if depth > MAX_RESOLUTION_JSON_DEPTH: + raise ValueError(_RESOLUTION_INPUT_ERROR) + if isinstance(current, Mapping): + stack.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + stack.extend((item, depth + 1) for item in current) if not isinstance(decisions, Mapping): raise ValueError(_RESOLUTION_INPUT_ERROR) return decisions @@ -486,9 +496,69 @@ def _resolution_plan(root, decisions): "approval": approval, "source_bytes": source_bytes, "proposed_bytes": proposed_text.encode("utf-8"), + "decisions": normalized, + "directory_routes": binding["directory_routes"], + "corpus_directories": [ + { + "selector": decision["selector"], + "routes": sorted( + ( + document.path + for document in manifest.documents + if ( + document.repository_status + != "ignored or excluded" + and route_key( + PurePosixPath(document.path).parent.as_posix() + ) + == route_key(decision["selector"]) + ) + ), + key=lambda value: (value.casefold(), value), + ), + } + for decision in normalized + if decision["directory"] + ], } +def _selected_corpus_is_current(root, plan): + """Confirm that every approved selector still names the same document set.""" + try: + inventory = repository_document_inventory( + root, + ".", + include_navigation=True, + ) + except (OSError, UnicodeError, ValueError): + return False + if inventory is None: + return False + present = { + route_key(route) + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + } + for decision in plan["decisions"]: + if not decision["directory"] and route_key(decision["selector"]) not in present: + return False + for binding in plan["corpus_directories"]: + selector_key = route_key(binding["selector"]) + current = sorted( + ( + route + for bucket in ("tracked", "untracked") + for route in inventory[bucket] + if route_key(PurePosixPath(route).parent.as_posix()) == selector_key + ), + key=lambda value: (value.casefold(), value), + ) + if current != binding["routes"]: + return False + return True + + def resolution_preview(root, decisions): """Preview one exact current-map policy edit and write nothing.""" plan = _resolution_plan(root, decisions) @@ -608,6 +678,32 @@ def resolution_apply(root, decisions, approval): ) if refusal is not None: return _refusal(refusal, map_route=map_route) + if not _selected_corpus_is_current(root, plan): + descriptor, rollback_name = tempfile.mkstemp( + prefix=f".{target.name}.docs-resolution-rollback-", + suffix=".tmp", + dir=target.parent, + ) + rollback = Path(rollback_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(plan["source_bytes"]) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(rollback, target_mode) + rollback_refusal = _install_if_unchanged( + target, + rollback, + plan["proposed_bytes"], + ) + finally: + try: + rollback.unlink() + except OSError: + pass + if rollback_refusal is not None: + return _refusal(rollback_refusal, map_route=map_route) + return _refusal("selected-corpus-drift", map_route=map_route) except (OSError, ValueError): return _refusal("selected-map-write-failed", map_route=map_route) finally: diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 193dd2a..f9df980 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -10,6 +10,7 @@ from .health import PROVISIONAL_TARGET_BYTES from .identity import finding_fingerprint, finding_id, slug from .navigation import resolve_navigation_link +from .rendered_markdown import rendered_markdown_evidence from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -25,32 +26,16 @@ ) +# Compatibility exports for hosts that import the historical scanner symbols. +# Evidence collection below is owned by ``rendered_markdown_evidence``. LINK = re.compile(r"\[[^\]]*\]\(([^)]*)\)") -LABELED_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)") -_REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" - r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" - r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", +CURRENT_ROUTE_LINK = re.compile( + r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" - r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", - re.I, -) HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.M) H1 = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.M) H2 = re.compile(r"^##\s+(.+?)\s*#*\s*$", re.M) -CURRENT_ROUTE_LINK = re.compile( - r"\[[^\]]*\]\(([^)]*)\)[ \t]*[ \t]*$", - re.M, -) SOURCES_LINE = re.compile(r"^Sources:[ \t]*(.+)$", re.M) BACKTICK_ROUTE = re.compile(r"`([^`]+)`") @@ -185,127 +170,11 @@ def _unwrap_angle_link_destination(target): return target -def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() - - -def _overlaps(span, occupied): - return any(span[0] < end and start < span[1] for start, end in occupied) - - -def _link_markdown_view(text): - """Mask inline and indented code before collecting rendered links.""" - characters = list(text) - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - if line.strip(" \t") and columns >= 4: - for index in range(offset, offset + len(raw_line)): - if characters[index] not in "\r\n": - characters[index] = " " - offset += len(raw_line) - - visible = "".join(characters) - runs = [] - index = 0 - while index < len(visible): - if visible[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and visible[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(visible) and visible[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - for cursor in range(start, runs[closing][1]): - if characters[cursor] not in "\r\n": - characters[cursor] = " " - index = closing + 1 - return "".join(characters) - - def _labeled_markdown_links(text): """Return rendered Markdown and HTML links in source order.""" - text = _link_markdown_view(text) - links = [ - (match.start(), match.group(1), match.group(2), False) - for match in LABELED_LINK.finditer(text) - ] - links.extend( - ( - match.start(), - "", - next(group for group in match.groups() if group is not None), - False, - ) - for match in _HTML_ANCHOR.finditer(text) - ) - definitions = {} - occupied = [match.span() for match in LABELED_LINK.finditer(text)] - for match in _REFERENCE_DEFINITION.finditer(text): - key = _reference_label(match.group(1)) - target = ( - match.group(2) - if match.group(2) is not None - else match.group(3) - ) - definitions.setdefault(key, target) - occupied.append(match.span()) - - for match in _FULL_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - if target is None: - links.append((match.start(), match.group(1), key, True)) - else: - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - - for match in _SHORTCUT_REFERENCE_LINK.finditer(text): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append((match.start(), match.group(1), target, False)) - occupied.append(match.span()) - return [ - (label, target, undefined) - for _position, label, target, undefined in sorted( - links, - key=lambda row: row[0], - ) + (link["label"], link["target"], link["undefined"]) + for link in rendered_markdown_evidence(text)["links"] ] @@ -484,6 +353,7 @@ def shared_markdown_route(relative): h1_values = {} frontmatter_titles = {} texts = {} + markdown_evidence = {} map_source_bytes = None for path in files: if path == mapfile: @@ -498,7 +368,9 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - text = strip_fences(raw_text) + evidence = rendered_markdown_evidence(raw_text) + markdown_evidence[path] = evidence + text = evidence["text"] texts[path] = text headings = HEADING.findall(text) anchors[path] = {slug(heading) for heading in headings} @@ -532,7 +404,10 @@ def shared_markdown_route(relative): for path in files: links[path] = [] text = texts[path] - for link_label, raw_target, undefined in _labeled_markdown_links(text): + for link in markdown_evidence[path]["links"]: + link_label = link["label"] + raw_target = link["target"] + undefined = link["undefined"] source_relative = _relative_posix(path, root) if undefined: checked_links += 1 @@ -724,7 +599,9 @@ def shared_markdown_route(relative): if _relative_posix(path, root) in navigation.get("navigated_pages", ()) ) elif map_exists: - for raw_target, marker in CURRENT_ROUTE_LINK.findall(texts[mapfile]): + for current in markdown_evidence[mapfile]["current"]: + raw_target = current["target"] + marker = current["marker"] target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): continue @@ -841,9 +718,12 @@ def shared_markdown_route(relative): path = selected_paths[relative] if not path.is_file(): continue - text = texts.get(path) - if text is None: - text = strip_fences(path.read_text(encoding="utf-8", errors="replace")) + evidence = markdown_evidence.get(path) + if evidence is None: + evidence = rendered_markdown_evidence( + path.read_text(encoding="utf-8", errors="replace") + ) + text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): for raw_route in BACKTICK_ROUTE.findall(source_group): # A ``Sources:`` line is prose a human wrote. One route that diff --git a/skills/docs/scripts/_docs_checker/treatment_policy.py b/skills/docs/scripts/_docs_checker/treatment_policy.py index a82d714..a8a5e51 100644 --- a/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -4,7 +4,11 @@ import hashlib import json +import posixpath import unicodedata +from urllib.parse import unquote, urlsplit + +from .rendered_markdown import rendered_markdown_evidence _SENSITIVE_FILENAMES = frozenset( @@ -149,6 +153,29 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } + wording_links = rendered_markdown_evidence(wording)["links"] + if not wording_links: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + if len(wording_links) != 1: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } + wording_link = wording_links[0] + if wording_link["undefined"]: + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "exact-live-link-required", + } documents = _documents(manifest) source_document = documents.get(_route_key(source)) target_document = documents.get(_route_key(target)) @@ -190,6 +217,29 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result + raw_destination = unquote(wording_link["target"]) + parsed = urlsplit(raw_destination) + if parsed.scheme or parsed.netloc: + resolved_target = None + else: + destination_path = parsed.path.replace("\\", "/") + if destination_path.startswith("/"): + resolved_target = None + else: + source_directory = posixpath.dirname(source_document.path) + resolved_target = posixpath.normpath( + posixpath.join(source_directory, destination_path) + ) + if ( + resolved_target is None + or _route_key(resolved_target) != _route_key(target_document.path) + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-destination-does-not-match-target", + } return { **details, "allowed": True, diff --git a/skills/docs/scripts/check.py b/skills/docs/scripts/check.py index 7bca499..94b632f 100644 --- a/skills/docs/scripts/check.py +++ b/skills/docs/scripts/check.py @@ -375,10 +375,12 @@ def main(argv=None): "--resolve-preview and --resolve-apply are mutually exclusive" ) if resolution_mode and ( - namespace.scope is not None or namespace.map is not None + namespace.scope is not None + or namespace.map is not None + or namespace.hot is not None ): raise ValueError( - "resolution does not accept --scope or --map" + "resolution does not accept --scope, --map, or --hot" ) if resolution_mode and ( recovery_mode diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 678addd..b113d25 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -121,6 +121,7 @@ def test_canonical_checker_registry_covers_package_and_generated_bundles(self): "evidence.py", "discovery_io.py", "discovery.py", + "rendered_markdown.py", "scan.py", "measurement.py", "documentation_policy.py", diff --git a/tests/test_docs_checker_architecture.py b/tests/test_docs_checker_architecture.py index bf0b14d..1245f7c 100644 --- a/tests/test_docs_checker_architecture.py +++ b/tests/test_docs_checker_architecture.py @@ -26,6 +26,7 @@ "evidence", "discovery_io", "discovery", + "rendered_markdown", "scan", "identity", "memory", @@ -250,8 +251,17 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): }, ) self.assertLessEqual(graph["health"], {"identity", "paths"}) + self.assertEqual(graph["rendered_markdown"], set()) self.assertLessEqual( - graph["scan"], {"formats", "paths", "identity", "health", "navigation"} + graph["scan"], + { + "formats", + "paths", + "identity", + "health", + "navigation", + "rendered_markdown", + }, ) self.assertLessEqual(graph["memory"], {"formats", "paths", "identity"}) self.assertEqual( @@ -305,7 +315,7 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): "selection", }, ) - self.assertEqual(graph["treatment_policy"], set()) + self.assertEqual(graph["treatment_policy"], {"rendered_markdown"}) # Selection owns the boundary policy and nothing else: it reads the # documentation-root and format policies, confines routes, and builds # the manifest. It must never reach into discovery, lifecycle, or diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 134b4f7..e7744aa 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -1405,6 +1405,112 @@ def test_raw_html_internal_links_feed_readiness(self): ["missing.md"], ) + def test_commented_and_escaped_link_syntax_is_not_rendered_evidence(self): + """HTML comments and escaped Markdown remain inert for links, + reachability, headings, and duplicate-title identity.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Alpha guide](a.md)\n" + "- [Beta guide](b.md)\n" + "\n\n" + "\n", + ) + write( + root, + "docs/a.md", + "\n" + "# Guide \n\n" + r"\[Escaped guide](missing-escaped.md)" + "\n", + ) + write( + root, + "docs/b.md", + "# Guide \n\nBody.\n", + ) + write(root, "docs/hidden.md", "# Hidden guide\n\nBody.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "inert") + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + a_findings = [ + finding + for finding in payload["findings"] + if finding.get("path") == "docs/a.md" + ] + + self.assertIs(rows["docs/a.md"]["facts"]["links"]["passed"], True) + self.assertNotIn("missing-commented.md", json.dumps(a_findings)) + self.assertNotIn("missing-html.md", json.dumps(a_findings)) + self.assertNotIn("missing-escaped.md", json.dumps(a_findings)) + for path in ("docs/a.md", "docs/b.md"): + self.assertIs(rows[path]["facts"]["title"]["passed"], True) + self.assertIn( + "distinct-map-label", + json.dumps(rows[path]["facts"]["title"]), + ) + self.assertIs( + rows["docs/hidden.md"]["facts"]["reachability"]["passed"], + False, + ) + + def test_valid_markdown_destination_forms_feed_readiness(self): + """Optional titles, angle destinations with spaces, and balanced + parentheses are parsed as their exact internal destination.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + '- [Titled guide](guide.md "Read guide")\n' + "- [Spaced guide]()\n" + "- [Parenthesized guide](guide(1).md)\n\n" + "\n", + ) + write(root, "docs/guide.md", "# Titled guide\n\nBody.\n") + write(root, "docs/guide with spaces.md", "# Spaced guide\n\nBody.\n") + write(root, "docs/guide(1).md", "# Parenthesized guide\n\nBody.\n") + git(root, "add", "--", "docs") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "destinations") + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + for path in ( + "docs/guide.md", + "docs/guide with spaces.md", + "docs/guide(1).md", + ): + with self.subTest(path=path): + self.assertIs(rows[path]["facts"]["reachability"]["passed"], True) + self.assertEqual( + [ + finding + for finding in payload["findings"] + if finding.get("kind") == "missing-link" + ], + [], + ) + def test_supported_frontmatter_title_counts_for_readiness_identity(self): """A supported scalar frontmatter title is the document's title identity when the maintained Markdown or MDX body has no literal H1.""" @@ -1696,7 +1802,8 @@ def test_duplicate_titles_need_distinct_map_labels_to_pass(self): git(root, "add", "--", "docs") git(root, "commit", "--quiet", "--no-gpg-sign", "-m", name) - readiness = section(run_checker(root), "readiness") + payload = run_checker(root) + readiness = section(payload, "readiness") rows = {row["path"]: row for row in readiness["documents"]} for path in ("docs/a.md", "docs/b.md"): @@ -1706,6 +1813,17 @@ def test_duplicate_titles_need_distinct_map_labels_to_pass(self): self.assertIn( "ambiguous-title", json.dumps(rows[path]) ) + if expect_ready: + self.assertEqual( + [ + finding + for finding in payload["findings"] + if finding.get("kind") == "duplicate-title" + and set(finding.get("paths", ())) + == {"docs/a.md", "docs/b.md"} + ], + [], + ) def test_trust_stays_out_of_readiness(self): """The selected map, reader entry, and current-truth declarations stay diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 445e552..971fd99 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -475,10 +475,10 @@ def test_leave_unresolved_alone_proposes_no_policy_write(self): self.assertIsNone(preview["approval"]) self.assertEqual(map_path.read_bytes(), before) - def test_resolution_apply_is_exact_single_use_and_visible_on_a_fresh_run(self): - """Stage 3B resolution apply changes only the selected map, consumes - its exact code once, and leaves the working-tree policy immediately - authoritative with honest uncommitted provenance.""" + def test_resolution_apply_is_snapshot_bound_and_visible_on_a_fresh_run(self): + """Stage 3B resolution apply changes only the selected map, invalidates + its exact code for the changed snapshot, and leaves the working-tree + policy immediately authoritative with honest uncommitted provenance.""" from _docs_checker.resolution import resolution_apply, resolution_preview with tempfile.TemporaryDirectory() as td: @@ -524,6 +524,47 @@ def test_resolution_apply_is_exact_single_use_and_visible_on_a_fresh_run(self): self.assertEqual(reused["writes"], 0) self.assertTrue(str(reused.get("reason", "")).strip()) + def test_resolution_approval_contract_names_snapshot_bound_semantics(self): + """The public contract must not promise durable one-time consumption + from a stateless digest: exact restored evidence makes the same + approval current again, while changed evidence still refuses.""" + product = (ROOT / "PRODUCT.md").read_text(encoding="utf-8") + state = (ROOT / "docs" / "STATE.md").read_text(encoding="utf-8") + + self.assertIn("snapshot-bound", product) + self.assertIn("snapshot-bound", state) + for document in (product, state): + docs_r_lines = [ + line + for line in document.splitlines() + if "DOCS-R-" in line or "A resolution previews" in line + ] + self.assertTrue(docs_r_lines) + self.assertNotIn("single-use", "\n".join(docs_r_lines)) + + def test_resolution_code_is_current_again_for_exactly_restored_snapshot(self): + """Stateless approval follows evidence: restoring the exact preview + bytes and decisions restores the same code instead of consulting an + impossible process-independent consumption ledger.""" + from _docs_checker.resolution import resolution_apply, resolution_preview + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + original = map_path.read_bytes() + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = resolution_preview(root, decisions) + + first = resolution_apply(root, decisions, preview["approval"]) + self.assertEqual(first["status"], "applied") + map_path.write_bytes(original) + restored = resolution_apply(root, decisions, preview["approval"]) + + self.assertEqual(restored["status"], "applied") + self.assertEqual(restored["writes"], 1) + def test_mintlify_policy_provenance_follows_the_entry_bytes_it_parses(self): """A provider authority and its readable entry are different routes: resolution writes the entry, and the next read reports that same route @@ -861,6 +902,53 @@ def read(self, size=-1): ) self.assertEqual(after, before) + def test_checker_rejects_excessively_nested_resolution_json_safely(self): + """A below-capacity but deeply nested request returns the documented + input envelope, never a RecursionError traceback or repository write.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + before = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + payload = ( + '{"HISTORY-2024.md":{"role":"historical","padding":' + + ("[" * 2000) + + "0" + + ("]" * 2000) + + "}}" + ) + + completed = subprocess.run( + [ + sys.executable, + "-B", + str(CHECKER), + str(root), + "--resolve-preview", + ], + input=payload, + capture_output=True, + text=True, + cwd=str(ROOT), + ) + after = { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + + self.assertEqual(completed.returncode, 2, completed.stderr) + self.assertEqual(completed.stderr, "") + self.assertEqual( + json.loads(completed.stdout)["error"], + "resolution decisions must be one JSON object on standard input", + ) + self.assertEqual(after, before) + def test_resolution_cli_rejects_selection_flags_it_does_not_consume(self): """Resolution may not accept an explicit scope or map and then apply the default repository selection to a different map.""" @@ -879,6 +967,7 @@ def test_resolution_cli_rejects_selection_flags_it_does_not_consume(self): cases = ( ("scope", ["--scope", "component"]), ("map", ["--map", "component/README.md"]), + ("hot", ["--hot", "component/note.md"]), ( "scope-and-map", [ @@ -984,6 +1073,56 @@ def save_then_replace(source, destination): self.assertEqual(refused["writes"], 0) self.assertEqual(map_path.read_bytes(), owner_bytes) + def test_resolution_apply_rolls_back_when_selected_corpus_drifts(self): + """If a selected document disappears after preview revalidation but + before installation, apply restores the exact map bytes and refuses.""" + import _docs_checker.resolution as resolution + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + hinted_fixture(root) + map_path = root / "docs" / "README.md" + selected = root / "HISTORY-2024.md" + decisions = {"HISTORY-2024.md": {"role": "historical"}} + preview = resolution.resolution_preview(root, decisions) + original_map = map_path.read_bytes() + real_install = resolution._install_if_unchanged + injected = False + + def delete_then_install(target, staged, expected_bytes, *args, **kwargs): + nonlocal injected + if not injected: + injected = True + selected.unlink() + return real_install( + target, + staged, + expected_bytes, + *args, + **kwargs, + ) + + resolution._install_if_unchanged = delete_then_install + try: + refused = resolution.resolution_apply( + root, + decisions, + preview["approval"], + ) + finally: + resolution._install_if_unchanged = real_install + + self.assertEqual(refused["status"], "refused") + self.assertIn( + refused["reason"], + {"approval-code-not-current", "selected-corpus-drift"}, + ) + self.assertEqual(refused["writes"], 0) + self.assertEqual(map_path.read_bytes(), original_map) + self.assertFalse(selected.exists()) + self.assertNotEqual(run_checker(root).get("classification"), "invalid-policy") + def test_a_superseding_resolution_preview_invalidates_the_older_code(self): """A code bound to older decisions cannot approve a newer decision mapping; exact recomputation makes the mismatch fail closed.""" @@ -1594,6 +1733,74 @@ def test_an_existing_broken_confined_link_repair_stays_eligible(self): "existing-broken-confined-link-repair", ) + def test_link_proposal_wording_names_one_live_exact_destination(self): + """Treatment accepts only wording containing one rendered link whose + confined destination resolves to the declared target; prose, comments, + and links to another route cannot approve a write.""" + from _docs_checker.treatment_policy import evaluate_link_proposal + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + write( + root, + "docs/archive/old/2019 review.md", + "# 2019 review with spaces\n\nRecord.\n", + ) + write( + root, + "docs/archive/old/2019(review).md", + "# 2019 review with parentheses\n\nRecord.\n", + ) + git(root, "add", "--", "docs/archive/old") + git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "wording") + manifest = build_manifest(root) + common = { + "manifest": manifest, + "source": "docs/README.md", + "target": "docs/archive/old/2019-review.md", + "purpose": "Readers auditing the decision need the record.", + } + + refused_wording = ( + "The review is archived.", + "", + "[Different review](archive/2023-review.md)", + ) + for wording in refused_wording: + with self.subTest(wording=wording): + result = evaluate_link_proposal(wording=wording, **common) + self.assertIs(result["allowed"], False) + self.assertIn( + result["reason"], + { + "exact-live-link-required", + "link-destination-does-not-match-target", + }, + ) + + valid_cases = ( + ( + "docs/archive/old/2019 review.md", + '- [Spaced review]( "Record")', + ), + ( + "docs/archive/old/2019(review).md", + "[Parenthesized review](archive/old/2019(review).md)", + ), + ) + for target, wording in valid_cases: + with self.subTest(target=target): + result = evaluate_link_proposal( + manifest, + source="docs/README.md", + target=target, + purpose="Readers auditing the decision need the record.", + wording=wording, + ) + self.assertIs(result["allowed"], True, result) + def test_unresolved_target_refusal_carries_its_classification_card(self): """An unresolved target is not a bare refusal: one attached card supplies the path, reason, choices, handle, and copy-paste reply needed diff --git a/tools/build_adapters.py b/tools/build_adapters.py index 29e7443..d8f4eea 100644 --- a/tools/build_adapters.py +++ b/tools/build_adapters.py @@ -42,6 +42,7 @@ "scripts/_docs_checker/evidence.py", "scripts/_docs_checker/discovery_io.py", "scripts/_docs_checker/discovery.py", + "scripts/_docs_checker/rendered_markdown.py", "scripts/_docs_checker/scan.py", "scripts/_docs_checker/measurement.py", "scripts/_docs_checker/documentation_policy.py", From e05002db24a32b0b448fd1612c920faeeac26fe4 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 07:53:20 -0400 Subject: [PATCH 16/19] Make rendered evidence syntax-complete --- AGENTS.md | 1 + BENCHMARK.md | 2 +- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- .../_docs_checker/documentation_policy.py | 3 +- .../_docs_checker/rendered_markdown.py | 41 +++- tests/test_docs_checker_architecture.py | 5 +- tests/test_documentation_result_contract.py | 203 ++++++++++++++++++ 18 files changed, 475 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c1bc22a..b410939 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,4 +69,5 @@ Universal rules. Obey host precedence; among recognized instruction files, the m - Documentation starts at `docs/README.md`; current verified status is `docs/STATE.md`. - Canonical source lives under `skills/docs/`; do not hand-edit generated adapters. - Run the repository checker before claiming completion. +- Use proactive proof, not reactive patching: before any behavior change, identify the adjacent input, state, and output classes, then test the invariant at the changed boundary instead of waiting for review to reveal siblings. For Markdown, MDX, and other multi-syntax evidence formats, define the supported syntax family before implementation, add table-driven invariant tests across every relevant form with live controls, and audit every consumer of the shared boundary. If review exposes a missed form, expand the family-level proof instead of patching only that example. - During the unreleased 0.1.8 PR train, classify a review finding as blocking only when it affects my daily-driver behavior, safety, privacy, data integrity, ability to understand or use the product, or an explicit PR contract. Park hypothetical-user polish and precision-only improvements instead of interrupting the current PR. diff --git a/BENCHMARK.md b/BENCHMARK.md index e678b67..5e3fcc1 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,175 tests — 544 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 87 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,179 tests — 548 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 91 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_policy.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_policy.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_policy.py +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/adapters/grok/scripts/_docs_checker/rendered_markdown.py b/adapters/grok/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/adapters/grok/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/grok/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py index bdf8c50..0e7e235 100644 --- a/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -15,6 +15,7 @@ from pathlib import Path from .paths import normalize_repo_relative, safe_path +from .rendered_markdown import mask_jsx_comments POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -161,7 +162,7 @@ def _policy_markdown_view(text): continue _mask_characters(characters, start, runs[closing][1]) index = closing + 1 - return "".join(characters) + return mask_jsx_comments("".join(characters)) def _policy_matches(text): diff --git a/skills/docs/scripts/_docs_checker/rendered_markdown.py b/skills/docs/scripts/_docs_checker/rendered_markdown.py index 4c13eb9..8b33928 100644 --- a/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -43,6 +43,33 @@ def _mask(characters, start, end): characters[index] = " " +def mask_jsx_comments(text): + """Mask non-rendered MDX ``{/* ... */}`` comments with stable offsets.""" + characters = list(text) + visible = "".join(characters) + index = 0 + while index < len(visible): + start = visible.find("{/*", index) + if start < 0: + break + end = visible.find("*/}", start + 3) + end = len(visible) if end < 0 else end + 3 + _mask(characters, start, end) + index = end + return "".join(characters) + + +def _blockquote_content(line): + """Return a line after its CommonMark blockquote container prefixes.""" + index = 0 + while index < len(line): + match = re.match(r" {0,3}>[ \t]?", line[index:]) + if match is None: + break + index += match.end() + return line[index:] + + def _block_code_masked_view(text): """Mask fenced and indented code while preserving offsets.""" characters = list(text) @@ -50,7 +77,11 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - match = re.match(r"^( {0,3})(`{3,}|~{3,})(.*)$", line) + container_line = _blockquote_content(line) + match = re.match( + r"^( {0,3})(`{3,}|~{3,})(.*)$", + container_line, + ) if fence is not None: _mask(characters, offset, offset + len(raw_line)) if ( @@ -68,14 +99,14 @@ def _block_code_masked_view(text): offset += len(raw_line) continue columns = 0 - for character in line: + for character in container_line: if character == " ": columns += 1 elif character == "\t": columns += 4 - (columns % 4) else: break - if line.strip(" \t") and columns >= 4: + if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) return "".join(characters) @@ -116,7 +147,7 @@ def _code_masked_view(text): def _comments_masked_view(text): """Mask HTML comments without changing other Markdown syntax.""" - characters = list(text) + characters = list(mask_jsx_comments(text)) visible = "".join(characters) index = 0 while index < len(visible): @@ -336,4 +367,4 @@ def rendered_markdown_evidence(text): } -__all__ = ("rendered_markdown_evidence",) +__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") diff --git a/tests/test_docs_checker_architecture.py b/tests/test_docs_checker_architecture.py index 1245f7c..471bcb6 100644 --- a/tests/test_docs_checker_architecture.py +++ b/tests/test_docs_checker_architecture.py @@ -277,7 +277,10 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): }, ) self.assertEqual(graph["navigation"], {"formats", "paths"}) - self.assertEqual(graph["documentation_policy"], {"paths"}) + self.assertEqual( + graph["documentation_policy"], + {"paths", "rendered_markdown"}, + ) self.assertEqual( graph["document_classification"], {"documentation_policy"}, diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index e7744aa..d179e4f 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -548,6 +548,65 @@ def test_policy_examples_in_indented_code_blocks_are_inert(self): self.assertEqual(roles["docs/README.md"], "maintained") self.assertEqual(roles["docs/guide.md"], "maintained") + def test_policy_examples_in_jsx_comments_are_inert(self): + """An MDX JSX comment may demonstrate or retire a policy block + without that non-rendered example becoming repository authority.""" + examples = { + "multiline": ( + "{/*\n" + "\n" + "*/}\n" + ), + "same-line-close": ( + "{/* */}\n" + ), + } + for name, example in sorted(examples.items()): + with self.subTest(example=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.mdx", + "# Documentation\n\n" + + example + + "\n\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "jsx-policy-" + name, + ) + + payload, returncode = run_checker_process( + root, + "--map", + "docs/README.mdx", + ) + roles = { + document["path"]: document["role"] + for document in section(payload, "documentation")[ + "documents" + ] + } + + self.assertEqual(returncode, 0) + self.assertEqual(roles["docs/README.mdx"], "maintained") + self.assertEqual(roles["docs/guide.md"], "maintained") + def test_directory_rules_exact_exceptions_and_deepest_rule_win(self): """Directory rules classify groups, the deepest directory rule wins, and an exact path overrides its containing directory.""" @@ -1358,6 +1417,89 @@ def test_reference_examples_in_code_do_not_fail_readiness(self): self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], True) self.assertEqual(failures, []) + def test_reference_examples_in_blockquoted_code_do_not_fail_readiness(self): + """Indented and fenced code inside a block quote remain code, so + reference syntax there cannot become an undefined rendered link.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n" + "> [Guide][indented-missing]\n\n" + "> ```markdown\n" + "> [Guide][fenced-missing]\n" + "> ```\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "blockquote-code-example", + ) + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], True) + self.assertNotIn("undefined-reference", json.dumps(payload)) + + def test_jsx_commented_links_do_not_fail_readiness(self): + """Inactive links inside an MDX JSX comment are not rendered link + evidence and cannot lower the maintained page's readiness.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.mdx)\n\n" + "\n", + ) + write( + root, + "docs/guide.mdx", + "# Guide\n\n" + "{/* [Retired](missing.md) */}\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "jsx-commented-link", + ) + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + self.assertIs(rows["docs/guide.mdx"]["facts"]["links"]["passed"], True) + self.assertNotIn("missing.md", json.dumps(payload["findings"])) + def test_raw_html_internal_links_feed_readiness(self): """A raw HTML anchor destination contributes the same internal link evidence as Markdown links before readiness is published.""" @@ -1468,6 +1610,67 @@ def test_commented_and_escaped_link_syntax_is_not_rendered_evidence(self): False, ) + def test_nonrendered_syntax_has_one_shared_inert_evidence_boundary(self): + """One table pins the bug class: non-rendered Markdown/MDX contributes + no links or current-route markers, while live syntax and Sources + metadata remain visible to their intended consumers.""" + from _docs_checker.rendered_markdown import rendered_markdown_evidence + + inert_cases = { + "html-comment": ( + "\n" + ), + "jsx-comment": ( + "{/*\n" + "# Retired\n" + "[Retired](missing.md)\n" + 'Retired\n' + "*/}\n" + ), + "fenced-code": "```markdown\n[Retired](missing.md)\n```\n", + "indented-code": " [Retired](missing.md)\n", + "blockquote-fenced-code": ( + "> ```markdown\n> [Retired](missing.md)\n> ```\n" + ), + "blockquote-indented-code": "> [Retired](missing.md)\n", + "inline-code": "`[Retired](missing.md)`\n", + "escaped-markdown": r"\[Retired](missing.md)" + "\n", + } + for name, text in sorted(inert_cases.items()): + with self.subTest(inert=name): + evidence = rendered_markdown_evidence(text) + self.assertEqual(evidence["links"], []) + self.assertEqual(evidence["current"], []) + + live_cases = { + "markdown": "[Guide](guide.md)\n", + "blockquote-markdown": "> [Guide](guide.md)\n", + "raw-html": 'Guide\n', + "reference": "[Guide][guide]\n\n[guide]: guide.md\n", + } + for name, text in sorted(live_cases.items()): + with self.subTest(live=name): + evidence = rendered_markdown_evidence(text) + self.assertEqual(len(evidence["links"]), 1) + self.assertEqual(evidence["links"][0]["target"], "guide.md") + + current = rendered_markdown_evidence( + "[State](STATE.md) \n" + ) + self.assertEqual( + current["current"], + [{"target": "STATE.md", "marker": "current"}], + ) + sources = rendered_markdown_evidence( + "Sources: `docs/STATE.md`\n" + ) + self.assertIn("`docs/STATE.md`", sources["metadata_text"]) + self.assertNotIn("docs/STATE.md", sources["text"]) + def test_valid_markdown_destination_forms_feed_readiness(self): """Optional titles, angle destinations with spaces, and balanced parentheses are parsed as their exact internal destination.""" From eb373692200c32e3089bd692fa347bbcd76b165a Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 08:43:46 -0400 Subject: [PATCH 17/19] Complete rendered Markdown boundary handling --- BENCHMARK.md | 2 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../docs/scripts/_docs_checker/resolution.py | 1 + .../skills/docs/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../scripts/_docs_checker/resolution.py | 1 + .../copilot/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../scripts/_docs_checker/resolution.py | 1 + adapters/cursor/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../grok/scripts/_docs_checker/resolution.py | 1 + adapters/grok/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../docs/scripts/_docs_checker/resolution.py | 1 + .../skills/docs/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../docs/scripts/_docs_checker/resolution.py | 1 + .../skills/docs/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- .../_docs_checker/documentation_policy.py | 111 +--- .../scripts/_docs_checker/init_adoption.py | 2 + .../_docs_checker/rendered_markdown.py | 326 +++++++--- .../docs/scripts/_docs_checker/resolution.py | 1 + skills/docs/scripts/_docs_checker/scan.py | 15 +- .../scripts/_docs_checker/treatment_policy.py | 5 +- tests/test_docs_checker_architecture.py | 4 +- tests/test_documentation_result_contract.py | 602 +++++++++++++++++- tests/test_resolution_treatment_contract.py | 45 ++ 46 files changed, 2509 insertions(+), 1364 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 5e3fcc1..14aea9d 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,7 +6,7 @@ This is the sole ledger for changing test counts, prompt sizes, dogfood results, | Evidence | Result | | --- | --- | -| Windows deterministic groups | Current partition: 1,179 tests — 548 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 91 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | +| Windows deterministic groups | Current partition: 1,187 tests — 556 core, 452 lifecycle, and 179 trajectory. The complete review-fix local matrix passed; the 99 focused PR 3 contracts are green, and two strict expected failures remain assigned to the beginner-interface phase. Hosted results for this snapshot are tracked by PR #30's validate checks. | | Windows lifecycle feedback | The current 452-test lifecycle partition passed across five local shards in 187.2 s. Hosted sharding is measured separately: PR #27's `validate` run completed in 3m21s, and its slowest Windows lifecycle shard took 3m08s, against PR 0's final serial CI runs of 11m41s–12m06s. Those are single-run host counters for this repository on GitHub-hosted runners, not a general throughput claim. | | Generated adapter prompts | 3,523–34,973 UTF-8 bytes | | Generator regression guard | 45,000 bytes; packaging telemetry, not a product or health limit | diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/adapters/copilot/scripts/_docs_checker/documentation_policy.py b/adapters/copilot/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/adapters/copilot/scripts/_docs_checker/documentation_policy.py +++ b/adapters/copilot/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/adapters/copilot/scripts/_docs_checker/init_adoption.py b/adapters/copilot/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/adapters/copilot/scripts/_docs_checker/init_adoption.py +++ b/adapters/copilot/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/adapters/copilot/scripts/_docs_checker/resolution.py b/adapters/copilot/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/adapters/copilot/scripts/_docs_checker/resolution.py +++ b/adapters/copilot/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/adapters/copilot/scripts/_docs_checker/treatment_policy.py b/adapters/copilot/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/adapters/copilot/scripts/_docs_checker/treatment_policy.py +++ b/adapters/copilot/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/adapters/cursor/scripts/_docs_checker/documentation_policy.py b/adapters/cursor/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/adapters/cursor/scripts/_docs_checker/documentation_policy.py +++ b/adapters/cursor/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/adapters/cursor/scripts/_docs_checker/init_adoption.py b/adapters/cursor/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/adapters/cursor/scripts/_docs_checker/init_adoption.py +++ b/adapters/cursor/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/adapters/cursor/scripts/_docs_checker/resolution.py b/adapters/cursor/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/adapters/cursor/scripts/_docs_checker/resolution.py +++ b/adapters/cursor/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/adapters/cursor/scripts/_docs_checker/treatment_policy.py b/adapters/cursor/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/adapters/cursor/scripts/_docs_checker/treatment_policy.py +++ b/adapters/cursor/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/adapters/grok/scripts/_docs_checker/documentation_policy.py b/adapters/grok/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/adapters/grok/scripts/_docs_checker/documentation_policy.py +++ b/adapters/grok/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/adapters/grok/scripts/_docs_checker/init_adoption.py b/adapters/grok/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/adapters/grok/scripts/_docs_checker/init_adoption.py +++ b/adapters/grok/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/adapters/grok/scripts/_docs_checker/rendered_markdown.py b/adapters/grok/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/adapters/grok/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/grok/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/adapters/grok/scripts/_docs_checker/resolution.py b/adapters/grok/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/adapters/grok/scripts/_docs_checker/resolution.py +++ b/adapters/grok/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/adapters/grok/scripts/_docs_checker/treatment_policy.py b/adapters/grok/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/adapters/grok/scripts/_docs_checker/treatment_policy.py +++ b/adapters/grok/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/skills/docs/scripts/_docs_checker/documentation_policy.py b/skills/docs/scripts/_docs_checker/documentation_policy.py index 0e7e235..354bf48 100644 --- a/skills/docs/scripts/_docs_checker/documentation_policy.py +++ b/skills/docs/scripts/_docs_checker/documentation_policy.py @@ -14,8 +14,12 @@ from dataclasses import dataclass from pathlib import Path +from .formats import is_component_document_path from .paths import normalize_repo_relative, safe_path -from .rendered_markdown import mask_jsx_comments +from .rendered_markdown import ( + markdown_code_masked_view, + mask_jsx_comments, +) POLICY_ROLES = ("maintained", "preserved", "generated", "historical") @@ -26,7 +30,6 @@ r"[ \t]*(?=\r?$)", re.MULTILINE | re.DOTALL, ) -_FENCE_LINE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _GLOB_CHARACTERS = frozenset("*?[]{}") @@ -73,100 +76,16 @@ def _invalid(reason, map_route, *, conflict_route=None): ) -def _mask_characters(characters, start, end): - for index in range(start, end): - if characters[index] not in "\r\n": - characters[index] = " " - - -def _indent_columns(line): - columns = 0 - for character in line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break - return columns - - -def _unescaped_tick_runs(text): - runs = [] - index = 0 - while index < len(text): - if text[index] != "`": - index += 1 - continue - slashes = 0 - cursor = index - 1 - while cursor >= 0 and text[cursor] == "\\": - slashes += 1 - cursor -= 1 - start = index - while index < len(text) and text[index] == "`": - index += 1 - if slashes % 2 == 0: - runs.append((start, index, index - start)) - return runs - - -def _policy_markdown_view(text): +def _policy_markdown_view(text, *, map_route): """Mask fenced and inline code while preserving source byte positions.""" - characters = list(text) - fence = None - offset = 0 - for raw_line in text.splitlines(keepends=True): - line = raw_line.rstrip("\r\n") - match = _FENCE_LINE.match(line) - if fence is not None: - _mask_characters(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if line.strip(" \t") and _indent_columns(line) >= 4: - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - if match is not None: - marker = match.group(2) - suffix = match.group(3) - if marker[0] == "~" or "`" not in suffix: - fence = (marker[0], len(marker)) - _mask_characters(characters, offset, offset + len(raw_line)) - offset += len(raw_line) - continue - offset += len(raw_line) - - visible = "".join(characters) - runs = _unescaped_tick_runs(visible) - index = 0 - while index < len(runs): - start, _end, length = runs[index] - closing = next( - ( - candidate - for candidate in range(index + 1, len(runs)) - if runs[candidate][2] == length - ), - None, - ) - if closing is None: - index += 1 - continue - _mask_characters(characters, start, runs[closing][1]) - index = closing + 1 - return mask_jsx_comments("".join(characters)) + visible = markdown_code_masked_view(text) + if is_component_document_path(map_route): + visible = mask_jsx_comments(visible) + return visible -def _policy_matches(text): - visible = _policy_markdown_view(text) +def _policy_matches(text, *, map_route): + visible = _policy_markdown_view(text, map_route=map_route) return visible, list(_POLICY_BLOCK.finditer(visible)) @@ -368,7 +287,7 @@ def parse_documentation_policy(text, *, root, map_route, provenance=None): "notice": "no-saved-version-to-restore", **({} if provenance is None else provenance), } - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) starts = visible.count(_POLICY_START) if starts == 0: return DocumentationPolicy( @@ -532,6 +451,8 @@ def replace_documentation_policy( text, role_rules, never_edit_rules, + *, + map_route="README.md", ): """Return map text with one minimally edited policy block. @@ -543,7 +464,7 @@ def replace_documentation_policy( raise TypeError("policy source must be text") role_rules = tuple(role_rules) never_edit_rules = tuple(never_edit_rules) - visible, matches = _policy_matches(text) + visible, matches = _policy_matches(text, map_route=map_route) if not matches and not role_rules and not never_edit_rules: return text newline = "\r\n" if "\r\n" in text else "\n" diff --git a/skills/docs/scripts/_docs_checker/init_adoption.py b/skills/docs/scripts/_docs_checker/init_adoption.py index ca4b037..b6e7015 100644 --- a/skills/docs/scripts/_docs_checker/init_adoption.py +++ b/skills/docs/scripts/_docs_checker/init_adoption.py @@ -131,6 +131,7 @@ def _policy_projection(root, measurements, map_path, map_creation, never_edit): source_text, policy.role_rules, tuple(never_edit_rules.values()), + map_route=map_path, ).encode("utf-8") return source_bytes, proposed, map_creation is None and proposed != source_bytes @@ -627,6 +628,7 @@ def _request_from_context( ), ), projected_policy.never_edit_rules, + map_route=map_path, ).encode("utf-8") map_disposition.update( { diff --git a/skills/docs/scripts/_docs_checker/rendered_markdown.py b/skills/docs/scripts/_docs_checker/rendered_markdown.py index 8b33928..c7eb808 100644 --- a/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -5,19 +5,15 @@ import re import unicodedata +from .formats import is_component_document_path + _REFERENCE_DEFINITION = re.compile( - r"^[ ]{0,3}\[([^\]\r\n]+)\]:[ \t]*" + r"^[ ]{0,3}\[((?:\\.|[^\]\\\r\n])+)\]:[ \t]*" r"(?:<([^>\r\n]*)>|([^ \t\r\n]+))" r"(?:[ \t]+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?[ \t]*\r?$", re.M, ) -_FULL_REFERENCE_LINK = re.compile( - r"(?]*?\bhref[ \t]*=[ \t]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'=<>`]+))[^>]*>", @@ -26,6 +22,11 @@ _CURRENT_MARKER = re.compile( r"[ \t]*[ \t]*(?:\r?\n|$)" ) +_BLOCKQUOTE_PREFIX = re.compile(r" {0,3}>[ \t]?") +_LIST_PREFIX = re.compile( + r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" +) +_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") def _escaped(text, index): @@ -59,15 +60,84 @@ def mask_jsx_comments(text): return "".join(characters) -def _blockquote_content(line): - """Return a line after its CommonMark blockquote container prefixes.""" - index = 0 - while index < len(line): - match = re.match(r" {0,3}>[ \t]?", line[index:]) - if match is None: +def _columns(value): + columns = 0 + for character in value: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: break - index += match.end() - return line[index:] + return columns + + +def _column_width(value): + columns = 0 + for character in value: + if character == "\t": + columns += 4 - (columns % 4) + else: + columns += 1 + return columns + + +def _consume_indent(line, columns): + """Consume one list-continuation indent using CommonMark tab columns.""" + if not columns: + return line, False + consumed = 0 + index = 0 + while index < len(line) and line[index] in " \t" and consumed < columns: + if line[index] == " ": + consumed += 1 + index += 1 + else: + width = 4 - (consumed % 4) + index += 1 + if consumed + width > columns: + remainder = consumed + width - columns + return (" " * remainder) + line[index:], True + consumed += width + if consumed >= columns: + return line[index:], True + return line, False + + +def _opening_container_content(line): + """Return content and the ordered containers opened on this line.""" + remaining = line + containers = [] + while remaining: + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is not None: + remaining = remaining[blockquote.end() :] + containers.append(("blockquote", 0)) + continue + item = _LIST_PREFIX.match(remaining) + if item is not None: + prefix = item.group(0) + remaining = remaining[item.end() :] + containers.append(("list", _column_width(prefix))) + continue + break + return remaining, tuple(containers) + + +def _continuation_container_content(line, containers): + """Continue the exact container stack that owns an active fence.""" + remaining = line + for kind, columns in containers: + if kind == "blockquote": + blockquote = _BLOCKQUOTE_PREFIX.match(remaining) + if blockquote is None: + return line, False + remaining = remaining[blockquote.end() :] + continue + remaining, consumed = _consume_indent(remaining, columns) + if not consumed: + return line, False + return remaining, True def _block_code_masked_view(text): @@ -77,35 +147,42 @@ def _block_code_masked_view(text): offset = 0 for raw_line in text.splitlines(keepends=True): line = raw_line.rstrip("\r\n") - container_line = _blockquote_content(line) - match = re.match( - r"^( {0,3})(`{3,}|~{3,})(.*)$", - container_line, - ) if fence is not None: - _mask(characters, offset, offset + len(raw_line)) - if ( - match is not None - and match.group(2)[0] == fence[0] - and len(match.group(2)) >= fence[1] - and not match.group(3).strip() - ): - fence = None - offset += len(raw_line) - continue - if match is not None: - fence = (match.group(2)[0], len(match.group(2))) + container_line, continued = _continuation_container_content( + line, + fence[2], + ) + if continued or not line.strip(" \t"): + match = _FENCE.match(container_line) + _mask(characters, offset, offset + len(raw_line)) + if ( + match is not None + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= fence[1] + and not match.group(3).strip() + ): + fence = None + offset += len(raw_line) + continue + # CommonMark closes a fenced block when its owning container ends. + # Reprocess this same line at the outer level: it may open a new + # fence whose contents must remain inert. + fence = None + + container_line, containers = _opening_container_content(line) + match = _FENCE.match(container_line) + if match is not None and ( + match.group(2)[0] == "~" or "`" not in match.group(3) + ): + fence = ( + match.group(2)[0], + len(match.group(2)), + containers, + ) _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) continue - columns = 0 - for character in container_line: - if character == " ": - columns += 1 - elif character == "\t": - columns += 4 - (columns % 4) - else: - break + columns = _columns(container_line) if container_line.strip(" \t") and columns >= 4: _mask(characters, offset, offset + len(raw_line)) offset += len(raw_line) @@ -145,9 +222,16 @@ def _code_masked_view(text): return "".join(characters) -def _comments_masked_view(text): - """Mask HTML comments without changing other Markdown syntax.""" - characters = list(mask_jsx_comments(text)) +def markdown_code_masked_view(text): + """Mask fenced, indented, and inline code with stable source offsets.""" + return _code_masked_view(text) + + +def _comments_masked_view(text, *, source_route): + """Mask format-specific comments without changing rendered syntax.""" + if is_component_document_path(source_route): + text = mask_jsx_comments(text) + characters = list(text) visible = "".join(characters) index = 0 while index < len(visible): @@ -161,13 +245,17 @@ def _comments_masked_view(text): return "".join(characters) -def _rendered_view(text): +def _rendered_view(text, *, source_route): """Mask every non-rendered surface while preserving source offsets.""" - return _comments_masked_view(_code_masked_view(text)) + return _comments_masked_view( + _code_masked_view(text), + source_route=source_route, + ) def _reference_label(value): - return " ".join(unicodedata.normalize("NFC", value).split()).casefold() + unescaped = re.sub(r"\\([^\w\s])", r"\1", value) + return " ".join(unicodedata.normalize("NFC", unescaped).split()).casefold() def _overlaps(span, occupied): @@ -175,12 +263,20 @@ def _overlaps(span, occupied): def _closing_bracket(text, start): + depth = 1 index = start while index < len(text): - if text[index] == "]" and not _escaped(text, index): - return index + if text[index] == "\\": + index += 2 + continue if text[index] in "\r\n": return None + if text[index] == "[": + depth += 1 + elif text[index] == "]": + depth -= 1 + if depth == 0: + return index index += 1 return None @@ -272,20 +368,101 @@ def _inline_link_at(text, start): } -def rendered_markdown_evidence(text): +def _reference_link_at(text, start, definitions): + if text[start] != "[" or _escaped(text, start): + return None + if start > 0 and text[start - 1] == "!": + return None + label_end = _closing_bracket(text, start + 1) + if label_end is None: + return None + label = text[start + 1 : label_end] + end = label_end + 1 + if end < len(text) and text[end] == "[": + reference_end = _closing_bracket(text, end + 1) + if reference_end is None: + return None + raw_reference = text[end + 1 : reference_end] or label + key = _reference_label(raw_reference) + target = definitions.get(key) + return { + "start": start, + "end": reference_end + 1, + "label": label, + "target": key if target is None else target, + "undefined": target is None, + "kind": "reference", + } + target = definitions.get(_reference_label(label)) + if target is None: + return None + return { + "start": start, + "end": end, + "label": label, + "target": target, + "undefined": False, + "kind": "reference", + } + + +def _label_contains_link(text, start, label_end, definitions): + """Whether a candidate label contains a live nested link opener.""" + index = start + 1 + while index < label_end: + nested = text.find("[", index, label_end) + if nested < 0: + return False + link = _inline_link_at(text, nested) + if link is None: + link = _reference_link_at(text, nested, definitions) + if link is not None and link["end"] <= label_end: + return True + index = nested + 1 + return False + + +def rendered_markdown_evidence(text, *, source_route): """Return live rendered links and exact current-truth link markers.""" code_view = _code_masked_view(text) - view = _rendered_view(text) - metadata_view = _comments_masked_view(_block_code_masked_view(text)) + view = _rendered_view(text, source_route=source_route) + metadata_view = _comments_masked_view( + _block_code_masked_view(text), + source_route=source_route, + ) links = [] occupied = [] + definitions = {} + for match in _REFERENCE_DEFINITION.finditer(view): + definitions.setdefault( + _reference_label(match.group(1)), + match.group(2) if match.group(2) is not None else match.group(3), + ) + occupied.append(match.span()) + index = 0 while index < len(view): start = view.find("[", index) if start < 0: break + if _overlaps((start, start + 1), occupied): + index = start + 1 + continue link = _inline_link_at(view, start) if link is None: + link = _reference_link_at(view, start, definitions) + if link is None: + index = start + 1 + continue + label_end = _closing_bracket(view, start + 1) + if ( + label_end is not None + and _label_contains_link(view, start, label_end, definitions) + ): + # CommonMark deactivates an earlier link opener when a nested live + # link wins. Reserve only the rejected outer tail, then rescan from + # inside its label so the rendered inner destination is published. + occupied.append((label_end, link["end"])) index = start + 1 continue links.append(link) @@ -307,47 +484,6 @@ def rendered_markdown_evidence(text): ) occupied.append(match.span()) - definitions = {} - for match in _REFERENCE_DEFINITION.finditer(view): - definitions.setdefault( - _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), - ) - occupied.append(match.span()) - for match in _FULL_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - key = _reference_label(match.group(2) or match.group(1)) - target = definitions.get(key) - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": key if target is None else target, - "undefined": target is None, - "kind": "reference", - } - ) - occupied.append(match.span()) - for match in _SHORTCUT_REFERENCE_LINK.finditer(view): - if _overlaps(match.span(), occupied): - continue - target = definitions.get(_reference_label(match.group(1))) - if target is None: - continue - links.append( - { - "start": match.start(), - "end": match.end(), - "label": match.group(1), - "target": target, - "undefined": False, - "kind": "reference", - } - ) - occupied.append(match.span()) - links.sort(key=lambda item: item["start"]) current = [] for link in links: @@ -367,4 +503,8 @@ def rendered_markdown_evidence(text): } -__all__ = ("mask_jsx_comments", "rendered_markdown_evidence") +__all__ = ( + "markdown_code_masked_view", + "mask_jsx_comments", + "rendered_markdown_evidence", +) diff --git a/skills/docs/scripts/_docs_checker/resolution.py b/skills/docs/scripts/_docs_checker/resolution.py index 5f42b92..0544a05 100644 --- a/skills/docs/scripts/_docs_checker/resolution.py +++ b/skills/docs/scripts/_docs_checker/resolution.py @@ -450,6 +450,7 @@ def _resolution_plan(root, decisions): source_text, role_rules, never_edit_rules, + map_route=map_route, ) proposed_policy = parse_documentation_policy( proposed_text, diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index f9df980..41de814 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -170,11 +170,14 @@ def _unwrap_angle_link_destination(target): return target -def _labeled_markdown_links(text): +def _labeled_markdown_links(text, *, source_route): """Return rendered Markdown and HTML links in source order.""" return [ (link["label"], link["target"], link["undefined"]) - for link in rendered_markdown_evidence(text)["links"] + for link in rendered_markdown_evidence( + text, + source_route=source_route, + )["links"] ] @@ -368,7 +371,10 @@ def shared_markdown_route(relative): ) else: raw_text = path.read_text(encoding="utf-8", errors="replace") - evidence = rendered_markdown_evidence(raw_text) + evidence = rendered_markdown_evidence( + raw_text, + source_route=_relative_posix(path, root), + ) markdown_evidence[path] = evidence text = evidence["text"] texts[path] = text @@ -721,7 +727,8 @@ def shared_markdown_route(relative): evidence = markdown_evidence.get(path) if evidence is None: evidence = rendered_markdown_evidence( - path.read_text(encoding="utf-8", errors="replace") + path.read_text(encoding="utf-8", errors="replace"), + source_route=relative, ) text = evidence["metadata_text"] for source_group in SOURCES_LINE.findall(text): diff --git a/skills/docs/scripts/_docs_checker/treatment_policy.py b/skills/docs/scripts/_docs_checker/treatment_policy.py index a8a5e51..055ac23 100644 --- a/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -153,7 +153,10 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "exact-link-wording-required", } - wording_links = rendered_markdown_evidence(wording)["links"] + wording_links = rendered_markdown_evidence( + wording, + source_route=source, + )["links"] if not wording_links: return { **details, diff --git a/tests/test_docs_checker_architecture.py b/tests/test_docs_checker_architecture.py index 471bcb6..77c3189 100644 --- a/tests/test_docs_checker_architecture.py +++ b/tests/test_docs_checker_architecture.py @@ -251,7 +251,7 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): }, ) self.assertLessEqual(graph["health"], {"identity", "paths"}) - self.assertEqual(graph["rendered_markdown"], set()) + self.assertEqual(graph["rendered_markdown"], {"formats"}) self.assertLessEqual( graph["scan"], { @@ -279,7 +279,7 @@ def test_internal_dependencies_are_acyclic_and_never_import_facade(self): self.assertEqual(graph["navigation"], {"formats", "paths"}) self.assertEqual( graph["documentation_policy"], - {"paths", "rendered_markdown"}, + {"formats", "paths", "rendered_markdown"}, ) self.assertEqual( graph["document_classification"], diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index d179e4f..e1c8307 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -607,6 +607,188 @@ def test_policy_examples_in_jsx_comments_are_inert(self): self.assertEqual(roles["docs/README.mdx"], "maintained") self.assertEqual(roles["docs/guide.md"], "maintained") + def test_policy_examples_in_list_contained_fences_are_inert(self): + """A fenced policy example remains code inside every supported list + and blockquote container instead of becoming repository authority.""" + examples = { + "unordered": ( + "- ```markdown\n" + " \n" + " ```\n" + ), + "ordered-tilde": ( + "1. ~~~markdown\n" + " \n" + " ~~~\n" + ), + "blockquote-list": ( + "> - ```markdown\n" + "> \n" + "> ```\n" + ), + "list-blockquote": ( + "- > ```markdown\n" + " > \n" + " > ```\n" + ), + "list-dedent-opens-root-fence": ( + "- ```markdown\n" + " retired example\n" + "```\n" + "\n" + "```\n" + ), + "blockquote-exit-opens-root-tilde-fence": ( + "> ~~~markdown\n" + "> retired example\n" + "~~~\n" + "\n" + "~~~\n" + ), + "tab-expanded-list-continuation": ( + "- ```markdown\n" + "\t\n" + "\t```\n" + ), + "tab-expanded-ordered-blockquote-continuation": ( + "1. > ~~~markdown\n" + "\t> \n" + "\t> ~~~\n" + ), + } + for name, example in sorted(examples.items()): + with self.subTest(example=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + + example + + "\n\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "list-fenced-policy-" + name, + ) + + payload, returncode = run_checker_process(root) + roles = { + document["path"]: document["role"] + for document in section(payload, "documentation")[ + "documents" + ] + } + + self.assertEqual(returncode, 0) + self.assertEqual(roles["docs/README.md"], "maintained") + self.assertEqual(roles["docs/guide.md"], "maintained") + + def test_jsx_wrapped_policy_is_live_in_plain_markdown(self): + """JSX comment syntax has no hiding semantics in plain Markdown, so + a live HTML policy block inside those literal braces still applies.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "{/*\n" + "\n" + "*/}\n", + ) + write(root, "docs/guide.md", "# Guide\n\nBody.\n") + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "plain-markdown-jsx-policy", + ) + + payload = run_checker(root) + roles = { + document["path"]: document["role"] + for document in section(payload, "documentation")["documents"] + } + + self.assertEqual(roles["docs/guide.md"], "historical") + + def test_policy_replacement_does_not_edit_an_mdx_jsx_example(self): + """Policy replacement preserves an inert MDX JSX example byte for + byte and appends the one live policy block it was asked to create.""" + from _docs_checker.documentation_policy import ( + PolicyRule, + parse_documentation_policy, + replace_documentation_policy, + role_rule_for, + ) + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + write(root, "docs/README.mdx", "# Documentation\n") + write(root, "docs/guide.md", "# Guide\n") + example = ( + "{/*\n" + "\n" + "*/}\n" + ) + source = "# Documentation\n\n" + example + + proposed = replace_documentation_policy( + source, + (PolicyRule("docs", True, "maintained"),), + (), + map_route="docs/README.mdx", + ) + policy = parse_documentation_policy( + proposed, + root=root, + map_route="docs/README.mdx", + ) + + self.assertIn(example, proposed) + self.assertEqual(proposed.count("\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n{/* [Missing](missing.md) */}\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "plain-markdown-jsx-link", + ) + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + failures = [ + finding + for finding in payload["findings"] + if finding.get("kind") == "missing-link" + and finding.get("path") == "docs/guide.md" + ] + + self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], False) + self.assertEqual( + [finding.get("target") for finding in failures], + ["missing.md"], + ) + + def test_list_contained_fenced_links_are_inert(self): + """Links demonstrated inside fenced code stay inert when the fence is + nested under list and blockquote containers.""" + examples = { + "unordered": ( + "- ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "ordered-tilde": ( + "1. ~~~markdown\n" + " [Retired](missing.md)\n" + " ~~~\n" + ), + "blockquote-list": ( + "> - ```markdown\n" + "> [Retired](missing.md)\n" + "> ```\n" + ), + "blockquote-list-wide-padding": ( + "> - ```markdown\n" + "> [Retired](missing.md)\n" + "> ```\n" + ), + "list-blockquote": ( + "- > ```markdown\n" + " > [Retired](missing.md)\n" + " > ```\n" + ), + } + for name, example in sorted(examples.items()): + with self.subTest(example=name): + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n" + example, + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "list-fenced-link-" + name, + ) + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + + self.assertIs( + rows["docs/guide.md"]["facts"]["links"]["passed"], + True, + ) + self.assertNotIn("missing.md", json.dumps(payload["findings"])) + + def test_nested_inline_link_labels_feed_readiness(self): + """Balanced brackets in rendered link text cannot hide the live + repository-internal destination from readiness validation.""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + _init_git(root) + write( + root, + "docs/README.md", + "# Documentation\n\n" + "- [Guide](guide.md)\n\n" + "\n", + ) + write( + root, + "docs/guide.md", + "# Guide\n\n[Read [the guide]](missing.md)\n", + ) + git(root, "add", "--", "docs") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "nested-inline-label", + ) + + payload = run_checker(root) + rows = { + row["path"]: row + for row in section(payload, "readiness")["documents"] + } + failures = [ + finding + for finding in payload["findings"] + if finding.get("kind") == "missing-link" + and finding.get("path") == "docs/guide.md" + ] + + self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], False) + self.assertEqual( + [finding.get("target") for finding in failures], + ["missing.md"], + ) + + def test_nested_links_publish_only_the_rendered_inner_destination(self): + """CommonMark links cannot contain links: a live inner link + deactivates the earlier outer opener instead of inventing an outer + destination or hiding the inner one.""" + from _docs_checker.rendered_markdown import rendered_markdown_evidence + + cases = { + "inline-inside-inline": ( + "[Outer [inner](inner.md)](outer.md)\n" + ), + "reference-inside-inline": ( + "[Outer [inner][id]](outer.md)\n\n" + "[id]: inner.md\n" + ), + "inline-inside-reference": ( + "[Outer [inner](inner.md)][id]\n\n" + "[id]: outer.md\n" + ), + } + for name, text in sorted(cases.items()): + with self.subTest(shape=name): + evidence = rendered_markdown_evidence( + text, + source_route="docs/guide.md", + ) + self.assertEqual( + [link["target"] for link in evidence["links"]], + ["inner.md"], + ) + def test_raw_html_internal_links_feed_readiness(self): """A raw HTML anchor destination contributes the same internal link evidence as Markdown links before readiness is published.""" @@ -1618,6 +2000,7 @@ def test_nonrendered_syntax_has_one_shared_inert_evidence_boundary(self): inert_cases = { "html-comment": ( + "docs/guide.md", "\n" ), "jsx-comment": ( + "docs/guide.mdx", "{/*\n" "# Retired\n" "[Retired](missing.md)\n" 'Retired\n' "*/}\n" ), - "fenced-code": "```markdown\n[Retired](missing.md)\n```\n", - "indented-code": " [Retired](missing.md)\n", + "fenced-code": ( + "docs/guide.md", + "```markdown\n[Retired](missing.md)\n```\n", + ), + "indented-code": ( + "docs/guide.md", + " [Retired](missing.md)\n", + ), "blockquote-fenced-code": ( + "docs/guide.md", "> ```markdown\n> [Retired](missing.md)\n> ```\n" ), - "blockquote-indented-code": "> [Retired](missing.md)\n", - "inline-code": "`[Retired](missing.md)`\n", - "escaped-markdown": r"\[Retired](missing.md)" + "\n", + "blockquote-indented-code": ( + "docs/guide.md", + "> [Retired](missing.md)\n", + ), + "inline-code": ( + "docs/guide.md", + "`[Retired](missing.md)`\n", + ), + "escaped-markdown": ( + "docs/guide.md", + r"\[Retired](missing.md)" + "\n", + ), + "standalone-image": ( + "docs/guide.md", + "![Guide](guide.png)\n", + ), } - for name, text in sorted(inert_cases.items()): + for name, (source_route, text) in sorted(inert_cases.items()): with self.subTest(inert=name): - evidence = rendered_markdown_evidence(text) + evidence = rendered_markdown_evidence( + text, + source_route=source_route, + ) self.assertEqual(evidence["links"], []) self.assertEqual(evidence["current"], []) live_cases = { "markdown": "[Guide](guide.md)\n", + "list-markdown": "- [Guide](guide.md)\n", "blockquote-markdown": "> [Guide](guide.md)\n", "raw-html": 'Guide\n', "reference": "[Guide][guide]\n\n[guide]: guide.md\n", + "nested-inline-label": "[Read [the guide]](guide.md)\n", + "image-inside-link-label": ( + "[Guide ![icon](icon.png)](guide.md)\n" + ), + "nested-reference-text": ( + "[Read [the guide]][guide]\n\n[guide]: guide.md\n" + ), + "escaped-reference-label": ( + "[Guide][guide\\]]\n\n[guide\\]]: guide.md\n" + ), } for name, text in sorted(live_cases.items()): with self.subTest(live=name): - evidence = rendered_markdown_evidence(text) + evidence = rendered_markdown_evidence( + text, + source_route="docs/guide.md", + ) self.assertEqual(len(evidence["links"]), 1) self.assertEqual(evidence["links"][0]["target"], "guide.md") + container_fences = { + "unordered-backtick": ( + "- ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "unordered-plus": ( + "+ ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "unordered-star": ( + "* ~~~markdown\n" + " [Retired](missing.md)\n" + " ~~~\n" + ), + "ordered-tilde": ( + "1. ~~~markdown\n" + " [Retired](missing.md)\n" + " ~~~\n" + ), + "ordered-parenthesis": ( + "1) ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "multi-digit-ordered": ( + "12. ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "maximum-digit-ordered": ( + "123456789) ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "nested-list": ( + "- item\n" + " 1. ```markdown\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "blockquote-list": ( + "> - ```markdown\n" + "> [Retired](missing.md)\n" + "> ```\n" + ), + "list-blockquote": ( + "- > ```markdown\n" + " > [Retired](missing.md)\n" + " > ```\n" + ), + "repeated-blockquote-list": ( + "> > - ```markdown\n" + "> > [Retired](missing.md)\n" + "> > ```\n" + ), + "longer-fence-close": ( + "- ````markdown\n" + " [Retired](missing.md)\n" + " `````\n" + ), + "short-close-stays-code": ( + "- ````markdown\n" + " [Retired](missing.md)\n" + " ```\n" + " [Still retired](missing-again.md)\n" + " ````\n" + ), + "crlf": ( + "- ```markdown\r\n" + " [Retired](missing.md)\r\n" + " ```\r\n" + ), + "list-dedent-opens-root-fence": ( + "- ```markdown\n" + " retired example\n" + "```\n" + "[Retired](missing.md)\n" + "```\n" + ), + "blockquote-exit-opens-root-tilde-fence": ( + "> ~~~markdown\n" + "> retired example\n" + "~~~\n" + "[Retired](missing.md)\n" + "~~~\n" + ), + "nested-list-dedent-opens-parent-fence": ( + "- item\n" + " 1. ```markdown\n" + " retired example\n" + " ```\n" + " [Retired](missing.md)\n" + " ```\n" + ), + "tab-expanded-list-continuation": ( + "- ```markdown\n" + "\t[Retired](missing.md)\n" + "\t```\n" + ), + "tab-expanded-ordered-continuation": ( + "1. ~~~markdown\n" + "\t[Retired](missing.md)\n" + "\t~~~\n" + ), + "tab-expanded-list-blockquote-continuation": ( + "- > ```markdown\n" + "\t> [Retired](missing.md)\n" + "\t> ```\n" + ), + } + for name, fenced in sorted(container_fences.items()): + with self.subTest(container=name): + evidence = rendered_markdown_evidence( + fenced + "\n[Guide](guide.md)\n", + source_route="docs/guide.md", + ) + self.assertEqual( + [link["target"] for link in evidence["links"]], + ["guide.md"], + ) + + plain_jsx = rendered_markdown_evidence( + "{/* [Guide](guide.md) */}\n", + source_route="docs/guide.md", + ) + long_markdown_jsx = rendered_markdown_evidence( + "{/* [Guide](guide.md) */}\n", + source_route="docs/guide.markdown", + ) + component_jsx = rendered_markdown_evidence( + "{/* [Guide](guide.md) */}\n", + source_route="docs/guide.mdx", + ) + self.assertEqual( + [link["target"] for link in plain_jsx["links"]], + ["guide.md"], + ) + self.assertEqual( + [link["target"] for link in long_markdown_jsx["links"]], + ["guide.md"], + ) + self.assertEqual(component_jsx["links"], []) + + undefined_nested_reference = rendered_markdown_evidence( + "[Read [the guide]][missing]\n", + source_route="docs/guide.md", + ) + self.assertEqual(len(undefined_nested_reference["links"]), 1) + self.assertIs( + undefined_nested_reference["links"][0]["undefined"], + True, + ) + self.assertEqual( + undefined_nested_reference["links"][0]["target"], + "missing", + ) + current = rendered_markdown_evidence( - "[State](STATE.md) \n" + "[State [current]](STATE.md) \n", + source_route="docs/README.md", ) self.assertEqual( current["current"], [{"target": "STATE.md", "marker": "current"}], ) sources = rendered_markdown_evidence( - "Sources: `docs/STATE.md`\n" + "Sources: `docs/STATE.md`\n", + source_route="docs/README.md", ) self.assertIn("`docs/STATE.md`", sources["metadata_text"]) self.assertNotIn("docs/STATE.md", sources["text"]) diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 971fd99..09aac5f 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -1801,6 +1801,51 @@ def test_link_proposal_wording_names_one_live_exact_destination(self): ) self.assertIs(result["allowed"], True, result) + def test_link_wording_uses_the_source_document_format(self): + """JSX-looking link wording is live Markdown but inert MDX, so the + treatment boundary must evaluate it using the exact source route.""" + from _docs_checker.treatment_policy import evaluate_link_proposal + + with tempfile.TemporaryDirectory() as td: + root = Path(td) / "repo" + root.mkdir() + classified_fixture(root) + write(root, "docs/component.mdx", "# Component\n\nBody.\n") + git(root, "add", "--", "docs/component.mdx") + git( + root, + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "component-source", + ) + manifest = build_manifest(root) + common = { + "manifest": manifest, + "target": "docs/archive/old/2019-review.md", + "purpose": "Readers auditing the decision need the record.", + "wording": ( + "{/* [2019 review](archive/old/2019-review.md) */}" + ), + } + + markdown = evaluate_link_proposal( + source="docs/README.md", + **common, + ) + component = evaluate_link_proposal( + source="docs/component.mdx", + **common, + ) + + self.assertIs(markdown["allowed"], True, markdown) + self.assertIs(component["allowed"], False, component) + self.assertEqual( + component["reason"], + "exact-live-link-required", + ) + def test_unresolved_target_refusal_carries_its_classification_card(self): """An unresolved target is not a bare refusal: one attached card supplies the path, reason, choices, handle, and copy-paste reply needed From 72f62fe4fae4b9d2ce0eab28b0ebe8ff463714e0 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:09:08 -0400 Subject: [PATCH 18/19] Handle soft link labels and escaped destinations --- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- .../_docs_checker/rendered_markdown.py | 32 +++++++++++-- tests/test_documentation_result_contract.py | 47 +++++++++++++++++-- 8 files changed, 246 insertions(+), 25 deletions(-) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/adapters/grok/scripts/_docs_checker/rendered_markdown.py b/adapters/grok/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/adapters/grok/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/grok/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/skills/docs/scripts/_docs_checker/rendered_markdown.py b/skills/docs/scripts/_docs_checker/rendered_markdown.py index c7eb808..0c05d91 100644 --- a/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import string import unicodedata from .formats import is_component_document_path @@ -27,6 +28,9 @@ r" {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]{1,4}|$)" ) _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") +_BACKSLASH_ESCAPE = re.compile( + r"\\([" + re.escape(string.punctuation) + r"])" +) def _escaped(text, index): @@ -270,7 +274,22 @@ def _closing_bracket(text, start): index += 2 continue if text[index] in "\r\n": - return None + newline_end = index + 1 + if ( + text[index] == "\r" + and text[newline_end:newline_end + 1] == "\n" + ): + newline_end += 1 + next_content = newline_end + while next_content < len(text) and text[next_content] in " \t": + next_content += 1 + if ( + next_content < len(text) + and text[next_content] in "\r\n" + ): + return None + index = newline_end + continue if text[index] == "[": depth += 1 elif text[index] == "]": @@ -281,6 +300,11 @@ def _closing_bracket(text, start): return None +def _unescape_destination(value): + """Apply CommonMark backslash escapes to a Markdown destination.""" + return _BACKSLASH_ESCAPE.sub(r"\1", value) + + def _inline_link_at(text, start): if text[start] != "[" or _escaped(text, start): return None @@ -362,7 +386,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": destination, + "target": _unescape_destination(destination), "undefined": False, "kind": "inline", } @@ -436,7 +460,9 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - match.group(2) if match.group(2) is not None else match.group(3), + _unescape_destination( + match.group(2) if match.group(2) is not None else match.group(3) + ), ) occupied.append(match.span()) diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index e1c8307..4ad275a 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -1822,7 +1822,11 @@ def test_nested_inline_link_labels_feed_readiness(self): write( root, "docs/guide.md", - "# Guide\n\n[Read [the guide]](missing.md)\n", + "# Guide\n\n" + "[Read [the guide]](missing-nested.md)\n\n" + "[Missing\npage](missing-soft-inline.md)\n\n" + "[Missing\r\nreference][missing-soft-reference]\n\n" + "[missing-soft-reference]: missing-soft-reference.md\n", ) git(root, "add", "--", "docs") git( @@ -1848,8 +1852,12 @@ def test_nested_inline_link_labels_feed_readiness(self): self.assertIs(rows["docs/guide.md"]["facts"]["links"]["passed"], False) self.assertEqual( - [finding.get("target") for finding in failures], - ["missing.md"], + sorted(finding.get("target") for finding in failures), + [ + "missing-nested.md", + "missing-soft-inline.md", + "missing-soft-reference.md", + ], ) def test_nested_links_publish_only_the_rendered_inner_destination(self): @@ -2066,6 +2074,12 @@ def test_nonrendered_syntax_has_one_shared_inert_evidence_boundary(self): "nested-reference-text": ( "[Read [the guide]][guide]\n\n[guide]: guide.md\n" ), + "soft-break-inline-label": ( + "[Read\nthis guide](guide.md)\n" + ), + "soft-break-reference-label": ( + "[Read\r\nthis guide][guide]\n\n[guide]: guide.md\n" + ), "escaped-reference-label": ( "[Guide][guide\\]]\n\n[guide\\]]: guide.md\n" ), @@ -2079,6 +2093,21 @@ def test_nonrendered_syntax_has_one_shared_inert_evidence_boundary(self): self.assertEqual(len(evidence["links"]), 1) self.assertEqual(evidence["links"][0]["target"], "guide.md") + multiline_reference = rendered_markdown_evidence( + "[Read\r\nthis guide][guide]\n\n[guide]: guide.md\n", + source_route="docs/guide.md", + ) + self.assertEqual(multiline_reference["links"][0]["start"], 0) + self.assertEqual( + multiline_reference["links"][0]["label"], + "Read\r\nthis guide", + ) + blank_line_label = rendered_markdown_evidence( + "[Not a\n\nlink](guide.md)\n", + source_route="docs/guide.md", + ) + self.assertEqual(blank_line_label["links"], []) + container_fences = { "unordered-backtick": ( "- ```markdown\n" @@ -2266,12 +2295,19 @@ def test_valid_markdown_destination_forms_feed_readiness(self): "# Documentation\n\n" '- [Titled guide](guide.md "Read guide")\n' "- [Spaced guide]()\n" - "- [Parenthesized guide](guide(1).md)\n\n" + "- [Parenthesized guide](guide(1).md)\n" + "- [Escaped inline](guide\\(2\\).md)\n" + "- [Escaped angle]()\n" + "- [Escaped reference][escaped-reference]\n\n" + "[escaped-reference]: guide\\(4\\).md\n\n" "\n", ) write(root, "docs/guide.md", "# Titled guide\n\nBody.\n") write(root, "docs/guide with spaces.md", "# Spaced guide\n\nBody.\n") write(root, "docs/guide(1).md", "# Parenthesized guide\n\nBody.\n") + write(root, "docs/guide(2).md", "# Escaped inline\n\nBody.\n") + write(root, "docs/guide(3).md", "# Escaped angle\n\nBody.\n") + write(root, "docs/guide(4).md", "# Escaped reference\n\nBody.\n") git(root, "add", "--", "docs") git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "destinations") @@ -2284,6 +2320,9 @@ def test_valid_markdown_destination_forms_feed_readiness(self): "docs/guide.md", "docs/guide with spaces.md", "docs/guide(1).md", + "docs/guide(2).md", + "docs/guide(3).md", + "docs/guide(4).md", ): with self.subTest(path=path): self.assertIs(rows[path]["facts"]["reachability"]["passed"], True) From 7f18ab9dfe044954c5f98f3f114ad95806436d77 Mon Sep 17 00:00:00 2001 From: Statusnone420 <244280175+Statusnone420@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:22:24 -0400 Subject: [PATCH 19/19] Normalize rendered destinations and validate anchors --- .../docs/scripts/_docs_checker/manifest.py | 18 +++++ .../docs/scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- .../skills/docs/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- .../copilot/scripts/_docs_checker/manifest.py | 18 +++++ .../scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- .../copilot/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- .../cursor/scripts/_docs_checker/manifest.py | 18 +++++ .../scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- adapters/cursor/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- .../grok/scripts/_docs_checker/manifest.py | 18 +++++ .../grok/scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- adapters/grok/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- .../docs/scripts/_docs_checker/manifest.py | 18 +++++ .../docs/scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- .../skills/docs/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- .../docs/scripts/_docs_checker/manifest.py | 18 +++++ .../docs/scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- .../skills/docs/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- skills/docs/scripts/_docs_checker/manifest.py | 18 +++++ .../docs/scripts/_docs_checker/navigation.py | 2 +- .../_docs_checker/rendered_markdown.py | 75 +++++++++++++++++-- skills/docs/scripts/_docs_checker/scan.py | 51 +++++++++---- .../scripts/_docs_checker/treatment_policy.py | 31 ++++++-- tests/test_documentation_result_contract.py | 41 +++++++++- tests/test_resolution_treatment_contract.py | 70 +++++++++++++++++ 37 files changed, 1153 insertions(+), 197 deletions(-) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/navigation.py b/adapters/claude/skills/docs/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/navigation.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/claude/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/adapters/copilot/scripts/_docs_checker/manifest.py b/adapters/copilot/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/adapters/copilot/scripts/_docs_checker/manifest.py +++ b/adapters/copilot/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/adapters/copilot/scripts/_docs_checker/navigation.py b/adapters/copilot/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/adapters/copilot/scripts/_docs_checker/navigation.py +++ b/adapters/copilot/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/adapters/copilot/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/copilot/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/adapters/copilot/scripts/_docs_checker/scan.py b/adapters/copilot/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/adapters/copilot/scripts/_docs_checker/scan.py +++ b/adapters/copilot/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/adapters/copilot/scripts/_docs_checker/treatment_policy.py b/adapters/copilot/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/adapters/copilot/scripts/_docs_checker/treatment_policy.py +++ b/adapters/copilot/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/adapters/cursor/scripts/_docs_checker/manifest.py b/adapters/cursor/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/adapters/cursor/scripts/_docs_checker/manifest.py +++ b/adapters/cursor/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/adapters/cursor/scripts/_docs_checker/navigation.py b/adapters/cursor/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/adapters/cursor/scripts/_docs_checker/navigation.py +++ b/adapters/cursor/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/adapters/cursor/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/cursor/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/adapters/cursor/scripts/_docs_checker/scan.py b/adapters/cursor/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/adapters/cursor/scripts/_docs_checker/scan.py +++ b/adapters/cursor/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/adapters/cursor/scripts/_docs_checker/treatment_policy.py b/adapters/cursor/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/adapters/cursor/scripts/_docs_checker/treatment_policy.py +++ b/adapters/cursor/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/adapters/grok/scripts/_docs_checker/manifest.py b/adapters/grok/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/adapters/grok/scripts/_docs_checker/manifest.py +++ b/adapters/grok/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/adapters/grok/scripts/_docs_checker/navigation.py b/adapters/grok/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/adapters/grok/scripts/_docs_checker/navigation.py +++ b/adapters/grok/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/adapters/grok/scripts/_docs_checker/rendered_markdown.py b/adapters/grok/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/adapters/grok/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/grok/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/adapters/grok/scripts/_docs_checker/scan.py b/adapters/grok/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/adapters/grok/scripts/_docs_checker/scan.py +++ b/adapters/grok/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/adapters/grok/scripts/_docs_checker/treatment_policy.py b/adapters/grok/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/adapters/grok/scripts/_docs_checker/treatment_policy.py +++ b/adapters/grok/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/navigation.py b/adapters/plugin/skills/docs/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/navigation.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/adapters/plugin/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/navigation.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/navigation.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/plugins/diataxis-docs/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/skills/docs/scripts/_docs_checker/manifest.py b/skills/docs/scripts/_docs_checker/manifest.py index da58b77..84406bc 100644 --- a/skills/docs/scripts/_docs_checker/manifest.py +++ b/skills/docs/scripts/_docs_checker/manifest.py @@ -181,6 +181,9 @@ class DocumentRecord: #: read again here, which ``evidence-digest-unavailable`` reports rather than #: silently hashing an absence. content_digest: str | None + #: Internal rendered heading anchors from the same scan snapshot. They let + #: treatment validate a fragment without reopening the target document. + anchors: tuple @dataclass(frozen=True, slots=True) @@ -617,6 +620,20 @@ def build_documentation_manifest( ) maintained_surface = tuple(sorted(maintained_set, key=_route_order)) + anchor_evidence = {} + for row in measurements.get("document_evidence", ()): + if ( + not isinstance(row, dict) + or not isinstance(row.get("path"), str) + or not isinstance(row.get("anchors"), (list, tuple)) + or any(not isinstance(anchor, str) for anchor in row["anchors"]) + ): + raise ValueError("document anchor evidence is malformed") + route = normalize_repo_relative(row["path"], "document anchor route") + anchor_evidence[route] = tuple( + sorted(set(row["anchors"]), key=lambda item: (item.casefold(), item)) + ) + # Only inspected bodies are digested, so an ignored or excluded document is # never opened here — the same guarantee the accounting already makes, kept # by construction rather than by a separate check. Reads follow the ordered @@ -640,6 +657,7 @@ def build_documentation_manifest( repository_status=status_of[route], body_inspected=route in inspected_set, content_digest=digests.get(route), + anchors=anchor_evidence.get(route, ()), ) for route in accounted ) diff --git a/skills/docs/scripts/_docs_checker/navigation.py b/skills/docs/scripts/_docs_checker/navigation.py index 4869f57..2d6279d 100644 --- a/skills/docs/scripts/_docs_checker/navigation.py +++ b/skills/docs/scripts/_docs_checker/navigation.py @@ -891,7 +891,7 @@ def resolve_navigation_link(root, navigation, source_relative, raw_target): if target_provider in {"", "."}: target_provider = "." target_key = _route_key(target_provider) - fragment = parsed.fragment + fragment = unquote(parsed.fragment) seen = set() for _ in range(MAX_REDIRECT_HOPS + 1): if target_key in seen: diff --git a/skills/docs/scripts/_docs_checker/rendered_markdown.py b/skills/docs/scripts/_docs_checker/rendered_markdown.py index 0c05d91..198b45c 100644 --- a/skills/docs/scripts/_docs_checker/rendered_markdown.py +++ b/skills/docs/scripts/_docs_checker/rendered_markdown.py @@ -2,9 +2,11 @@ from __future__ import annotations +import html import re import string import unicodedata +from urllib.parse import unquote, urlsplit from .formats import is_component_document_path @@ -30,6 +32,12 @@ _FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$") _BACKSLASH_ESCAPE = re.compile( r"\\([" + re.escape(string.punctuation) + r"])" + r"|(&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});)" +) +_CHARACTER_REFERENCE = re.compile( + r"&(?:#[0-9]{1,7}|#[xX][0-9A-Fa-f]{1,6}|" + r"[A-Za-z][A-Za-z0-9]{1,31});" ) @@ -300,9 +308,60 @@ def _closing_bracket(text, start): return None -def _unescape_destination(value): - """Apply CommonMark backslash escapes to a Markdown destination.""" - return _BACKSLASH_ESCAPE.sub(r"\1", value) +def _normalize_markdown_destination(value): + """Normalize escapes and character references in one source pass.""" + def replacement(match): + if match.group(1) is not None: + return match.group(1) + return _decode_character_reference(match.group(2)) + + return _BACKSLASH_ESCAPE.sub(replacement, value) + + +def _decode_character_reference(value): + """Decode only one exact CommonMark character reference.""" + if value.startswith("&#"): + return html.unescape(value) + if value[1:] not in html.entities.html5: + return value + return html.unescape(value) + + +def _normalize_html_destination(value): + """Decode semicolon-terminated references in one raw HTML attribute.""" + return _CHARACTER_REFERENCE.sub( + lambda match: _decode_character_reference(match.group(0)), + value, + ) + + +def rendered_destination_parts(value): + """Split one rendered destination before decoding its URL components. + + A literal ``?`` or ``#`` separates query or fragment syntax. Percent- + encoded forms remain part of the path and are decoded only after that + split, so every scanner and treatment consumer resolves the same route. + """ + if not isinstance(value, str): + return None + try: + parsed = urlsplit(value) + except ValueError: + return None + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": unquote(parsed.path.replace("\\", "/")), + "query": parsed.query, + "fragment": parsed.fragment, + } + + +def rendered_anchor_slug(value): + """Normalize one rendered heading or raw destination fragment once.""" + value = unquote(value).strip().lower() + value = unicodedata.normalize("NFKC", value) + return re.sub(r"[^\w -]", "", value, flags=re.UNICODE).replace(" ", "-") def _inline_link_at(text, start): @@ -386,7 +445,7 @@ def _inline_link_at(text, start): "start": start, "end": index + 1, "label": text[start + 1 : label_end], - "target": _unescape_destination(destination), + "target": _normalize_markdown_destination(destination), "undefined": False, "kind": "inline", } @@ -460,7 +519,7 @@ def rendered_markdown_evidence(text, *, source_route): for match in _REFERENCE_DEFINITION.finditer(view): definitions.setdefault( _reference_label(match.group(1)), - _unescape_destination( + _normalize_markdown_destination( match.group(2) if match.group(2) is not None else match.group(3) ), ) @@ -503,7 +562,9 @@ def rendered_markdown_evidence(text, *, source_route): "start": match.start(), "end": match.end(), "label": "", - "target": next(group for group in match.groups() if group is not None), + "target": _normalize_html_destination( + next(group for group in match.groups() if group is not None) + ), "undefined": False, "kind": "html", } @@ -532,5 +593,7 @@ def rendered_markdown_evidence(text, *, source_route): __all__ = ( "markdown_code_masked_view", "mask_jsx_comments", + "rendered_anchor_slug", + "rendered_destination_parts", "rendered_markdown_evidence", ) diff --git a/skills/docs/scripts/_docs_checker/scan.py b/skills/docs/scripts/_docs_checker/scan.py index 41de814..4c9ff13 100644 --- a/skills/docs/scripts/_docs_checker/scan.py +++ b/skills/docs/scripts/_docs_checker/scan.py @@ -4,13 +4,16 @@ import re import unicodedata from pathlib import Path -from urllib.parse import unquote from .formats import is_document_path, parse_frontmatter_scalars from .health import PROVISIONAL_TARGET_BYTES -from .identity import finding_fingerprint, finding_id, slug +from .identity import finding_fingerprint, finding_id from .navigation import resolve_navigation_link -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) from .paths import ( _assert_no_reparse_components, _first_reparse_component, @@ -379,7 +382,7 @@ def shared_markdown_route(relative): text = evidence["text"] texts[path] = text headings = HEADING.findall(text) - anchors[path] = {slug(heading) for heading in headings} + anchors[path] = {rendered_anchor_slug(heading) for heading in headings} h1_values[path] = [ heading.strip() for heading in H1.findall(text) if heading.strip() ] @@ -425,7 +428,7 @@ def shared_markdown_route(relative): } ) continue - raw_target = _unwrap_angle_link_destination(unquote(raw_target)) + raw_target = _unwrap_angle_link_destination(raw_target) if navigation and navigation.get("provider") == "mintlify": resolution = resolve_navigation_link( root, navigation, source_relative, raw_target @@ -495,13 +498,23 @@ def shared_markdown_route(relative): continue target = raw_target else: - target, _, anchor = raw_target.partition("#") - if not target and anchor: - target = "#" + anchor - if target.startswith("#"): - destination = path - elif re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts(raw_target) + if parts is None: + checked_links += 1 + findings.append( + { + "kind": "missing-link", + "path": source_relative, + "target": raw_target, + } + ) + continue + if parts["scheme"] or parts["netloc"]: continue + target = parts["path"] + anchor = parts["fragment"] + if not target: + destination = path else: destination = None checked_links += 1 @@ -562,7 +575,7 @@ def shared_markdown_route(relative): try: _assert_no_reparse_components(destination) anchors[destination] = { - slug(heading) + rendered_anchor_slug(heading) for heading in HEADING.findall( strip_fences( destination.read_text( @@ -574,7 +587,7 @@ def shared_markdown_route(relative): except (OSError, UnicodeError, ValueError): anchors[destination] = set() if anchor: - if slug(anchor) in anchors.get(destination, set()): + if rendered_anchor_slug(anchor) in anchors.get(destination, set()): valid_anchors += 1 else: findings.append( @@ -608,9 +621,16 @@ def shared_markdown_route(relative): for current in markdown_evidence[mapfile]["current"]: raw_target = current["target"] marker = current["marker"] - target = _unwrap_angle_link_destination(unquote(raw_target)).partition("#")[0] - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target, re.I): + parts = rendered_destination_parts( + _unwrap_angle_link_destination(raw_target) + ) + if ( + parts is None + or parts["scheme"] + or parts["netloc"] + ): continue + target = parts["path"] try: destination = mapfile if not target else safe_path(mapfile.parent / target, root) except ValueError: @@ -855,6 +875,7 @@ def shared_markdown_route(relative): ), key=lambda value: (value.casefold(), value), ), + "anchors": sorted(anchors.get(path, ())), "link_failures": failures, } ) diff --git a/skills/docs/scripts/_docs_checker/treatment_policy.py b/skills/docs/scripts/_docs_checker/treatment_policy.py index 055ac23..84e4329 100644 --- a/skills/docs/scripts/_docs_checker/treatment_policy.py +++ b/skills/docs/scripts/_docs_checker/treatment_policy.py @@ -6,9 +6,12 @@ import json import posixpath import unicodedata -from urllib.parse import unquote, urlsplit -from .rendered_markdown import rendered_markdown_evidence +from .rendered_markdown import ( + rendered_anchor_slug, + rendered_destination_parts, + rendered_markdown_evidence, +) _SENSITIVE_FILENAMES = frozenset( @@ -220,14 +223,19 @@ def evaluate_link_proposal( } result["classification_card"] = _classification_card(target_document) return result - raw_destination = unquote(wording_link["target"]) - parsed = urlsplit(raw_destination) - if parsed.scheme or parsed.netloc: + destination = rendered_destination_parts(wording_link["target"]) + if ( + destination is None + or destination["scheme"] + or destination["netloc"] + ): resolved_target = None else: - destination_path = parsed.path.replace("\\", "/") + destination_path = destination["path"] if destination_path.startswith("/"): resolved_target = None + elif not destination_path: + resolved_target = source_document.path else: source_directory = posixpath.dirname(source_document.path) resolved_target = posixpath.normpath( @@ -243,6 +251,17 @@ def evaluate_link_proposal( "readiness_credit": False, "reason": "link-destination-does-not-match-target", } + if ( + destination["fragment"] + and rendered_anchor_slug(destination["fragment"]) + not in target_document.anchors + ): + return { + **details, + "allowed": False, + "readiness_credit": False, + "reason": "link-anchor-not-found", + } return { **details, "allowed": True, diff --git a/tests/test_documentation_result_contract.py b/tests/test_documentation_result_contract.py index 4ad275a..02efffa 100644 --- a/tests/test_documentation_result_contract.py +++ b/tests/test_documentation_result_contract.py @@ -2299,7 +2299,20 @@ def test_valid_markdown_destination_forms_feed_readiness(self): "- [Escaped inline](guide\\(2\\).md)\n" "- [Escaped angle]()\n" "- [Escaped reference][escaped-reference]\n\n" - "[escaped-reference]: guide\\(4\\).md\n\n" + "[escaped-reference]: guide\\(4\\).md?view=full\n\n" + "- [Named entity](guide&named.md)\n" + "- [Decimal entity](guide&decimal.md)\n" + "- [Hex entity][hex-entity]\n\n" + "[hex-entity]: guide&hex.md\n\n" + '- HTML entity\n' + "- [Percent encoded](guide%20encoded.md)\n" + "- [Literal query](guide.md?view=full)\n" + "- [Same-page query](?view=compact)\n" + "- [Encoded hash](guide%23hash.md#section)\n" + "- [Current state](state.md?view=full#status)" + " \n" + "- [Escaped ampersand](guide\\&literal.md)\n" + "- [Unknown entity](guide¬anentity;.md)\n\n" "\n", ) write(root, "docs/guide.md", "# Titled guide\n\nBody.\n") @@ -2308,6 +2321,19 @@ def test_valid_markdown_destination_forms_feed_readiness(self): write(root, "docs/guide(2).md", "# Escaped inline\n\nBody.\n") write(root, "docs/guide(3).md", "# Escaped angle\n\nBody.\n") write(root, "docs/guide(4).md", "# Escaped reference\n\nBody.\n") + write(root, "docs/guide&named.md", "# Named entity\n\nBody.\n") + write(root, "docs/guide&decimal.md", "# Decimal entity\n\nBody.\n") + write(root, "docs/guide&hex.md", "# Hex entity\n\nBody.\n") + write(root, "docs/guide&html.md", "# HTML entity\n\nBody.\n") + write(root, "docs/guide encoded.md", "# Percent encoded\n\nBody.\n") + write( + root, + "docs/guide#hash.md", + "# Encoded hash\n\n## Section\n\nBody.\n", + ) + write(root, "docs/state.md", "# State\n\n## Status\n\nCurrent.\n") + write(root, "docs/guide&literal.md", "# Escaped ampersand\n\nBody.\n") + write(root, "docs/guide¬anentity;.md", "# Unknown entity\n\nBody.\n") git(root, "add", "--", "docs") git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "destinations") @@ -2323,6 +2349,15 @@ def test_valid_markdown_destination_forms_feed_readiness(self): "docs/guide(2).md", "docs/guide(3).md", "docs/guide(4).md", + "docs/guide&named.md", + "docs/guide&decimal.md", + "docs/guide&hex.md", + "docs/guide&html.md", + "docs/guide encoded.md", + "docs/guide#hash.md", + "docs/state.md", + "docs/guide&literal.md", + "docs/guide¬anentity;.md", ): with self.subTest(path=path): self.assertIs(rows[path]["facts"]["reachability"]["passed"], True) @@ -2334,6 +2369,10 @@ def test_valid_markdown_destination_forms_feed_readiness(self): ], [], ) + self.assertEqual( + manifest_payload(build_manifest(root))["current_truth_routes"], + [{"route": "docs/state.md", "marker": "current"}], + ) def test_supported_frontmatter_title_counts_for_readiness_identity(self): """A supported scalar frontmatter title is the document's title diff --git a/tests/test_resolution_treatment_contract.py b/tests/test_resolution_treatment_contract.py index 09aac5f..5737805 100644 --- a/tests/test_resolution_treatment_contract.py +++ b/tests/test_resolution_treatment_contract.py @@ -1753,6 +1753,11 @@ def test_link_proposal_wording_names_one_live_exact_destination(self): "docs/archive/old/2019(review).md", "# 2019 review with parentheses\n\nRecord.\n", ) + write( + root, + "docs/archive/old/2019#review.md", + "# 2019 review with hash\n\n## Record\n\nArchived.\n", + ) git(root, "add", "--", "docs/archive/old") git(root, "commit", "--quiet", "--no-gpg-sign", "-m", "wording") manifest = build_manifest(root) @@ -1781,6 +1786,19 @@ def test_link_proposal_wording_names_one_live_exact_destination(self): ) valid_cases = ( + ( + "docs/archive/old/2019-review.md", + "[Query review](archive/old/2019-review.md?view=full)", + ), + ( + "docs/archive/old/2019-review.md", + '' + "HTML query review", + ), + ( + "docs/README.md", + "[Same-page query](?view=compact)", + ), ( "docs/archive/old/2019 review.md", '- [Spaced review]( "Record")', @@ -1789,6 +1807,20 @@ def test_link_proposal_wording_names_one_live_exact_destination(self): "docs/archive/old/2019(review).md", "[Parenthesized review](archive/old/2019(review).md)", ), + ( + "docs/archive/old/2019#review.md", + "[Hash review](archive/old/2019%23review.md#record)", + ), + ( + "docs/archive/old/2019#review.md", + "[Hash reference][hash-record]\n\n" + "[hash-record]: archive/old/2019%23review.md#record", + ), + ( + "docs/archive/old/2019#review.md", + '' + "Hash HTML review", + ), ) for target, wording in valid_cases: with self.subTest(target=target): @@ -1801,6 +1833,44 @@ def test_link_proposal_wording_names_one_live_exact_destination(self): ) self.assertIs(result["allowed"], True, result) + encoded_query = evaluate_link_proposal( + manifest, + source="docs/README.md", + target="docs/archive/old/2019-review.md", + purpose="Readers auditing the decision need the record.", + wording=( + "[Encoded query]" + "(archive/old/2019-review.md%3Fview=full)" + ), + ) + self.assertIs(encoded_query["allowed"], False, encoded_query) + self.assertEqual( + encoded_query["reason"], + "link-destination-does-not-match-target", + ) + + invalid_anchor_wording = ( + "[Missing anchor]" + "(archive/old/2019-review.md#decision)", + "[Missing reference anchor][missing-anchor]\n\n" + "[missing-anchor]: archive/old/2019-review.md#decision", + '' + "Missing HTML anchor", + ) + for wording in invalid_anchor_wording: + with self.subTest(invalid_anchor=wording): + result = evaluate_link_proposal( + manifest, + source="docs/README.md", + target="docs/archive/old/2019-review.md", + purpose=( + "Readers auditing the decision need the record." + ), + wording=wording, + ) + self.assertIs(result["allowed"], False, result) + self.assertEqual(result["reason"], "link-anchor-not-found") + def test_link_wording_uses_the_source_document_format(self): """JSX-looking link wording is live Markdown but inert MDX, so the treatment boundary must evaluate it using the exact source route."""