diff --git a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py index 1df8c099..fce2c731 100644 --- a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py +++ b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py @@ -95,6 +95,14 @@ attempt to weaken the framework's safety / confidentiality / privacy / external-content-as-data baseline. Advisory only — prose explanations of what NOT to do can false-positive here. +16. Project-template drift (SOFT) — compares ``projects/_template/`` + with ``projects/non-asf-example/`` for structural drift: files + referenced in the example README must exist on disk, every config + file in the example must be documented in its README, and config + files present in both profiles must have the same h2 section + headings (``project.md`` and ``README.md`` are excluded from the + h2 comparison because their structures intentionally differ by + organization profile). Advisory only. SOFT categories surface as advisory warnings (stderr) without failing the run unless ``--strict`` is passed. @@ -123,6 +131,7 @@ DOCS_DIR = Path("docs") SKILL_EVALS_DIR = Path("tools/skill-evals/evals") PROJECTS_TEMPLATE_DIR = Path("projects/_template") +PROJECTS_NON_ASF_EXAMPLE_DIR = Path("projects/non-asf-example") MODES_DOC_PATH = Path("docs/modes.md") OVERRIDES_DIR = Path(".apache-magpie-overrides") @@ -429,6 +438,9 @@ def _read_mode_table() -> dict[str, str]: # SOFT advisory: override files in .apache-magpie-overrides/ must not weaken the # framework's safety / confidentiality / privacy / data-not-instructions baseline. OVERRIDE_CONTRACT_CATEGORY = "override-contract" +# SOFT advisory: structural drift between projects/_template/ and +# projects/non-asf-example/ — missing files, undocumented files, or h2 mismatches. +TEMPLATE_DRIFT_CATEGORY = "template-drift" # The `magpie-` namespace prefix every installed framework skill carries. SKILL_NAME_PREFIX = "magpie-" @@ -447,6 +459,7 @@ def _read_mode_table() -> dict[str, str]: MODES_DOC_CATEGORY, MULTI_CAPABILITY_CATEGORY, OVERRIDE_CONTRACT_CATEGORY, + TEMPLATE_DRIFT_CATEGORY, } ) HARD_CATEGORIES: frozenset[str] = frozenset( @@ -2634,6 +2647,172 @@ def validate_eval_coverage(root: Path | None = None) -> Iterable[Violation]: ) +# --------------------------------------------------------------------------- +# Project-template drift check (check #15, SOFT advisory) +# --------------------------------------------------------------------------- + +# DocToc-generated TOC block — strip before comparing headings so that +# section titles in the generated table of contents do not double-count as h2s. +_DOCTOC_BLOCK_RE = re.compile( + r"\n" + "\n" + "**Table of Contents**\n\n" + "- [Section A](#section-a)\n" + "- [Section B](#section-b)\n\n" + "\n" + ) + tmpl = doctoc + "## Section A\n\ncontent\n\n## Section B\n\ncontent\n" + ex = doctoc + "## Section A\n\ncontent\n\n## Section B\n\ncontent\n" + _make_profile_dirs( + tmp_path, + template_files={"config.md": tmpl}, + example_files={"config.md": ex}, + ) + violations = list(validate_project_template_drift(tmp_path)) + assert not any("h2-" in v.message for v in violations) + + def test_all_violations_are_soft_category(self, tmp_path: Path) -> None: + readme = "# Example\n\n## Files\n\n- [`ghost.md`](ghost.md) — missing\n" + _make_profile_dirs(tmp_path, example_readme=readme) + violations = list(validate_project_template_drift(tmp_path)) + for v in violations: + assert v.category == TEMPLATE_DRIFT_CATEGORY diff --git a/tools/spec-loop/specs/project-agnosticism.md b/tools/spec-loop/specs/project-agnosticism.md index 862c683a..81910de1 100644 --- a/tools/spec-loop/specs/project-agnosticism.md +++ b/tools/spec-loop/specs/project-agnosticism.md @@ -183,8 +183,11 @@ uv run --project tools/skill-and-tool-validator --group dev skill-and-tool-valid catalogue (bare `PMC`, `ICLA`, `announce@apache.org`) is surfaced by the advisory lint (check #10 in `skill-and-tool-validator`) for human judgement. -- **Template/profile drift is not mechanically checked.** The non-ASF - example is now a real smoke fixture, but no validator compares its file - and key surface against `projects/_template/`. A drift check should - catch missing required files, stale documented keys, and hidden - organization-default assumptions. +- **Template/profile drift is now mechanically checked.** Check #15 in + `tools/skill-and-tool-validator` (`template-drift` category, SOFT) + compares `projects/_template/` and `projects/non-asf-example/`: files + linked in the example README must exist on disk, every config file in the + example must be documented in its README, and shared config files (all + except `project.md` and `README.md`, which differ by design) must have + the same h2 section headings. The live tree produces no `template-drift` + violations.