From ede7ad493acd6fcc4ac665be0b6d7f9272d70cb3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:05:31 +0100 Subject: [PATCH 1/4] build(docs): move the site to Zensical Material for MkDocs reaches end of life on 2027-05-05 and its maintainers now build Zensical, so the docs site moves there before that date. Zensical builds the same 93 pages from the same Markdown, adds instant navigation, a Copy as Markdown button and an llms.txt index, but silently drops the nav and absolute-link validation MkDocs enforced; the new site-check.py step after the strict build puts those checks back. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/scripts/site-check.py | 233 ++++++++++++++++++++++++++++++ .github/workflows/docs.yml | 56 +++++--- .github/workflows/nextflow.yml | 2 +- .gitignore | 2 +- CLAUDE.md | 6 +- CONTRIBUTING.md | 4 +- docs/requirements-docs.txt | 2 +- docs/stylesheets/theme.css | 12 +- docs/updating.md | 2 +- mkdocs.yml | 240 ------------------------------- zensical.toml | 203 ++++++++++++++++++++++++++ 12 files changed, 492 insertions(+), 272 deletions(-) create mode 100644 .github/scripts/site-check.py delete mode 100644 mkdocs.yml create mode 100644 zensical.toml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0346a8e35..ca668ba25 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,7 +13,7 @@ ## Checklist - [ ] Relevant scripts/docs stay in sync -- [ ] Image tags and data versions changed only in `versions.env`; a new doc page is in the `mkdocs.yml` nav +- [ ] Image tags and data versions changed only in `versions.env`; a new doc page is in the `zensical.toml` nav - [ ] `shellcheck` passes for changed shell scripts - [ ] Smoke-test assumptions were checked for changed pipeline contracts - [ ] No personal paths, hostnames, or sample-specific defaults were introduced diff --git a/.github/scripts/site-check.py b/.github/scripts/site-check.py new file mode 100644 index 000000000..31a8df514 --- /dev/null +++ b/.github/scripts/site-check.py @@ -0,0 +1,233 @@ +# Copied from the new-repo templates into every docs repo. Repos run different ruff +# settings, so their formatter leaves this file as it is: lint it, do not reformat it. +# fmt: off +"""Post-build check of a Zensical site: the rules `zensical build --strict` does not enforce. + +usage: python3 -I .github/scripts/site-check.py [zensical.toml] + +Run from the repository root (the workflow does): the config is read from ./zensical.toml +unless its path is given. No config file is an error, never a skipped pass. + +Exit 1 on any of these, each printed as one ERROR line: + - a link or image whose target is not in the built site: a nav entry whose page does not + exist, a missing image, any other dangling href or src (Zensical validates Markdown page + links and anchors; it does not validate images, nav entries or raw HTML links). A redirect + stub's own links (canonical and body) are checked the same way; + - an href ending in .md: a correct page link is rewritten to a directory URL, so a .md left + in the HTML is a raw-HTML or unprocessed link, and it opens the llmstxt Markdown copy; + - a root-absolute link (/getting-started/) outside the site's own path prefix: it breaks + under https://geiserx.github.io// (MkDocs' `absolute_links: warn`, which Zensical + ignores); + - a page that no navigation entry links to: a stray file under docs/ published at a public + URL that nothing reaches (MkDocs' `nav.omitted_files: warn`, which Zensical ignores); + - a .py, .pyc or __pycache__ entry in the site: the awesome lists keep their Markdown + extension under docs/hooks/, and Zensical publishes every file not in exclude_docs; + - an awesome list's count token left in a page (``, `n:categorias`, + `n:retirados`): the Markdown extension that fills them in did not run. Only those three + names: CashPilot writes other `` comments of its own in its pages; + - a raw , <textarea>, <xmp>, <plaintext>, <noembed> or <noframes> tag inside a page's + article: a browser's HTML parser treats everything after it as text, so the rest of the + page disappears (Markdown passes `<title>` written in prose through as a real tag; + Telegram-Archive shipped that way); + - a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist: the + llmstxt plugin writes the copies, and only for pages its `sections` globs match. +And, from zensical.toml (Zensical drops what it does not know without a word; under MkDocs +strict an unknown plugin aborted the build): + - a plugin name Zensical 0.1.0 does not know (`redirect` for `redirects`, `llmstxtt`): the + plugin's whole output silently vanishes otherwise; + - a theme feature name the Zensical UI 0.0.41 does not know (the list below is what its + templates and bundle test for; `search.suggest` and `search.share` are not in it); + - `theme.variant` other than "modern" or "classic" (anything else is classic, silently), + and a "modern" config whose index.html does not load the modern stylesheet; + - a `redirect_maps` entry whose stub was not written; + - a `[project.plugins.llmstxt]` table with no site/llms.txt, or `content.action.copy` in + the features with a page that carries no copy button. +Stdlib only (tomllib needs Python 3.11+), so it runs under `python -I` on any runner. +""" + +import re +import sys +import tomllib +from pathlib import Path +from urllib.parse import unquote, urlparse + +SWALLOW = re.compile(r"<(title|textarea|xmp|plaintext|noembed|noframes)\b", re.IGNORECASE) +TOKEN = re.compile(r"<!--\s*n:(?:proyectos|categorias|retirados)\s*-->") +ARTICLE = re.compile(r"<article\b.*?</article>", re.DOTALL) +ATTR = re.compile(r'\s(?:href|src)="([^"]*)"') +COPY = re.compile(r'\sdata-md-copy-url="([^"]*)"') +NAV = re.compile(r'<a\s+href="([^"]*)"\s+class="md-(?:nav|tabs)__link') +STUB = "<title>Redirecting..." + +# Keys of _PLUGIN_UNSUPPORTED_OPTIONS in zensical/config.py (0.1.0, same set as 0.0.74): every other name is +# dropped by `_convert_plugins` without a message. `material/` prefixes and `social/`, +# `privacy/`, `optimize/` instance names canonicalise the same way Zensical does. +KNOWN_PLUGINS = { + "api-autonav", "autorefs", "awesome-nav", "blog", "callouts", "exclude", "gh-admonitions", + "glightbox", "include-markdown", "literate-nav", "llmstxt", "macros", "markdown-exec", + "markdownextradata", "meta", "mike", "minify", "mkdocs-audio", "mkdocs-autoapi", + "mkdocs-nav-weight", "mkdocs-video", "mkdocstrings", "offline", "optimize", "privacy", + "redirects", "rss", "search", "social", "table-reader", "tags", +} +# Feature names the shipped UI tests for (zensical_ui 0.0.41, read from its templates and +# bundle.*.min.js). Regenerate after a bump with: +# grep -rhoE '"[a-z]+(\.[a-z]+)+" in features' /zensical_ui/templates | sort -u +# grep -oE '"(navigation|content|search|toc|header|announce)\.[a-z.]+"' .../bundle.*.min.js | sort -u +KNOWN_FEATURES = { + "announce.dismiss", "content.action.copy", "content.action.edit", "content.action.view", + "content.code.annotate", "content.code.copy", "content.code.select", + "content.footnote.tooltips", "content.lazy", "content.tabs.link", "content.tooltips", + "header.autohide", "navigation.expand", "navigation.footer", "navigation.indexes", + "navigation.instant", "navigation.instant.prefetch", "navigation.instant.preview", + "navigation.instant.progress", "navigation.path", "navigation.prune", "navigation.sections", + "navigation.tabs", "navigation.tabs.sticky", "navigation.top", "navigation.tracking", + "search.highlight", "toc.follow", "toc.integrate", +} + + +def body_urls(body: str) -> set[str]: + return set(ATTR.findall(body)) + + +def canonical_plugin(name: str) -> str: + name = name.removeprefix("material/") + return name.split("/", 1)[0] if name.startswith(("social/", "privacy/", "optimize/")) else name + + +def read_config(path: Path, errors: set[str]) -> dict: + if not path.is_file(): + errors.add(f"{path}: config file not found (run from the repository root, or pass its path)") + return {} + try: + project = tomllib.loads(path.read_text(encoding="utf-8")).get("project") + except (tomllib.TOMLDecodeError, UnicodeDecodeError) as e: + errors.add(f"{path}: not valid TOML: {e}") + return {} + # Every config check reads [project]: without it they would all pass by skipping. + if not isinstance(project, dict) or not project: + errors.add(f"{path}: no [project] table, so no config check can run") + return {} + return project + + +def main(site_dir: str, config_path: str) -> int: + site = Path(site_dir).resolve() + index = site / "index.html" + if not index.is_file(): + print(f"ERROR {site_dir}: no index.html (did the build run?)") + return 1 + m = re.search(r' in the page body hides everything after it in a browser") + for tok in TOKEN.findall(art): + errors.add(f"{rel}: build token {tok} was not filled in (extension not loaded?)") + # holds from the + # llmstxt plugin, a legitimate .md href; the .md rule looks at the body only. + body = text.split("", 1)[-1] + for url in ATTR.findall(text): + t = resolve(page, url) + if t == "ABS": + errors.add(f"{rel}: absolute link {url} (outside {prefix})") + elif t is not None and not t.is_relative_to(site): + # ../LICENSE from the top page: the file exists in the checkout, not on Pages. + errors.add(f"{rel}: {url} leaves the built site (resolves to {t})") + elif t is not None and not t.exists(): + shown = t.relative_to(site) if t.is_relative_to(site) else t + errors.add(f"{rel}: {url} -> missing {shown}") + elif t is not None and unquote(urlparse(url).path).endswith(".md") and url in body_urls(body): + errors.add(f"{rel}: {url} links a Markdown source, not a page URL (raw HTML link?)") + copies = COPY.findall(text) + for url in copies: + t = resolve(page, url) + if isinstance(t, Path) and not t.is_file(): + errors.add(f"{rel}: Copy as Markdown points at {url}, which was not written (llmstxt sections?)") + if rel.name == "index.html" and not stub and not copies \ + and "content.action.copy" in ((cfg.get("theme") or {}).get("features") or []): + errors.add(f"{rel}: content.action.copy is on but the page has no Copy as Markdown button") + for url in NAV.findall(text): + t = resolve(page, url) + if isinstance(t, Path): + nav_targets.add(t) + + for page in pages: + if page.resolve() not in nav_targets: + errors.add(f"{page.relative_to(site)}: page is in no navigation entry (stray page)") + + if cfg: + plugins = cfg.get("plugins") or {} + for name in plugins: + if canonical_plugin(name) not in KNOWN_PLUGINS: + errors.add(f"zensical.toml: plugin {name!r} is unknown to Zensical and silently ignored") + theme = cfg.get("theme") or {} + for f in theme.get("features") or []: + if f not in KNOWN_FEATURES: + errors.add(f"zensical.toml: feature {f!r} is unknown to the Zensical UI and silently ignored") + variant = theme.get("variant", "classic") + if variant not in ("modern", "classic"): + errors.add(f"zensical.toml: theme.variant {variant!r} is not modern or classic (Zensical falls back to classic silently)") + elif variant == "modern" and "assets/stylesheets/modern/" not in index.read_text(errors="replace"): + errors.add("index.html does not load the modern stylesheet although theme.variant is modern") + # The offline plugin turns directory URLs off (zensical/config.py, + # _apply_offline_plugin), so a redirect stub becomes old.html there. + offline = plugins.get("offline", False) + offline_on = "offline" in plugins and (offline or {}).get("enabled", True) is not False + directory_urls = cfg.get("use_directory_urls", True) and not offline_on + for src in (plugins.get("redirects") or {}).get("redirect_maps") or {}: + s = src.removesuffix(".md") + # A README page is its folder's index: sub/README.md -> sub/index.html in every mode. + if s == "README" or s.endswith("/README"): + s = s[: -len("README")] + "index" + if not directory_urls or s == "index" or s.endswith("/index"): + stub = site / (s + ".html") + else: + stub = site / s / "index.html" + if not stub.is_file() or STUB not in stub.read_text(errors="replace"): + errors.add(f"redirect_maps {src!r}: no redirect stub at {stub.relative_to(site)}") + if "llmstxt" in plugins and not (site / "llms.txt").is_file(): + errors.add("llmstxt plugin is configured but site/llms.txt was not written") + + for e in sorted(errors): + print("ERROR", e) + print(f"site-check: {len(pages)} pages, {len(errors)} errors, prefix {prefix}") + return 1 if errors else 0 + + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3): + sys.exit("usage: site-check.py [zensical.toml]") + sys.exit(main(sys.argv[1], sys.argv[2] if len(sys.argv) == 3 else "zensical.toml")) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0a835bf33..e10ef927f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,20 +6,25 @@ name: Docs # so the source is "GitHub Actions" and there is no gh-pages branch. Until # then the deploy job fails, which is the test that the switch happened. # -# Pull requests that touch the docs run the strict build, so a broken link, a -# missing page or an unreachable page fails the check instead of reaching the -# site. A push to main deploys, and only after the same build has passed. +# Pull requests that touch the docs run the strict build and the site check, +# so a broken link, a missing page or an unreachable page fails the check +# instead of reaching the site. A push to main deploys, and only after the +# same build has passed. # -# The site is published as a Pages artifact, never with `mkdocs gh-deploy`: -# no gh-pages branch, no `contents: write`, and one artifact is the whole site. +# The site is published as a Pages artifact: no gh-pages branch, no +# `contents: write`, and one artifact is the whole site. on: push: branches: - main paths: - 'docs/**' - - 'mkdocs.yml' + - 'zensical.toml' - '.github/workflows/docs.yml' + - '.github/scripts/site-check.py' + # Add every file a page includes with pymdownx.snippets (for example + # docker-compose.yml, .env.example): a change to it must rebuild the + # site, and a rename must fail the build. # The live-site demo at /demo/ is built from these (tests/demo/build_live_demo.py). # assets/** because render_report.py also reads the gene and rule tables there. - 'assets/**' @@ -32,12 +37,11 @@ on: pull_request: workflow_dispatch: -# One run at a time per ref, and a running deploy is never cancelled. Locking -# only the deploy job is not enough: an older push whose build finishes later -# would deploy last and publish a stale site. GitHub does not promise push order -# for runs waiting on the lock, so in a rare race an older commit can still be -# published; the next push to the docs corrects it. Pull requests get their own -# group. +# One run at a time per ref, never cancelled. GitHub keeps one running and one +# pending run per group and a newer push replaces the pending one, so the last +# deploy is the newest push; it does not promise strict push order beyond that. +# Locking only the deploy job is not enough: an older push whose build finishes +# later could deploy last and publish a stale site. Pull requests get their own group. concurrency: group: pages-${{ github.ref }} cancel-in-progress: false @@ -54,41 +58,55 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # This job never pushes; a persisted token is readable by pip and every MkDocs plugin. + # This job never pushes; a persisted token is readable by pip and by the build. persist-credentials: false - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: '3.15' + # pip's download cache only. Zensical's own build cache (.cache/) is + # never kept between runs: its docs advise against it on CI, and + # --clean below starts from nothing anyway. cache: pip cache-dependency-path: docs/requirements-docs.txt - name: Install dependencies run: pip install -r docs/requirements-docs.txt - # mkdocs.yml sets `strict: true`; the flag here keeps the build strict - # even if someone removes that line. + # zensical.toml sets `strict = true`; the flag here keeps the build strict + # even if someone removes that line. `python -m zensical`, never the bare + # `zensical` command: it puts the repo root on sys.path, which the awesome + # lists' Markdown extension needs, and every repo builds the same way. - name: Build - run: mkdocs build --strict + run: python -m zensical build --clean --strict + + # What the strict build does not check: a page in no nav entry, a nav + # entry for a missing file, a missing image, a /rooted link, a raw + # in a page. Each one was proven to turn this step red (validate-zensical.sh). + - name: Check the built site + run: python -I .github/scripts/site-check.py site # /demo/: the live site replaying an invented run of DEMO-001, the invented # sample of the docs pictures, through the real page and plugin code # (tests/demo/build_live_demo.py; standard library only). It is a static - # folder beside the pages, not a page of the nav. + # folder beside the pages, not a page of the nav, so it goes in after the + # site check. - name: Add the live-site demo run: | python3 tests/demo/build_live_demo.py --out site/demo test -s site/demo/runs/DEMO-001/frames.json + # Push to main only: workflow_dispatch can be run on any branch, and the + # push filter above does not apply to it. Same line as docs-pages-helm.yml. - name: Upload the site - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site deploy: # Runs on push to main only, after the build job has passed. - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' needs: build runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index 7c7563248..f72f38fd6 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -51,7 +51,7 @@ jobs: run=false while IFS= read -r f; do case "$f" in - docs/*|*.md|mkdocs.yml|LICENSE) ;; + docs/*|*.md|zensical.toml|LICENSE) ;; *) run=true ;; esac done <<<"$changed" diff --git a/.gitignore b/.gitignore index c67331840..e09a812fb 100644 --- a/.gitignore +++ b/.gitignore @@ -94,7 +94,7 @@ ci.config # OMC / local agent operational state — may contain local paths or notes; never commit .omc/ -# MkDocs build output +# Docs build output site/ # Beads / Dolt files (added by bd init) diff --git a/CLAUDE.md b/CLAUDE.md index c6aedd66f..30d6960bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,7 +77,7 @@ owngenome/ nextflow.yml # Nextflow config, lint, schema and stub runs on the pinned release and the newest 26.10.x renovate-dry-run.yml # Renovate lookups without an app, and their checks freshness.yml # Monthly issue of pins that fell behind upstream - docs.yml # mkdocs build + docs.yml # Zensical build + site check release.yml, stale.yml # Releases, stale issues ``` @@ -119,7 +119,7 @@ User's FASTQ/BAM/VCF ### Documentation Conventions - Each pipeline step has a matching doc in `docs/XX-name.md` and script in `scripts/XX-name.sh` - Docs must include: What it does, Why, Tool name, Docker image, Command, Output, Runtime estimate, Notes -- The category table in `docs/pipeline-overview.md` and the `nav:` of `mkdocs.yml` must stay in sync with actual docs and scripts +- The category table in `docs/pipeline-overview.md` and the `nav` of `zensical.toml` must stay in sync with actual docs and scripts - All Docker images must include the exact tag (not floating) ### Lessons Learned @@ -131,7 +131,7 @@ User's FASTQ/BAM/VCF 1. Add the image as one line in `versions.env` (`setup.sh` and `validate-setup.sh` read their list from it) 2. Create `scripts/NN-tool-name.sh` following script conventions 3. Create `modules/local/<tool>/main.nf` with no `container` line, add its process to the table in `scripts/ci/gen-containers-config.sh` and run it; or note in `docs/nextflow.md` why the step stays bash-only. Adding a step means a module plus, optionally, its script -4. Create `docs/NN-tool-name.md` following existing template, and add it to `nav:` in `mkdocs.yml` +4. Create `docs/NN-tool-name.md` following existing template, and add it to `nav` in `zensical.toml` 5. Add the step to the category table in `docs/pipeline-overview.md` 6. Add the module to the step list (a `plan` call) in `scripts/run-all.sh` (its `--tools` name and database parameter), so a default run includes it 7. Add a row for the image to `tests/smoke/commands.tsv`: a real command on the fixture and a check on what it wrote (an image with no row fails `container-test.yml`) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 95dcf2c2e..f3382205e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ validate_sample "$SAMPLE" Every pipeline step needs: - `docs/NN-tool-name.md` — What it does, why, Docker image, command, output, runtime, notes - `scripts/NN-tool-name.sh` — The executable script -- An entry in the `nav:` of [`mkdocs.yml`](mkdocs.yml), or the docs build fails on the orphan page +- An entry in the `nav` of [`zensical.toml`](zensical.toml), or the docs site check fails on the orphan page - A mention in the category table of [`docs/pipeline-overview.md`](docs/pipeline-overview.md) - Section in `docs/interpreting-results.md` if the output needs explanation @@ -94,7 +94,7 @@ The CI pipeline enforces this with automated scanning. 5. **Create the Nextflow module** `modules/local/<tool>/main.nf` and wire it into the workflow, or say in [`docs/nextflow.md`](docs/nextflow.md) why the step stays bash-only. A module has no `container` line: add its process to the table in `scripts/ci/gen-containers-config.sh` and run that script. 6. **Create the documentation** `docs/NN-tool-name.md` following the existing step docs. 7. **Update these files:** - - [`mkdocs.yml`](mkdocs.yml): the page in `nav:` + - [`zensical.toml`](zensical.toml): the page in `nav` - [`docs/pipeline-overview.md`](docs/pipeline-overview.md): the category table - `scripts/run-all.sh`: one entry in its step list (a `plan` call), so a default run passes the module's name in `--tools` (and its database parameter when the data is under `GENOME_DIR`). A bash-only step that a default run should include gets a switch there instead, run after the pipeline - [`tests/smoke/commands.tsv`](tests/smoke/commands.tsv): a row for the image, a real command on the fixture and a check on what the tool wrote. [`container-test.yml`](.github/workflows/container-test.yml) fails for an image with no row diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 48b1d43d4..f01d4c2fa 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1 +1 @@ -mkdocs-material==9.8.0 +zensical==0.1.0 diff --git a/docs/stylesheets/theme.css b/docs/stylesheets/theme.css index 69d3603d6..5824741bd 100644 --- a/docs/stylesheets/theme.css +++ b/docs/stylesheets/theme.css @@ -4,8 +4,8 @@ * (#1c1917) and a row of base tiles in green #16a34a (A), blue #2563eb (C), * amber #d97706 (G) and red #dc2626 (T). The header inverts the banner: the * near-black as the surface, the cream as the text, the blue as the accent. - * mkdocs.yml sets `primary: custom` and `accent: custom`, so Material paints - * nothing until these variables exist. + * zensical.toml sets `primary: custom` and `accent: custom`, so the theme + * paints nothing until these variables exist. */ :root { --md-primary-fg-color: #1c1917; /* banner wordmark, the header surface */ @@ -45,11 +45,17 @@ * .md-header__topic and the page title in a second one, and swaps them on * scroll. Styling both would change every page title too. */ .md-header__topic:first-child .md-ellipsis { - color: #f6f1e7; font-weight: 800; letter-spacing: -0.01em; } +/* That colour only on the dark scheme: Zensical's modern header takes the page + background, so on the light scheme it is white and a light name would vanish. + There the name keeps the default text colour. */ +[data-md-color-scheme="slate"] .md-header__topic:first-child .md-ellipsis { + color: #f6f1e7; +} + /* Keep the mark and the name close, so they read as one unit. */ .md-header__button.md-logo { padding-right: 0.2rem; diff --git a/docs/updating.md b/docs/updating.md index f3b54e14e..e0d6532b3 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -16,7 +16,7 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | Python packages of the screenshot script | the `==` pins in `tests/demo/retake-screenshots.sh` | Renovate | | Aldy, the alternative CYP2D6 caller | its version in [Cyrius](21-cyrius.md), in the prose and the `pip install` line | Renovate | | GitHub Actions (`uses:` lines) | the workflows | Dependabot, monthly | -| MkDocs and its plugins | `docs/requirements-docs.txt` | Dependabot, monthly | +| Zensical, the docs site builder | `docs/requirements-docs.txt` | Dependabot, monthly | | Data releases: VEP cache, PCGR bundle, pypgx bundle, AnnotSV annotations, Yleaf data, gnomAD constraint, the GRIDSS blacklist commit | the variables at the end of `versions.env` | nobody: bump them by hand, together with the image they belong to. Guard's `coupled-versions` job fails while the VEP cache, pypgx bundle, AnnotSV or Yleaf data disagrees with its image | | GENCODE gene lines (step 8) and exons (steps 6 and 39) | `GENCODE_RELEASE` in `scripts/lib/common.sh` | nobody: it moves with a VEP major, to the GENCODE release of that Ensembl release (the VEP major minus 66). The `coupled-versions` job fails until it does | | VerifyBamID2 marker panel (step 33) | the tag in the `VB2_PANEL_FILES` URLs of `scripts/setup.sh`, with their sha256 | nobody: it moves with `VERIFYBAMID2_IMAGE`. The `coupled-versions` job fails while the tag and the image version differ | diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index e42f84d68..000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,240 +0,0 @@ -site_name: OwnGenome -site_url: https://geiserx.github.io/OwnGenome/ -site_description: Whole genome sequencing analysis pipeline for consumer hardware. 100% local, Docker-powered, free and open source. -repo_url: https://github.com/GeiserX/OwnGenome -repo_name: GeiserX/OwnGenome -edit_uri: edit/main/docs/ - -# A warning is a broken link, a missing page or a page nobody can reach. All of -# them fail the build, locally and in CI, so nobody has to remember --strict. -strict: true - -theme: - name: material - language: en - logo: images/logo.svg - favicon: images/icon.svg - # Dark first: the first palette entry is the default. - palette: - - scheme: slate - primary: custom - accent: custom - toggle: - icon: material/brightness-4 - name: Switch to light mode - - scheme: default - primary: custom - accent: custom - toggle: - icon: material/brightness-7 - name: Switch to dark mode - features: - - navigation.tabs - - navigation.sections - - navigation.expand - - navigation.top - - content.code.copy - - content.action.edit - - search.highlight - - search.suggest - - toc.follow - icon: - repo: fontawesome/brands/github - -# MkDocs reports most link problems as `info`, which --strict ignores. Every -# line here promotes one of them to a warning so the build fails on it: -# nav.omitted_files a Markdown file under docs/ that is in no nav entry and -# not in exclude_docs (a page nobody can reach, published -# at a public URL anyway); -# nav.not_found a nav entry whose file does not exist; -# links.not_found a relative link to a page or file that does not exist; -# links.anchors a link to a heading that does not exist; -# links.absolute_links a link written as /path/ (breaks under the -# /OwnGenome/ prefix of the site); -# links.unrecognized_links a relative link that leaves docs/ (../README.md). -# Repo files are linked by absolute GitHub URL instead. -validation: - nav: - omitted_files: warn - not_found: warn - absolute_links: warn - links: - not_found: warn - anchors: warn - absolute_links: warn - unrecognized_links: warn - -plugins: - - search - # No page has been renamed, so there is nothing to redirect. When one is, - # add `mkdocs-redirects==1.2.2` to docs/requirements-docs.txt and uncomment. - # - redirects: - # redirect_maps: - # old-name.md: new-name.md - -markdown_extensions: - - admonition - - pymdownx.details - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.tabbed: - alternate_style: true - - pymdownx.highlight: - anchor_linenums: true - - pymdownx.inlinehilite - # base_path includes the repository root, so a page can include a file that - # ships (versions.env, nextflow.config) word for word with - # `--8<-- "versions.env"`. A copied example drifts from the real file; - # an included one cannot. check_paths makes a missing include fail the build - # instead of rendering an empty block. No page uses it yet. - - pymdownx.snippets: - base_path: - - . - - docs - check_paths: true - - pymdownx.mark - - attr_list - - md_in_html - # Renders :material-xxx: shortcodes as icons. The grid cards on index.md use - # them; without this they print as literal text. - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - - tables - - toc: - permalink: true - -extra: - social: - - icon: fontawesome/brands/github - link: https://github.com/GeiserX/OwnGenome - -extra_css: - - stylesheets/theme.css - -# Files under docs/ that are not pages. MkDocs publishes every Markdown file in -# docs/ whether or not it is in the nav, so anything that is not a page must be -# listed here. A page is a lowercase kebab-case .md file; anything with a -# capital letter is a repo-facing note, and research/ holds the triage and -# state-of-the-art notes read on GitHub. Nothing under research/ is published. -exclude_docs: | - [A-Z]*.md - design/ - research/ - notes/ - requirements-docs.txt - -# Every page, grouped by what a reader is doing. The nav title equals the -# page's H1, except where one page holds two steps (35 and 35b). Step pages -# are grouped by pipeline stage; the numbers stay in the titles because the -# scripts, the output directories and every other page refer to steps by -# number. Step 38 lives in the multi-sample guide, reached from the step -# table. -nav: - - Home: index.md - - Get started: - - Getting started: getting-started.md - - Hardware and Storage Requirements: hardware-requirements.md - - Why run locally?: why-local.md - - Vendor Compatibility Guide: vendor-guide.md - - Starting from a Vendor VCF: vcf-first.md - - "Step 0: Reference Data Setup": 00-reference-setup.md - - Realigning After a Reference Change: realignment.md - - "Quick Test: Verify Your Setup Before Running on Real Data": quick-test.md - - Nextflow Execution (v0.5.0): nextflow.md - - Pipeline steps: - - Pipeline overview: pipeline-overview.md - - Reads and small variants: - - "Step 1: ORA to FASTQ Conversion": 01-ora-to-fastq.md - - "Step 1b: fastp QC + Adapter Trimming": 01b-fastp-qc.md - - "Step 2: Alignment (FASTQ to BAM)": 02-alignment.md - - "Step 3: Variant Calling (BAM to VCF)": 03-variant-calling.md - - "Step 3d: Octopus Variant Caller (Alternative)": 03d-octopus.md - - Structural and copy number variants: - - "Step 4: Structural Variant Calling (Manta)": 04-structural-variants.md - - "Step 4b: GRIDSS Structural Variant Calling": 04b-gridss.md - - "Step 5: Structural Variant Annotation (AnnotSV)": 05-annotsv.md - - "Step 15: SV Quality Annotation with duphold": 15-duphold.md - - "Step 18: Depth-Based CNV Calling with CNVpytor": 18-cnvpytor.md - - "Step 19: Structural Variant Calling with Delly": 19-delly.md - - "Step 22: Structural Variant Consensus Merge": 22-survivor-merge.md - - "Step 50: Mobile Element Insertions (Alu, L1, SVA)": 50-mei.md - - Clinical screening and annotation: - - "Step 6: ClinVar Pathogenic Variant Screening": 06-clinvar-screen.md - - "Step 13: Variant Effect Predictor (VEP) Annotation": 13-vep-annotation.md - - "Step 17: Cancer Predisposition Screening with CPSR": 17-cpsr.md - - "Step 23: Clinical Variant Filter": 23-clinical-filter.md - - "Step 35 and 35b: Paralog Genes (Parascopy, Paraphase)": 35-paralogs.md - - "Step 39: Couple Carrier Screen": 39-couple.md - - "Step 48: Symptom-Driven Variant Ranking (Exomiser)": 48-exomiser.md - - "Step 29: Somatic Variant Calling (Mutect2 Tumor-Only) [EXPERIMENTAL]": 29-mutect2-somatic.md - - "Step 30: Annotation Enrichment (vcfanno)": 30-vcfanno.md - - "Step 31: Variant Prioritization (slivar)": 31-slivar.md - - Pharmacogenomics: - - "Step 7: Pharmacogenomics (PharmCAT)": 07-pharmacogenomics.md - - "Step 21: CYP2D6 Star Allele Calling with Cyrius": 21-cyrius.md - - "Step 27: CPIC Drug-Gene Recommendation Lookup": 27-cpic-lookup.md - - "Step 32: Comprehensive Pharmacogenomics (pypgx)": 32-pypgx.md - - "Step 36: PGx Consensus (PharmCAT Outside Calls)": 36-pgx-consensus.md - - HLA, repeats and telomeres: - - "Step 8: HLA Typing (T1K)": 08-hla-typing.md - - "Step 9: Short Tandem Repeat (STR) Expansion Screening": 09-str-expansions.md - - "Step 9b: STR Clinical Annotation (Stranger)": 09b-stranger.md - - "Step 10: Telomere Length Estimation": 10-telomere-analysis.md - - Ancestry, risk and mitochondria: - - "Step 11: Runs of Homozygosity (ROH) Analysis": 11-roh-analysis.md - - "Step 12: Mitochondrial Haplogroup": 12-mito-haplogroup.md - - "Step 14: Imputation Preparation": 14-imputation-prep.md - - "Step 20: Mitochondrial Variant Calling and Heteroplasmy Detection": 20-mtoolbox.md - - "Step 25: Polygenic Risk Scores (PRS)": 25-prs.md - - "Step 26: Ancestry (projection onto a reference panel)": 26-ancestry.md - - "Step 26b: Fine-Grained Ancestry (admixture fractions and a chromosome painting)": 26b-ancestry-fine.md - - "Step 37: Y-Chromosome Haplogroup": 37-y-haplogroup.md - - "Step 40: Raw-Data Export (23andMe Layout)": 40-export-consumer.md - - "Step 41: Blood Groups (predicted)": 41-bloodtype.md - - "Step 42: Traits Panel": 42-traits.md - - "Step 43: Archaic Ancestry (Neanderthal and Denisovan)": 43-archaic.md - - "Step 44: Ancient Populations (AADR PCA)": 44-ancient.md - - "Step 45: Y-STR Markers": 45-y-str.md - - "Step 46: Lp(a), the LPA KIV-2 Repeat Count": 46-lpa.md - - "Step 47: Viral DNA in the Unmapped Reads": 47-viral.md - - "Step 49: Clonal Haematopoiesis (CHIP)": 49-chip.md - - "Step 51: Mosaic Chromosomal Alterations (MoChA)": 51-mocha.md - - "Step 52: MUC1 VNTR Frameshifts (VNtyper)": 52-vntyper.md - - "Step 53: Hard Loci Second Opinion (Locityper)": 53-locityper.md - - Quality control and reports: - - "Step 16: Coverage QC and Sex Chromosome Verification": 16-indexcov.md - - "Step 16b: mosdepth Coverage Statistics": 16b-mosdepth.md - - "Step 24: HTML Report": 24-html-report.md - - What Changed Since the Previous Run: delta-report.md - - Newer Databases Since a Run: recheck.md - - Where a Run Stands: run-state.md - - Run Alerts: alerts.md - - Look Up a Variant, an rsID or a Gene: lookup.md - - "Step 28: MultiQC Aggregated QC Report": 28-multiqc.md - - "Step 33: Sample Identity and Contamination": 33-sample-qc.md - - "Step 34: CRAM Archive": 34-cram-archive.md - - Guides: - - Interpreting Your Results: interpreting-results.md - - What the Pipeline Does Not Report: not-reported.md - - "Genome Viewer: See the Reads Behind a Finding": viewer.md - - Watch a Run Live: live-site.md - - Multi-Sample Comparison Guide: multi-sample.md - - Long-Read Sequencing Guide: long-read-guide.md - - Whole Exome Sequencing (WES) Guide: wes-guide.md - - "Using This Pipeline with Genotyping Array Data (23andMe, MyHeritage, AncestryDNA)": chip-data-guide.md - - Variant Caller Benchmarking: benchmarking.md - - Reference: - - Common issues and FAQ: faq.md - - Troubleshooting Guide: troubleshooting.md - - Lessons Learned: lessons-learned.md - - Tool Selection Rationale: tool-rationale.md - - Image versions: versions.md - - "Copy Number from Depth (the locus catalog)": cn-loci.md - - Testing: testing.md - - Keeping versions up to date: updating.md - - Glossary: glossary.md - - Recommended Resources: resources.md diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 000000000..2c4506752 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,203 @@ +# Generated from mkdocs.yml by mkdocs-to-zensical.py; the why of every block is in +# new-repo/templates/zensical.toml. Edit this file, not mkdocs.yml, which is gone. + +[project] +site_name = "OwnGenome" +site_url = "https://geiserx.github.io/OwnGenome/" +site_description = "Whole genome sequencing analysis pipeline for consumer hardware. 100% local, Docker-powered, free and open source." +repo_url = "https://github.com/GeiserX/OwnGenome" +repo_name = "GeiserX/OwnGenome" +edit_uri = "edit/main/docs/" +strict = true +extra_css = ["stylesheets/theme.css"] +exclude_docs = """ +[A-Z]*.md +design/ +research/ +notes/ +requirements-docs.txt +""" +nav = [ + { Home = "index.md" }, + { "Get started" = [ + { "Getting started" = "getting-started.md" }, + { "Hardware and Storage Requirements" = "hardware-requirements.md" }, + { "Why run locally?" = "why-local.md" }, + { "Vendor Compatibility Guide" = "vendor-guide.md" }, + { "Starting from a Vendor VCF" = "vcf-first.md" }, + { "Step 0: Reference Data Setup" = "00-reference-setup.md" }, + { "Realigning After a Reference Change" = "realignment.md" }, + { "Quick Test: Verify Your Setup Before Running on Real Data" = "quick-test.md" }, + { "Nextflow Execution (v0.5.0)" = "nextflow.md" }, + ] }, + { "Pipeline steps" = [ + { "Pipeline overview" = "pipeline-overview.md" }, + { "Reads and small variants" = [ + { "Step 1: ORA to FASTQ Conversion" = "01-ora-to-fastq.md" }, + { "Step 1b: fastp QC + Adapter Trimming" = "01b-fastp-qc.md" }, + { "Step 2: Alignment (FASTQ to BAM)" = "02-alignment.md" }, + { "Step 3: Variant Calling (BAM to VCF)" = "03-variant-calling.md" }, + { "Step 3d: Octopus Variant Caller (Alternative)" = "03d-octopus.md" }, + ] }, + { "Structural and copy number variants" = [ + { "Step 4: Structural Variant Calling (Manta)" = "04-structural-variants.md" }, + { "Step 4b: GRIDSS Structural Variant Calling" = "04b-gridss.md" }, + { "Step 5: Structural Variant Annotation (AnnotSV)" = "05-annotsv.md" }, + { "Step 15: SV Quality Annotation with duphold" = "15-duphold.md" }, + { "Step 18: Depth-Based CNV Calling with CNVpytor" = "18-cnvpytor.md" }, + { "Step 19: Structural Variant Calling with Delly" = "19-delly.md" }, + { "Step 22: Structural Variant Consensus Merge" = "22-survivor-merge.md" }, + { "Step 50: Mobile Element Insertions (Alu, L1, SVA)" = "50-mei.md" }, + ] }, + { "Clinical screening and annotation" = [ + { "Step 6: ClinVar Pathogenic Variant Screening" = "06-clinvar-screen.md" }, + { "Step 13: Variant Effect Predictor (VEP) Annotation" = "13-vep-annotation.md" }, + { "Step 17: Cancer Predisposition Screening with CPSR" = "17-cpsr.md" }, + { "Step 23: Clinical Variant Filter" = "23-clinical-filter.md" }, + { "Step 35 and 35b: Paralog Genes (Parascopy, Paraphase)" = "35-paralogs.md" }, + { "Step 39: Couple Carrier Screen" = "39-couple.md" }, + { "Step 48: Symptom-Driven Variant Ranking (Exomiser)" = "48-exomiser.md" }, + { "Step 29: Somatic Variant Calling (Mutect2 Tumor-Only) [EXPERIMENTAL]" = "29-mutect2-somatic.md" }, + { "Step 30: Annotation Enrichment (vcfanno)" = "30-vcfanno.md" }, + { "Step 31: Variant Prioritization (slivar)" = "31-slivar.md" }, + ] }, + { Pharmacogenomics = [ + { "Step 7: Pharmacogenomics (PharmCAT)" = "07-pharmacogenomics.md" }, + { "Step 21: CYP2D6 Star Allele Calling with Cyrius" = "21-cyrius.md" }, + { "Step 27: CPIC Drug-Gene Recommendation Lookup" = "27-cpic-lookup.md" }, + { "Step 32: Comprehensive Pharmacogenomics (pypgx)" = "32-pypgx.md" }, + { "Step 36: PGx Consensus (PharmCAT Outside Calls)" = "36-pgx-consensus.md" }, + ] }, + { "HLA, repeats and telomeres" = [ + { "Step 8: HLA Typing (T1K)" = "08-hla-typing.md" }, + { "Step 9: Short Tandem Repeat (STR) Expansion Screening" = "09-str-expansions.md" }, + { "Step 9b: STR Clinical Annotation (Stranger)" = "09b-stranger.md" }, + { "Step 10: Telomere Length Estimation" = "10-telomere-analysis.md" }, + ] }, + { "Ancestry, risk and mitochondria" = [ + { "Step 11: Runs of Homozygosity (ROH) Analysis" = "11-roh-analysis.md" }, + { "Step 12: Mitochondrial Haplogroup" = "12-mito-haplogroup.md" }, + { "Step 14: Imputation Preparation" = "14-imputation-prep.md" }, + { "Step 20: Mitochondrial Variant Calling and Heteroplasmy Detection" = "20-mtoolbox.md" }, + { "Step 25: Polygenic Risk Scores (PRS)" = "25-prs.md" }, + { "Step 26: Ancestry (projection onto a reference panel)" = "26-ancestry.md" }, + { "Step 26b: Fine-Grained Ancestry (admixture fractions and a chromosome painting)" = "26b-ancestry-fine.md" }, + { "Step 37: Y-Chromosome Haplogroup" = "37-y-haplogroup.md" }, + { "Step 40: Raw-Data Export (23andMe Layout)" = "40-export-consumer.md" }, + { "Step 41: Blood Groups (predicted)" = "41-bloodtype.md" }, + { "Step 42: Traits Panel" = "42-traits.md" }, + { "Step 43: Archaic Ancestry (Neanderthal and Denisovan)" = "43-archaic.md" }, + { "Step 44: Ancient Populations (AADR PCA)" = "44-ancient.md" }, + { "Step 45: Y-STR Markers" = "45-y-str.md" }, + { "Step 46: Lp(a), the LPA KIV-2 Repeat Count" = "46-lpa.md" }, + { "Step 47: Viral DNA in the Unmapped Reads" = "47-viral.md" }, + { "Step 49: Clonal Haematopoiesis (CHIP)" = "49-chip.md" }, + { "Step 51: Mosaic Chromosomal Alterations (MoChA)" = "51-mocha.md" }, + { "Step 52: MUC1 VNTR Frameshifts (VNtyper)" = "52-vntyper.md" }, + { "Step 53: Hard Loci Second Opinion (Locityper)" = "53-locityper.md" }, + ] }, + { "Quality control and reports" = [ + { "Step 16: Coverage QC and Sex Chromosome Verification" = "16-indexcov.md" }, + { "Step 16b: mosdepth Coverage Statistics" = "16b-mosdepth.md" }, + { "Step 24: HTML Report" = "24-html-report.md" }, + { "What Changed Since the Previous Run" = "delta-report.md" }, + { "Newer Databases Since a Run" = "recheck.md" }, + { "Where a Run Stands" = "run-state.md" }, + { "Run Alerts" = "alerts.md" }, + { "Look Up a Variant, an rsID or a Gene" = "lookup.md" }, + { "Step 28: MultiQC Aggregated QC Report" = "28-multiqc.md" }, + { "Step 33: Sample Identity and Contamination" = "33-sample-qc.md" }, + { "Step 34: CRAM Archive" = "34-cram-archive.md" }, + ] }, + ] }, + { Guides = [ + { "Interpreting Your Results" = "interpreting-results.md" }, + { "What the Pipeline Does Not Report" = "not-reported.md" }, + { "Genome Viewer: See the Reads Behind a Finding" = "viewer.md" }, + { "Watch a Run Live" = "live-site.md" }, + { "Multi-Sample Comparison Guide" = "multi-sample.md" }, + { "Long-Read Sequencing Guide" = "long-read-guide.md" }, + { "Whole Exome Sequencing (WES) Guide" = "wes-guide.md" }, + { "Using This Pipeline with Genotyping Array Data (23andMe, MyHeritage, AncestryDNA)" = "chip-data-guide.md" }, + { "Variant Caller Benchmarking" = "benchmarking.md" }, + ] }, + { Reference = [ + { "Common issues and FAQ" = "faq.md" }, + { "Troubleshooting Guide" = "troubleshooting.md" }, + { "Lessons Learned" = "lessons-learned.md" }, + { "Tool Selection Rationale" = "tool-rationale.md" }, + { "Image versions" = "versions.md" }, + { "Copy Number from Depth (the locus catalog)" = "cn-loci.md" }, + { Testing = "testing.md" }, + { "Keeping versions up to date" = "updating.md" }, + { Glossary = "glossary.md" }, + { "Recommended Resources" = "resources.md" }, + ] }, +] + +[project.validation] +invalid_links = true +invalid_link_anchors = true + +[project.theme] +language = "en" +variant = "modern" +logo = "images/logo.svg" +favicon = "images/icon.svg" +font = false +features = [ + "navigation.tabs", + "navigation.sections", + "navigation.expand", + "navigation.top", + "content.code.copy", + "content.action.edit", + "search.highlight", + "toc.follow", + "navigation.instant", + "navigation.instant.progress", + "content.action.copy", +] +icon.repo = "fontawesome/brands/github" + +[[project.theme.palette]] +scheme = "slate" +primary = "custom" +accent = "custom" +toggle.icon = "lucide/sun" +toggle.name = "Switch to light mode" + +[[project.theme.palette]] +scheme = "default" +primary = "custom" +accent = "custom" +toggle.icon = "lucide/moon" +toggle.name = "Switch to dark mode" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/GeiserX/OwnGenome" + +[project.plugins.search] + +[project.plugins.llmstxt.sections] +Documentation = ["*.md", "*/*.md"] + +[project.markdown_extensions] +admonition = {} +"pymdownx.details" = {} +"pymdownx.superfences".custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, +] +"pymdownx.tabbed".alternate_style = true +"pymdownx.highlight".anchor_linenums = true +"pymdownx.inlinehilite" = {} +"pymdownx.snippets".base_path = [".", "docs"] +"pymdownx.snippets".check_paths = true +"pymdownx.mark" = {} +attr_list = {} +md_in_html = {} +"pymdownx.emoji".emoji_index = "zensical.extensions.emoji.twemoji" +"pymdownx.emoji".emoji_generator = "zensical.extensions.emoji.to_svg" +tables = {} +toc.permalink = true From 30bc2afb7876d472fa1e62646a0442e87c46d5cc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:09:27 +0100 Subject: [PATCH 2/4] docs: make the zensical.toml header stand on its own --- zensical.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zensical.toml b/zensical.toml index 2c4506752..52689772f 100644 --- a/zensical.toml +++ b/zensical.toml @@ -1,5 +1,10 @@ -# Generated from mkdocs.yml by mkdocs-to-zensical.py; the why of every block is in -# new-repo/templates/zensical.toml. Edit this file, not mkdocs.yml, which is gone. +# The docs site, built with Zensical and published at the site_url below. +# Build: `python -m zensical build --clean --strict`, then +# `python -I .github/scripts/site-check.py site` (the Docs workflow runs both). +# The strict build fails on a broken link or anchor. site-check.py fails on what +# the build does not check: a page in no nav entry, a nav entry for a missing +# file, a missing image, a link rooted at /. exclude_docs lists what under docs/ +# is not a page; the nav title of each page equals its H1. [project] site_name = "OwnGenome" From 528606de872624a0833b286403e2104873fe315d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:03:52 +0100 Subject: [PATCH 3/4] fix(docs): take the fixed shared site check The old check found href and src values with a regex that only matched double quotes, so a missing image or link written as raw HTML with single quotes or no quotes passed. The shared check now parses the HTML, so those fail too, and it also checks #fragments, full URLs into the site and the plugin keys the way Zensical reads them. The Docs workflow comment about the concurrency group now says what GitHub actually promises. --- .github/scripts/site-check.py | 216 +++++++++++++++++++++++++--------- .github/workflows/docs.yml | 5 +- 2 files changed, 162 insertions(+), 59 deletions(-) diff --git a/.github/scripts/site-check.py b/.github/scripts/site-check.py index 31a8df514..03b449f44 100644 --- a/.github/scripts/site-check.py +++ b/.github/scripts/site-check.py @@ -10,14 +10,19 @@ Exit 1 on any of these, each printed as one ERROR line: - a link or image whose target is not in the built site: a nav entry whose page does not - exist, a missing image, any other dangling href or src (Zensical validates Markdown page - links and anchors; it does not validate images, nav entries or raw HTML links). A redirect - stub's own links (canonical and body) are checked the same way; + exist, a missing image, any other dangling href or src, in any quoting and with HTML + entities decoded (Zensical validates Markdown page links; it does not validate images, + nav entries or raw HTML links). A full URL into the site itself + (https://geiserx.github.io/<repo>/...) is checked the same way. A redirect stub's own + links (canonical and body) are checked too; + - a link whose #fragment is not an id on the target page, including links that come in + through a snippet include (Zensical 0.1.0 does not check those; MkDocs strict did); - an href ending in .md: a correct page link is rewritten to a directory URL, so a .md left in the HTML is a raw-HTML or unprocessed link, and it opens the llmstxt Markdown copy; - a root-absolute link (/getting-started/) outside the site's own path prefix: it breaks under https://geiserx.github.io/<repo>/ (MkDocs' `absolute_links: warn`, which Zensical - ignores); + ignores). The prefix comes from site_url, else the home page's canonical link; with + neither, that is an error rather than a check that cannot fire; - a page that no navigation entry links to: a stray file under docs/ published at a public URL that nothing reaches (MkDocs' `nav.omitted_files: warn`, which Zensical ignores); - a .py, .pyc or __pycache__ entry in the site: the awesome lists keep their Markdown @@ -25,12 +30,12 @@ - an awesome list's count token left in a page (`<!-- n:proyectos -->`, `n:categorias`, `n:retirados`): the Markdown extension that fills them in did not run. Only those three names: CashPilot writes other `<!-- n:... -->` comments of its own in its pages; - - a raw <title>, <textarea>, <xmp>, <plaintext>, <noembed> or <noframes> tag inside a page's - article: a browser's HTML parser treats everything after it as text, so the rest of the - page disappears (Markdown passes `<title>` written in prose through as a real tag; - Telegram-Archive shipped that way); - - a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist: the - llmstxt plugin writes the copies, and only for pages its `sections` globs match. + - a raw <title>, or an unclosed <textarea>, <xmp>, <noembed>, <noframes>, or any + <plaintext>, in a page's article outside inline SVG and MathML: a browser's HTML parser + treats everything after it as text, so the rest of the page disappears (Markdown passes + `<title>` written in prose through as a real tag; Telegram-Archive shipped that way); + - a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist in the + site: the llmstxt plugin writes the copies, and only for pages its `sections` globs match. And, from zensical.toml (Zensical drops what it does not know without a word; under MkDocs strict an unknown plugin aborted the build): - a plugin name Zensical 0.1.0 does not know (`redirect` for `redirects`, `llmstxtt`): the @@ -40,23 +45,25 @@ - `theme.variant` other than "modern" or "classic" (anything else is classic, silently), and a "modern" config whose index.html does not load the modern stylesheet; - a `redirect_maps` entry whose stub was not written; - - a `[project.plugins.llmstxt]` table with no site/llms.txt, or `content.action.copy` in - the features with a page that carries no copy button. + - an enabled llmstxt plugin with no site/llms.txt, or `content.action.copy` in the + features with a page that carries no copy button. +Plugin keys are read the way Zensical reads them (`material/redirects` is `redirects`), and +a plugin with `enabled = false` is treated as off. With directory URLs off (the offline +plugin turns them off) every .html file is a page. Stdlib only (tomllib needs Python 3.11+), so it runs under `python -I` on any runner. """ import re import sys import tomllib +from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urlparse SWALLOW = re.compile(r"<(title|textarea|xmp|plaintext|noembed|noframes)\b", re.IGNORECASE) +FOREIGN = re.compile(r"<(svg|math)\b.*?</\1\s*>", re.IGNORECASE | re.DOTALL) TOKEN = re.compile(r"<!--\s*n:(?:proyectos|categorias|retirados)\s*-->") ARTICLE = re.compile(r"<article\b.*?</article>", re.DOTALL) -ATTR = re.compile(r'\s(?:href|src)="([^"]*)"') -COPY = re.compile(r'\sdata-md-copy-url="([^"]*)"') -NAV = re.compile(r'<a\s+href="([^"]*)"\s+class="md-(?:nav|tabs)__link') STUB = "<title>Redirecting..." # Keys of _PLUGIN_UNSUPPORTED_OPTIONS in zensical/config.py (0.1.0, same set as 0.0.74): every other name is @@ -85,16 +92,86 @@ } -def body_urls(body: str) -> set[str]: - return set(ATTR.findall(body)) + + +class Page(HTMLParser): + """The parts of one built HTML file the checks read, from a real HTML parser: any + quoting, entities decoded, nothing taken from