From 852f4a4054611ba6a83f26ce0481ac8c73480780 Mon Sep 17 00:00:00 2001 From: Adil Faisal Date: Tue, 15 Sep 2026 12:21:32 -0400 Subject: [PATCH] docs: add auto-generated API reference site Adds an mkdocs-material site whose API reference is generated from source rather than hand-maintained, plus a Pages deploy workflow. The reference is derived from each subpackage's __all__: scripts/gen_api.py walks __all__ -> one page per subpackage + the nav. Subpackages without __all__ (components.py, utils/) fall back to an AST scan of their source files. Adding an export is therefore all it takes for it to appear in the docs; no config, nav, or symbol-list edit is needed. Verified by injecting a class into controllers/ and observing the page grow from 7 to 8 exports. scripts/sphinx_compat.py is a griffe extension handling the Sphinx-flavored docstring markup used throughout the source (:class:, :meth:, :func:, :math:, and .. math:: blocks), which griffe's Google parser does not understand and would otherwise render as literal text. Without it the docs build still succeeds but every formula is broken, so the key must stay under `options:` in mkdocs.yml rather than at handler level. Generated output (docs/reference/, docs/SUMMARY.md, site/) is gitignored; CI regenerates it on every run. Notes: - The workflow deliberately omits --strict: griffe reports the unannotated public parameters as warnings (204 today), so --strict fails out of the box. Worth adding once annotation coverage improves. - 53% of public symbols have docstrings, so roughly 281 render as bare signatures. This is a source-coverage limit, not a tooling one. - Prose pages remain hand-written and are ordered in docs/_nav_prose.md. Commands: make docs, make docs-serve, make docs-build. --- .github/workflows/docs.yml | 64 ++++++++++++++++ .gitignore | 6 ++ AGENTS.md | 18 +++++ Makefile | 19 +++++ docs/_nav_prose.md | 19 +++++ docs/index.md | 59 +++++++++++++++ docs/javascripts/mathjax.js | 20 +++++ mkdocs.yml | 99 ++++++++++++++++++++++++ requirements-docs.txt | 6 ++ scripts/gen_api.py | 146 ++++++++++++++++++++++++++++++++++++ scripts/sphinx_compat.py | 79 +++++++++++++++++++ 11 files changed, 535 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/_nav_prose.md create mode 100644 docs/index.md create mode 100644 docs/javascripts/mathjax.js create mode 100644 mkdocs.yml create mode 100644 requirements-docs.txt create mode 100644 scripts/gen_api.py create mode 100644 scripts/sphinx_compat.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..82f134d --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,64 @@ +# FILE: .github/workflows/docs.yml +name: docs + +on: + push: + branches: [main] + paths: + - "src/**" + - "docs/**" + - "scripts/gen_api.py" + - "scripts/sphinx_compat.py" + - "mkdocs.yml" + - "requirements-docs.txt" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +# Let one deploy finish rather than cancelling mid-publish. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install deps + run: | + pip install -r requirements.txt + pip install -r requirements-docs.txt + + - name: Regenerate reference pages + nav from source + run: python scripts/gen_api.py + + - name: Build site + # NOT --strict: griffe flags the unannotated public parameters as + # warnings, which would fail the build out of the box. Add --strict once + # annotation coverage is up. + run: mkdocs build + + - uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index ba9491d..9af03ef 100644 --- a/.gitignore +++ b/.gitignore @@ -237,3 +237,9 @@ __marimo__/ # opencode-mem preseed (ephemeral, regenerate per-repo) lab-notes/memories.json + +# --- Docs (generated by scripts/gen_api.py) --- +site/ +docs/reference/ +docs/SUMMARY.md +docs/_nav_api.yml diff --git a/AGENTS.md b/AGENTS.md index 6815d57..b69e092 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,24 @@ joint space**. backend test "didn't run", that's why. Requires Python ≥3.12; CI matrix is 3.12–3.14. - Demos: `python -m demos.demo_*`. +- **Docs** (API reference is generated, not hand-written): + - `make docs` — install the docs toolchain (`requirements-docs.txt`). + - `make docs-serve` — regenerate reference + serve at http://127.0.0.1:8000. + - `make docs-build` — regenerate reference + build static `site/`. + - `scripts/gen_api.py` walks each subpackage's `__all__` and emits one page + per subpackage plus the nav — **adding an export is all it takes** for it to + appear in the docs. Subpackages without `__all__` (`components.py`, + `utils/`) fall back to an AST scan of their source files. + - Generated output (`docs/reference/`, `docs/SUMMARY.md`, `site/`) is + gitignored. Prose pages are hand-written and ordered in + `docs/_nav_prose.md`. + - `scripts/sphinx_compat.py` is a griffe extension that makes the Sphinx-flavored + docstring markup (`:class:`, `:math:`, `.. math::`) render instead of leaking + as literal text. Removing it silently breaks all math **without failing the + build** — the key must stay under `options:` in `mkdocs.yml`. + - `.github/workflows/docs.yml` deploys to GitHub Pages on push to `main`. + It does **not** use `--strict`: griffe flags the many unannotated public + parameters as warnings, so `--strict` is red out of the box. - `make compile SCENARIO=` — e2e scenario → verified `.so`: `scripts/gen_scenario.py` (zig-free: trace+compose+lower to an isolated path) then `scripts/build_scenario.py` (zig build + oracle + stamp + verify). diff --git a/Makefile b/Makefile index 63deaec..b9db468 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: test test-quick test-functional test-all test-integration lint install build \ release-patch release-minor release-major changelog \ + docs docs-serve docs-build \ test-controllers test-estimators test-plants test-trajectories test-armrobot \ test-components test-array-backend test-batched-adapter test-controllability test-factories \ test-linearization test-adversarial test-mcp-server test-mcp-functional \ @@ -129,6 +130,24 @@ lint: ruff check . pyright src/shinro/utils/ src/shinro/components.py src/shinro/controllers/ src/shinro/estimators/ src/shinro/trajectories/ src/shinro/plants/ +# --- Docs ------------------------------------------------------------------- +# The API reference pages AND their nav are generated from each subpackage's +# __all__ (see scripts/gen_api.py), so there is no symbol list to maintain. + +# Install the docs toolchain. +docs: + pip install -r requirements-docs.txt + +# Regenerate the reference and serve with live reload at http://127.0.0.1:8000 +docs-serve: + python3 scripts/gen_api.py + mkdocs serve + +# Regenerate the reference and build the static site into site/ +docs-build: + python3 scripts/gen_api.py + mkdocs build + # Run an individual test group by short name, e.g. `make test-controllers` test-controllers: python3 -m pytest tests/test_controllers.py -v --tb=short diff --git a/docs/_nav_prose.md b/docs/_nav_prose.md new file mode 100644 index 0000000..4d4bb77 --- /dev/null +++ b/docs/_nav_prose.md @@ -0,0 +1,19 @@ +# Hand-maintained navigation for the PROSE pages. +# +# Format is mkdocs-literate-nav markdown (NOT yaml): +# * [Title](path.md) +# * Section title +# * [Sub page](sub/page.md) +# +# Edit this file to add or reorder prose pages — it is safe from regeneration. +# scripts/gen_api.py reads it, appends the generated "API Reference" section, and +# writes the combined result to docs/SUMMARY.md (a build artifact — never edit +# SUMMARY.md or docs/_nav_api.yml by hand). + +* [Home](index.md) +* [Quickstart](quickstart.md) +* [How it works](how-it-works.md) +* [Components](components.md) +* [Testing](testing.md) +* [MCP server](mcp_server.md) +* [Codegen](codegen.md) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..fa0d15f --- /dev/null +++ b/docs/index.md @@ -0,0 +1,59 @@ +--- +title: Home +--- + +# shinro + +Whole-body control framework: controllers, plants, estimators, trajectories, and +a MuJoCo-backed simulation factory. + +`shinro` is built on **five abstract base classes** — `Controller`, `Plant`, +`StateEstimator`, `TrajectoryGenerator`, and `PhysicsEngine` — with concrete +implementations assembled from TOML config via registry-based factories, and a +swappable numpy/torch array backend. + +## Start here + +
+ +- **[Quickstart](quickstart.md)** — install and run your first simulation end to end. +- **[How it works](how-it-works.md)** — the ABC model, factories, and the compose/lower pipeline. +- **[Components](components.md)** — how config-driven components are declared and validated. +- **[API Reference](reference/components.md)** — every exported symbol, generated from source. + +
+ +## Layout + +| Package | Contents | +|---|---| +| [`shinro.components`](reference/components.md) | The five ABCs and the `ConfigDriven` mixin | +| [`shinro.trajectories`](reference/trajectories.md) | Reference path generators | +| [`shinro.controllers`](reference/controllers.md) | LQR, PID, MPC, MPPI, SMC, RL adapters | +| [`shinro.estimators`](reference/estimators.md) | Kalman filter, Luenberger observer | +| [`shinro.plants`](reference/plants.md) | Robot models | +| [`shinro.factories`](reference/factories.md) | Registry-based TOML factories, `Scenario` | +| [`shinro.utils`](reference/utils.md) | Array backend, linearization, controllability | +| [`shinro.simulation`](reference/simulation.md) | Robot simulation factory | + +## Install + +```bash +pip install -e ".[mujoco,media]" # MuJoCo + plotting for the demos +``` + +## How this reference stays current + +The **API Reference** section is generated, not written by hand. +`scripts/gen_api.py` walks each subpackage's `__all__` and emits one page per +subpackage, so **adding an export is all it takes** for it to appear here on the +next build. A subpackage without `__all__` falls back to an AST scan of its +source files. + +```bash +python scripts/gen_api.py # regenerate reference page + nav +mkdocs serve # preview at http://127.0.0.1:8000 +``` + +Prose pages are hand-written and listed in `docs/_nav_prose.yml`. The generated +nav is composed into `docs/SUMMARY.md`, which is a build artifact. diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 0000000..675af0d --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,20 @@ +// FILE: /mnt/E/sabrina-sandbox/docs-poc/docs/javascripts/mathjax.js +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"], ["$", "$"]], + displayMath: [["\\[", "\\]"], ["$$", "$$"]], + processEscapes: true, + processEnvironments: true + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex" + } +}; + +document$.subscribe(() => { + MathJax.startup.output.clearCache(); + MathJax.typesetClear(); + MathJax.texReset(); + MathJax.typesetPromise(); +}); diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..7b687fa --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,99 @@ +# FILE: mkdocs.yml +# API reference site for shinro. +# +# python scripts/gen_api.py && mkdocs build +# python scripts/gen_api.py && mkdocs serve # live reload +# +# The reference pages AND the reference nav are generated by scripts/gen_api.py +# from each subpackage's __all__. Only prose pages are declared by hand below. + +site_name: shinro +site_description: Whole-body control framework — controllers, plants, estimators, trajectories, and a MuJoCo-backed simulation factory. +docs_dir: docs +site_dir: site +use_directory_urls: true + +theme: + name: material + features: + - navigation.sections + - navigation.indexes + - navigation.top + - content.code.copy + - content.code.annotate + - search.suggest + - search.highlight + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/weather-sunny + name: Switch to light mode + +plugins: + # literate-nav lets docs/_nav_api.yml supply the reference section, so the + # generated pages never need a hand-edited nav entry. + - literate-nav: + nav_file: SUMMARY.md + - search + - mkdocstrings: + handlers: + python: + # Read the source tree directly — no install needed. + paths: + - src + options: + # Normalizes Sphinx-only markup (:math:, .. math::, :class:) that + # would otherwise render as literal text. + # GOTCHA: this key MUST live under options:, not beside paths:. + # Placed at handler level it is silently ignored — you get a green + # build with broken math. + extensions: + - scripts/sphinx_compat.py + docstring_style: google + show_source: true + show_root_heading: false + show_root_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + members_order: source + separate_signature: true + show_signature_annotations: true + signature_crossrefs: true + merge_init_into_class: true + docstring_section_style: table + filters: + - "!^_" + +# Strict mode is intentionally NOT used in CI: griffe reports the unannotated +# public parameters (see README) as warnings, so --strict is red out of the box. +# Tighten annotations first, then add --strict to the workflow. +validation: + links: + absolute_links: ignore + unrecognized_links: ignore + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.arithmatex: + generic: true + - toc: + permalink: true + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..7c09577 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,6 @@ +# FILE: requirements-docs.txt +# Docs toolchain. Kept separate from requirements-dev.txt so the CI test matrix +# doesn't pull a web toolchain. +mkdocs-material>=9.5 +mkdocstrings[python]>=0.26 +mkdocs-literate-nav>=0.6 diff --git a/scripts/gen_api.py b/scripts/gen_api.py new file mode 100644 index 0000000..f011488 --- /dev/null +++ b/scripts/gen_api.py @@ -0,0 +1,146 @@ +# FILE: scripts/gen_api.py +"""Generate the API-reference pages from the package's own ``__all__``. + +The package source is the single source of truth. Each subpackage's ``__all__`` +becomes one page; a subpackage without ``__all__`` falls back to an AST scan of +its source files. Adding an export therefore makes it appear in the docs on the +next build with no config, nav, or symbol-list edit. + +Run directly (``python scripts/gen_api.py``) or from CI before ``mkdocs build``. +""" + +from __future__ import annotations + +import ast +import importlib +import pathlib +import sys +import warnings + +# Import noise (registration warnings) is expected and irrelevant here. +warnings.filterwarnings("ignore") + +REPO = pathlib.Path(__file__).resolve().parent.parent +SRC = REPO / "src" +DOCS = REPO / "docs" +OUT = DOCS / "reference" + +# Pages to generate, in nav order. `shinro.components` is a module, the rest are +# subpackages. Anything not listed simply doesn't get a page. +PACKAGES: list[tuple[str, str]] = [ + ("shinro.components", "Core ABCs"), + ("shinro.trajectories", "Trajectories"), + ("shinro.controllers", "Controllers"), + ("shinro.estimators", "Estimators"), + ("shinro.plants", "Plants"), + ("shinro.factories", "Factories"), + ("shinro.utils", "Utilities"), + ("shinro.simulation", "Simulation"), +] + + +def _slug(modname: str) -> str: + return modname.replace("shinro.", "").replace(".", "-").replace("_", "-") + + +def fallback_exports(modname: str) -> list[str]: + """Fully-qualified public names for a package/module with no ``__all__``. + + Handles both a package directory (``shinro/utils/``) and a single module + (``shinro/components.py``). Returns dotted identifiers resolving to the real + defining module, which is what mkdocstrings needs. + """ + candidates: list[str] = [] + + pkg_dir = SRC / modname.replace(".", "/") + if pkg_dir.is_dir(): + sources = sorted(p for p in pkg_dir.glob("*.py") if p.name != "__init__.py") + prefix = modname + else: + as_file = SRC / (modname.replace(".", "/") + ".py") + if not as_file.is_file(): + return [] + sources = [as_file] + prefix = modname + + for py in sources: + try: + tree = ast.parse(py.read_text(encoding="utf-8", errors="replace")) + except SyntaxError: + continue + submod = prefix if not pkg_dir.is_dir() else f"{prefix}.{py.stem}" + for node in tree.body: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) and not node.name.startswith("_"): + candidates.append(f"{submod}.{node.name}") + + seen: set[str] = set() + return [c for c in candidates if not (c in seen or seen.add(c))] + + +def main() -> int: + if not SRC.is_dir(): + print(f"error: source tree not found at {SRC}", file=sys.stderr) + return 1 + + # Import from the repo source directly — no install step required. + sys.path.insert(0, str(SRC)) + + OUT.mkdir(parents=True, exist_ok=True) + api_nav: list[str] = [] + total = 0 + pages = 0 + + for modname, title in PACKAGES: + try: + mod = importlib.import_module(modname) + except Exception as exc: # a broken import shouldn't kill the whole build + print(f" SKIP {modname}: import failed ({exc})", file=sys.stderr) + continue + + exported = list(getattr(mod, "__all__", [])) + if exported: + ids = [f"{modname}.{s}" for s in exported] + labels = exported + source = "__all__" + else: + ids = fallback_exports(modname) + labels = [i.rsplit(".", 1)[-1] for i in ids] + source = "AST fallback (no __all__)" + + if not ids: + print(f" SKIP {modname}: no exports found", file=sys.stderr) + continue + + lines = ["---", f"title: {title}", "---", "", f"# {title}", "", f"`{modname}`", ""] + if mod.__doc__: + lines += [mod.__doc__.strip(), ""] + + for label, dotted in zip(labels, ids): + lines += ["---", "", f"## `{label}`", "", f"::: {dotted}", ""] + + (OUT / f"{_slug(modname)}.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + api_nav.append(f" * [{title}](reference/{_slug(modname)}.md)") + pages += 1 + total += len(ids) + print(f" wrote reference/{_slug(modname)}.md ({len(ids)} exports, via {source})") + + # Compose the full nav in mkdocs-literate-nav markdown format. + # Comment lines in the prose file are stripped so they don't reach the nav. + prose_nav = DOCS / "_nav_prose.md" + nav = "" + if prose_nav.is_file(): + keep = [ln for ln in prose_nav.read_text(encoding="utf-8").splitlines() if ln.strip() and not ln.lstrip().startswith("#")] + nav = "\n".join(keep) + else: + print(f" note: {prose_nav.name} not found — prose pages omitted from nav", file=sys.stderr) + + api_section = "* API Reference\n" + "\n".join(api_nav) + (DOCS / "SUMMARY.md").write_text(nav + "\n" + api_section + "\n", encoding="utf-8") + (DOCS / "_nav_api.yml").write_text(api_section + "\n", encoding="utf-8") + + print(f"\n{pages} pages, {total} exported symbols") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sphinx_compat.py b/scripts/sphinx_compat.py new file mode 100644 index 0000000..5434781 --- /dev/null +++ b/scripts/sphinx_compat.py @@ -0,0 +1,79 @@ +# FILE: scripts/sphinx_compat.py +"""Griffe extension: make Sphinx-flavored docstrings render under mkdocstrings. + +The shinro source uses a hybrid convention: + +* Google ``Args:`` / ``Returns:`` sections — griffe parses these natively. +* Sphinx roles (``:class:``, ``:meth:``, ``:func:``, ``:mod:``) — NOT handled, + so they leak into the rendered page as literal text. +* Sphinx math (``:math:`` inline, ``.. math::`` blocks) — NOT handled, so + formulas appear as raw markup instead of equations. + +This rewrites the three leaking forms as griffe walks each object, so +mkdocstrings parses already-normalized text. It runs on every build, meaning a +docstring edit needs no manual follow-up. + +NOTE: the equivalent of this file does not exist for the Sphinx toolchain, which +understands these forms natively — that path's trade-off is RST config surface +and a less polished theme. +""" + +from __future__ import annotations + +import re + +from griffe import Docstring, Extension + +# Inline math: `:math:`expr`` -> `$expr$`. Must run BEFORE the generic role +# stripper, otherwise `:math:` would match the `:role:` pattern first. +_RE_MATH_INLINE = re.compile(r":math:`([^`]+)`") + +# Block math: an indented `.. math::` directive -> a fenced `$$` block. +_RE_MATH_BLOCK = re.compile( + r"^[ \t]*\.\.\s*math::[ \t]*\n(?P(?:[ \t]+\S.*\n?|\n)*)", + re.MULTILINE, +) + +# Remaining Sphinx roles: `:class:`Foo`` -> `` `Foo` `` (keep text, drop role). +_RE_SPHINX_ROLE = re.compile(r":(?:class|meth|func|mod|attr|data|obj|ref|term|exc|doc):`~?([^`]+)`") + +# `:func:`target`` leaves empty inline literals behind after the rewrite. +_RE_EMPTY_LITERAL = re.compile(r"``\s*``") + + +def _math_block_sub(match: re.Match[str]) -> str: + body = [ln.strip() for ln in match.group("body").splitlines() if ln.strip()] + if not body: + return "" + return "\n\n$$\n" + " ".join(body) + "\n$$\n\n" + + +def normalize_text(text: str) -> str: + """Rewrite Sphinx-only markup into mkdocstrings/MathJax-friendly markdown.""" + if not text: + return text + text = _RE_MATH_BLOCK.sub(_math_block_sub, text) + text = _RE_MATH_INLINE.sub(lambda m: f"${m.group(1)}$", text) + text = _RE_SPHINX_ROLE.sub(lambda m: f"`{m.group(1)}`", text) + text = _RE_EMPTY_LITERAL.sub("", text) + return text + + +class SphinxCompat(Extension): + """Rewrite Sphinx-only markup so it renders instead of leaking as text.""" + + def __init__(self, **kwargs: object) -> None: + super().__init__(**kwargs) # type: ignore[arg-type] + self.normalized = 0 + + def on_instance(self, *, node=None, obj=None, agent=None, **kwargs: object) -> None: + """Fires once per parsed object as griffe walks the tree.""" + if obj is None: + return + doc = getattr(obj, "docstring", None) + if not isinstance(doc, Docstring) or not doc.value: + return + new_value = normalize_text(doc.value) + if new_value != doc.value: + doc.value = new_value + self.normalized += 1