From 0191ffcc1d7f6fe343b0a6f24fd2cbfe6d8216eb Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 6 Oct 2026 12:53:48 -0400 Subject: [PATCH] Add reader-oriented documentation and correct API examples --- .github/workflows/documentation.yml | 41 +++++++++ .gitignore | 1 + README.md | 134 ++++++++-------------------- docs.sh | 4 + docs/dev/documentation-style.md | 27 ++++++ docs/guides/annotations.md | 68 ++++++++++++++ docs/guides/migration.md | 34 +++++++ docs/guides/progress.md | 45 ++++++++++ docs/index.md | 65 ++++++++++++++ docs/reference.md | 27 ++++++ docs/stylesheets/docs.css | 69 ++++++++++++++ gtfparse/__init__.py | 2 +- gtfparse/attribute_parsing.py | 7 +- gtfparse/create_missing_features.py | 7 +- mkdocs.yml | 55 ++++++++++++ requirements-docs.txt | 3 + scripts/check_docs_examples.py | 33 +++++++ scripts/check_docs_links.py | 54 +++++++++++ 18 files changed, 575 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/documentation.yml create mode 100755 docs.sh create mode 100644 docs/dev/documentation-style.md create mode 100644 docs/guides/annotations.md create mode 100644 docs/guides/migration.md create mode 100644 docs/guides/progress.md create mode 100644 docs/index.md create mode 100644 docs/reference.md create mode 100644 docs/stylesheets/docs.css create mode 100644 mkdocs.yml create mode 100644 requirements-docs.txt create mode 100644 scripts/check_docs_examples.py create mode 100644 scripts/check_docs_links.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..5499215 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,41 @@ +name: Documentation +on: + pull_request: + push: + branches: [master] + workflow_dispatch: +permissions: + contents: read +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-docs.txt + - run: ./docs.sh + - uses: actions/upload-artifact@v4 + with: + name: documentation + path: site/ + publish: + needs: build + if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v4 + with: + name: documentation + path: site/ + - run: | + python -m pip install 'ghp-import>=2.1,<3' + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + ghp-import --no-jekyll --push --force site/ diff --git a/.gitignore b/.gitignore index 65637cb..b1131fc 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ docs/_build/ # PyBuilder target/ +site/ diff --git a/README.md b/README.md index ed2585f..7868e91 100644 --- a/README.md +++ b/README.md @@ -8,117 +8,59 @@ gtfparse ======== Parsing tools for GTF (gene transfer format) files. -## Version 3 migration -`read_gtf`, `parse_gtf`, and `parse_gtf_and_expand_attributes` now return pandas -DataFrames by default. GTF reading uses PyArrow; attribute processing and writing -use pandas. Polars is no longer a required dependency and is imported only for -explicit Polars conversion. Ordinary installations need only pandas and PyArrow. +## Install ```sh -pip install gtfparse -# Optional, for applications that still request Polars frames: -pip install 'gtfparse[polars]' +python -m pip install gtfparse ``` -```python -from gtfparse import read_gtf, write_gtf - -df = read_gtf("gene_annotations.gtf") # pandas.DataFrame -write_gtf(df, "gene_annotations.gtf.gz") - -# Requires the optional extra: -polars_df = read_gtf("gene_annotations.gtf", result_type="polars") -``` - -Existing pandas and dictionary callers retain their output shape, fixed-column -dtypes, attribute handling, filters, converters and progress callbacks. Applications -that used the previous default Polars output should add `result_type="polars"` -and install the extra, or use pandas methods. `write_gtf` accepts either frame -type. The legacy `parse_with_polars_lazy` helper remains an optional adapter: -it reads the file eagerly and returns a Polars LazyFrame, as before. - -Empty or malformed GTF fields now raise `gtfparse.ParsingError`, including wrong -field counts and invalid numeric fields. Whole comment lines are ignored; -literal `#` inside attributes is preserved. Text and binary streams stay open -after parsing, and gzip is detected from its contents. - -## Reporting parsing progress - -Pass `progress_callback(stage, completed, total)` to `read_gtf` to connect -parsing to your own progress UI. The callback runs synchronously; exceptions -from it stop parsing and propagate to the caller. Without a callback, gtfparse -does not display a progress bar or add a progress-library dependency. - -| Stage | Reports | -| --- | --- | -| `read` | `(0, None)` before loading/filtering the file; `(rows, rows)` after it succeeds | -| `attributes` | `(0, rows)`, then every 10,000 rows, then `(rows, rows)` | -| `convert` | `(0, None)` before output conversion/transforms; `(rows, rows)` after success | - -Counts refer to rows retained after `features` filtering, not bytes or overall -percentages. `None` means the total is unknown: loading and conversion expose -only stage boundaries, so use an indeterminate indicator during those steps. -Attribute expansion is omitted when `expand_attribute_column=False`. A stage -that fails does not emit completion. An empty filtered result is supported; -its attributes stage emits a single `(0, 0)` event. Empty input files raise `gtfparse.ParsingError`. - -For example, with the optional `tqdm` package installed: +Python 3.9 or later is required. Version 3 uses pandas and PyArrow; Polars +output is optional and requires `gtfparse[polars]`. -```python -from gtfparse import read_gtf -from tqdm.auto import tqdm - -with tqdm(unit="rows") as bar: - def show_progress(stage, completed, total): - if stage != bar.desc: - bar.total = total - bar.reset() - bar.set_description_str(stage) - bar.total = total - bar.update(completed - bar.n) - bar.refresh() - - df = read_gtf("gene_annotations.gtf", progress_callback=show_progress) -``` - -`parse_gtf` reports the `read` stage, `parse_gtf_and_expand_attributes` reports -`read` and `attributes`, `parse_gtf_pandas` reports `read` and `convert`, and -`expand_attribute_strings` reports only `attributes` using the same callback. - -Quoted attribute values preserve spaces, semicolons, and apostrophes. Raw -attributes (`expand_attribute_column=False`) retain their original quote marks. - -# Example usage +## Read an annotation and find genes -## Parsing all rows of a GTF file into a Pandas DataFrame +This small artificial GTF runs without a download. Replace the stream with +your annotation filename when working with a real dataset. ```python +from io import StringIO from gtfparse import read_gtf -# returns GTF with essential columns such as "feature", "seqname", "start", "end" -# alongside the names of any optional keys which appeared in the attribute column -df = read_gtf("gene_annotations.gtf") +annotation = StringIO( + '1\texample\tgene\t101\t200\t.\t+\t.\tgene_id "g1"; gene_name "EXAMPLE";\n' + '1\texample\texon\t101\t150\t.\t+\t.\tgene_id "g1"; transcript_id "t1";\n' +) +features = read_gtf(annotation) +genes = features.loc[features["feature"] == "gene"] +print(genes["gene_name"].tolist()) +for row in features.itertuples(): + print(row.feature, row.seqname, row.start, row.end, row.gene_id) +``` -# filter DataFrame to gene entries on chrY -df_genes = df[df["feature"] == "gene"] -df_genes_chrY = df_genes[df_genes["seqname"] == "Y"] +```text +['EXAMPLE'] +gene 1 101 200 g1 +exon 1 101 150 g1 ``` +`feature` identifies the row type. `seqname` is the chromosome or contig name; +it is not the gene ID. The ninth GTF field supplies attributes such as +`gene_id`, `gene_name` and `transcript_id`. -## Getting gene FPKM values from a StringTie GTF file -```python -from gtfparse import read_gtf + + + + + -df = read_gtf( - "Transcripts.gtf", - column_converters={"FPKM": float}) +## Documentation -gene_fpkms = { - gene_name: fpkm - for (gene_name, fpkm, feature) - in zip(df["seqname"], df["FPKM"], df["feature"]) - if feature == "gene" -} -``` +- [Filter features, select chromosomes, write GTF and read numeric attributes](docs/guides/annotations.md) +- [Progress callbacks](docs/guides/progress.md) +- [Version 3 migration and optional Polars output](docs/guides/migration.md) +- [Complete API reference](docs/reference.md) + +Build with `python -m pip install -r requirements-docs.txt` and `./docs.sh`. +Run `python scripts/check_docs_examples.py` to verify the printed examples. diff --git a/docs.sh b/docs.sh new file mode 100755 index 0000000..800ac4a --- /dev/null +++ b/docs.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +python -m mkdocs build --strict +python scripts/check_docs_links.py diff --git a/docs/dev/documentation-style.md b/docs/dev/documentation-style.md new file mode 100644 index 0000000..6f77810 --- /dev/null +++ b/docs/dev/documentation-style.md @@ -0,0 +1,27 @@ +# Documentation style + +Follow the [PyEnsembl writing guide](https://github.com/openvax/pyensembl/blob/edd183d/docs/dev/documentation-style.md) +and the [mhctools example](https://github.com/openvax/mhctools/tree/master/docs). + +Put installation and the first useful example on the home page. Give complete +imports and concrete inputs, explain the result, and state prerequisites before +the reader runs a command. Show output only when it has been checked. Label +replaceable paths and templates. + +Order guides by common tasks before advanced setup and implementation details. +Keep desktop sidebar groups expanded initially and individually collapsible. +Split a page when a substantial task or reference needs its own explanation. +Preserve useful old links and anchors when moving content. + +Use ordinary text for concepts and library names; reserve inline code for +literal commands, API names, parameters, paths and values. Keep qualifications, +experimental status and scientific limitations beside the claims they qualify. + +Use system fonts, a readable line length and restrained headings. Let wide +tables and code blocks scroll inside their containers. Avoid repetitive +navigation lists in prose and excessive callouts. + +Install requirements-docs.txt and run ./docs.sh before opening a PR. Review +the home page and a reference page at desktop and narrow widths. Run the +repository's required lint and test commands and execute complete examples. +Documentation dependencies are separate from runtime dependencies. diff --git a/docs/guides/annotations.md b/docs/guides/annotations.md new file mode 100644 index 0000000..35f77ea --- /dev/null +++ b/docs/guides/annotations.md @@ -0,0 +1,68 @@ +# Filter and write annotations + +## Select a chromosome and feature + +This standalone artificial input has two gene rows on different chromosomes: + +```python +from io import StringIO +from gtfparse import read_gtf + +annotation = StringIO( + '1\texample\tgene\t101\t200\t.\t+\t.\tgene_id "g1"; gene_name "FIRST";\n' + 'Y\texample\tgene\t301\t400\t.\t+\t.\tgene_id "g2"; gene_name "SECOND";\n' +) +genes = read_gtf(annotation, features={"gene"}) +selected = genes.loc[genes["seqname"] == "Y"] +print(selected["gene_id"].tolist()) +``` + +```text +['g2'] +``` + +Match the contig spelling in your file: `Y` and `chrY` are different strings. +The parser does not infer reference assembly compatibility from those names. + +## Write a compressed GTF + +Continuing with `genes` above, write and reread a temporary gzip file: + +```python +from pathlib import Path +from tempfile import TemporaryDirectory +from gtfparse import write_gtf + +with TemporaryDirectory() as directory: + path = Path(directory) / "genes.gtf.gz" + write_gtf(genes, path) + restored = read_gtf(path) + print(restored["gene_id"].tolist()) +``` + +```text +['g1', 'g2'] +``` + +The writer reconstructs the attribute column. It is intended to preserve +annotation values, rather than the original file's exact spacing and ordering. + +## Read numeric attributes + +GTF attributes initially represent text. Supply `column_converters` when an +attribute has a numeric meaning in your input file. This template requires a +file with gene rows carrying an FPKM attribute: + +```python +from gtfparse import read_gtf + +features = read_gtf("expression.gtf", column_converters={"FPKM": float}) +genes = features.loc[features["feature"] == "gene"] +gene_fpkms = dict(zip(genes["gene_id"], genes["FPKM"])) +``` + +Use a gene identifier as the key, rather than `seqname`, which would collapse +all genes on a chromosome into one entry. For transcript-level expression, +select transcript rows and key by `transcript_id`. Parsing does not aggregate +transcript expression into gene expression, and a file without gene rows +will not produce a gene-level table from this filter. diff --git a/docs/guides/migration.md b/docs/guides/migration.md new file mode 100644 index 0000000..850c223 --- /dev/null +++ b/docs/guides/migration.md @@ -0,0 +1,34 @@ +# Version 3 migration + +`read_gtf`, `parse_gtf`, and `parse_gtf_and_expand_attributes` now return pandas +DataFrames by default. GTF reading uses PyArrow; attribute processing and writing +use pandas. Polars is no longer a required dependency and is imported only for +explicit Polars conversion. Ordinary installations need only pandas and PyArrow. + +```sh +pip install gtfparse +# Optional, for applications that still request Polars frames: +pip install 'gtfparse[polars]' +``` + +```python +from gtfparse import read_gtf, write_gtf + +df = read_gtf("gene_annotations.gtf") # pandas.DataFrame +write_gtf(df, "gene_annotations.gtf.gz") + +# Requires the optional extra: +polars_df = read_gtf("gene_annotations.gtf", result_type="polars") +``` + +Existing pandas and dictionary callers retain their output shape, fixed-column +dtypes, attribute handling, filters, converters and progress callbacks. Applications +that used the previous default Polars output should add `result_type="polars"` +and install the extra, or use pandas methods. `write_gtf` accepts either frame +type. The legacy `parse_with_polars_lazy` helper remains an optional adapter: +it reads the file eagerly and returns a Polars LazyFrame, as before. + +Empty or malformed GTF fields now raise `gtfparse.ParsingError`, including wrong +field counts and invalid numeric fields. Whole comment lines are ignored; +literal `#` inside attributes is preserved. Text and binary streams stay open +after parsing, and gzip is detected from its contents. diff --git a/docs/guides/progress.md b/docs/guides/progress.md new file mode 100644 index 0000000..1e3a3eb --- /dev/null +++ b/docs/guides/progress.md @@ -0,0 +1,45 @@ +# Report parsing progress + +Pass `progress_callback(stage, completed, total)` to `read_gtf` to connect +parsing to your own progress UI. The callback runs synchronously; exceptions +from it stop parsing and propagate to the caller. Without a callback, gtfparse +does not display a progress bar or add a progress-library dependency. + +| Stage | Reports | +| --- | --- | +| `read` | `(0, None)` before loading/filtering the file; `(rows, rows)` after it succeeds | +| `attributes` | `(0, rows)`, then every 10,000 rows, then `(rows, rows)` | +| `convert` | `(0, None)` before output conversion/transforms; `(rows, rows)` after success | + +Counts refer to rows retained after `features` filtering, not bytes or overall +percentages. `None` means the total is unknown: loading and conversion expose +only stage boundaries, so use an indeterminate indicator during those steps. +Attribute expansion is omitted when `expand_attribute_column=False`. A stage +that fails does not emit completion. An empty filtered result is supported; +its attributes stage emits a single `(0, 0)` event. Empty input files raise `gtfparse.ParsingError`. + +For example, with the optional `tqdm` package installed: + +```python +from gtfparse import read_gtf +from tqdm.auto import tqdm + +with tqdm(unit="rows") as bar: + def show_progress(stage, completed, total): + if stage != bar.desc: + bar.total = total + bar.reset() + bar.set_description_str(stage) + bar.total = total + bar.update(completed - bar.n) + bar.refresh() + + df = read_gtf("gene_annotations.gtf", progress_callback=show_progress) +``` + +`parse_gtf` reports the `read` stage, `parse_gtf_and_expand_attributes` reports +`read` and `attributes`, `parse_gtf_pandas` reports `read` and `convert`, and +`expand_attribute_strings` reports only `attributes` using the same callback. + +Quoted attribute values preserve spaces, semicolons, and apostrophes. Raw +attributes (`expand_attribute_column=False`) retain their original quote marks. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..e4a7c7a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,65 @@ +# gtfparse + +Read GTF annotations into a pandas DataFrame, with one row per feature and +attribute values such as gene and transcript IDs expanded into columns. + +## Install + +```sh +python -m pip install gtfparse +``` + +Python 3.9 or later is required. Version 3 uses pandas and PyArrow; Polars +output is optional and requires `gtfparse[polars]`. + +## Read an annotation and find genes + +This small artificial GTF runs without a download. Replace the stream with +your annotation filename when working with a real dataset. + +```python +from io import StringIO +from gtfparse import read_gtf + +annotation = StringIO( + '1\texample\tgene\t101\t200\t.\t+\t.\tgene_id "g1"; gene_name "EXAMPLE";\n' + '1\texample\texon\t101\t150\t.\t+\t.\tgene_id "g1"; transcript_id "t1";\n' +) +features = read_gtf(annotation) +genes = features.loc[features["feature"] == "gene"] +print(genes["gene_name"].tolist()) +for row in features.itertuples(): + print(row.feature, row.seqname, row.start, row.end, row.gene_id) +``` + +```text +['EXAMPLE'] +gene 1 101 200 g1 +exon 1 101 150 g1 +``` + +`feature` identifies the row type. `seqname` is the chromosome or contig name; +it is not the gene ID. The ninth GTF field supplies attributes such as +`gene_id`, `gene_name` and `transcript_id`. + +GTF coordinates are one-based and inclusive: 101–150 spans 50 bases. Parsing +preserves this convention rather than converting it to BED coordinates. See +the [GENCODE format description](https://www.gencodegenes.org/pages/data_format.html). + +## Use your files + +This template requires an existing annotation file: + +```python +from gtfparse import read_gtf + +features = read_gtf("annotations.gtf.gz", features={"gene", "transcript"}) +``` + +The `features` option filters row types while reading. Compressed inputs are +detected from their contents. Attribute expansion does not turn every row into +a gene or automatically supply missing parent features. + +Continue with [filter and write annotations](guides/annotations.md), then +[progress callbacks](guides/progress.md) or [version 3 migration](guides/migration.md). +The [API reference](reference.md) documents the public parsing functions. diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..a61b944 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,27 @@ +# API reference + +| Task | Interface | +| --- | --- | +| Read a file with expanded attributes and output conversion | `read_gtf()` | +| Write annotation rows | `write_gtf()` | +| Read fixed fields without attribute expansion | `parse_gtf()` | +| Parse and expand attribute strings | `parse_gtf_and_expand_attributes()` | +| Request pandas output explicitly | `parse_gtf_pandas()` | +| Expand a collection of attribute strings | `expand_attribute_strings()` | +| Synthesize missing parent rows explicitly | `create_missing_features()` | + +::: gtfparse.read_gtf + +::: gtfparse.write_gtf + +::: gtfparse.parse_gtf + +::: gtfparse.parse_gtf_and_expand_attributes + +::: gtfparse.parse_gtf_pandas + +::: gtfparse.expand_attribute_strings + +::: gtfparse.create_missing_features + +::: gtfparse.ParsingError diff --git a/docs/stylesheets/docs.css b/docs/stylesheets/docs.css new file mode 100644 index 0000000..44d017c --- /dev/null +++ b/docs/stylesheets/docs.css @@ -0,0 +1,69 @@ +/* A quiet reading surface for scientific guides and API reference. */ +:root { + --md-text-font: "Helvetica Neue", Arial, sans-serif; + --md-code-font: "SFMono-Regular", Consolas, "Liberation Mono", monospace; +} + +body[data-md-color-scheme="default"] { + --md-default-bg-color: #ffffff; + --md-default-fg-color: #263238; + --md-default-fg-color--light: #59636b; + --md-typeset-a-color: #1769aa; + --md-code-bg-color: #f5f7f9; +} + +.md-header { + box-shadow: none; + border-bottom: 1px solid #e2e6ea; +} + +.md-content__inner { + max-width: 76ch; +} + +.md-typeset { + font-size: 0.85rem; + line-height: 1.65; +} + +.md-typeset h1, +.md-typeset h2, +.md-typeset h3 { + color: var(--md-default-fg-color); + font-weight: 600; + letter-spacing: normal; +} + +.md-typeset h1 { + font-size: 1.8rem; +} + +.md-typeset h2 { + font-size: 1.25rem; +} + +/* Inline identifiers read as part of a sentence, rather than little badges. */ +.md-typeset :not(pre) > code { + background: transparent; + border-radius: 0; + padding: 0; + font-size: 0.88em; + color: inherit; + box-shadow: none; +} + +.md-typeset table:not([class]) { + font-size: 0.8rem; + line-height: 1.5; +} + +/* Keep an identifier intact; wide reference tables can scroll. */ +.md-typeset table code { + white-space: nowrap; +} + +.md-typeset h3 { font-size: 1.05rem; } +.md-typeset pre { overflow-x: auto; } + +/* Keep the repository link without unrelated GitHub release/star counters. */ +.md-source__facts { display: none; } diff --git a/gtfparse/__init__.py b/gtfparse/__init__.py index 20c94a5..71397b7 100644 --- a/gtfparse/__init__.py +++ b/gtfparse/__init__.py @@ -24,7 +24,7 @@ ) from .write_gtf import write_gtf -__version__ = "3.0.0" +__version__ = "3.0.1" __all__ = [ "GENCODE_BIOTYPE_ALIASES", diff --git a/gtfparse/attribute_parsing.py b/gtfparse/attribute_parsing.py index 495156c..a2f2fe2 100644 --- a/gtfparse/attribute_parsing.py +++ b/gtfparse/attribute_parsing.py @@ -55,8 +55,11 @@ def expand_attribute_strings( the start, every 10,000 rows, and at completion. An empty input emits one ``("attributes", 0, 0)`` event. Callback exceptions propagate. - Returns OrderedDict of column->value list mappings, in the order they - appeared in the attribute strings. + Returns + ------- + collections.OrderedDict + Column-to-value-list mappings, in the order the columns appeared in + the attribute strings. """ n = len(attribute_strings) diff --git a/gtfparse/create_missing_features.py b/gtfparse/create_missing_features.py index e8b649e..8a2abc4 100644 --- a/gtfparse/create_missing_features.py +++ b/gtfparse/create_missing_features.py @@ -44,8 +44,11 @@ def create_missing_features(dataframe, unique_keys={}, extra_columns={}, missing missing_value : any Which value to fill in for columns that we don't infer values for. - Returns original dataframe (converted to Pandas if necessary) along with all - extra rows created for missing features. + Returns + ------- + pandas.DataFrame + Original rows (converted to pandas if necessary) together with the + extra rows created for missing features. """ if hasattr(dataframe, "to_pandas"): dataframe = dataframe.to_pandas() diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..e90a5b1 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,55 @@ +site_name: gtfparse +site_description: Read and write GTF annotation tables +site_url: https://openvax.github.io/gtfparse/ +repo_url: https://github.com/openvax/gtfparse +repo_name: openvax/gtfparse +edit_uri: edit/master/docs/ +strict: true +theme: + name: material + font: false + palette: + primary: white + accent: blue + features: + - navigation.indexes + - navigation.expand + - content.code.copy +validation: + links: + not_found: warn + unrecognized_links: warn + anchors: warn +plugins: +- search +- mkdocstrings: + handlers: + python: + paths: + - . + options: + docstring_style: numpy + show_root_heading: true + show_root_full_path: false + show_source: false + separate_signature: true + heading_level: 2 + docstring_section_style: list + members_order: source +markdown_extensions: +- tables +- toc: + permalink: true +- pymdownx.superfences +extra_css: +- stylesheets/docs.css +nav: +- Home: index.md +- Common tasks: + - Filter and write annotations: guides/annotations.md + - Report parsing progress: guides/progress.md +- Reference: + - API reference: reference.md + - Version 3 migration: guides/migration.md +- Contributing: + - Documentation style: dev/documentation-style.md diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..4e790b5 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,3 @@ +mkdocs>=1.6,<2 +mkdocs-material>=9.5,<10 +mkdocstrings[python]>=0.27,<2 diff --git a/scripts/check_docs_examples.py b/scripts/check_docs_examples.py new file mode 100644 index 0000000..536ecda --- /dev/null +++ b/scripts/check_docs_examples.py @@ -0,0 +1,33 @@ +"""Execute documentation examples and compare the printed output.""" + +import contextlib +import io +import re +import sys +import types +from pathlib import Path + + +def main(): + count = 0 + for path in sorted(Path("docs").rglob("*.md")): + module = types.ModuleType("docs_example_" + path.stem.replace("-", "_")) + sys.modules[module.__name__] = module + blocks = list(re.finditer(r"```(python|text)\n(.*?)\n```", path.read_text(), re.S)) + for index, match in enumerate(blocks): + if match[1] != "python" or index + 1 == len(blocks) or blocks[index + 1][1] != "text": + continue + output = io.StringIO() + with contextlib.redirect_stdout(output): + exec(compile(match[2], str(path), "exec"), module.__dict__) + if index + 1 < len(blocks) and blocks[index + 1][1] == "text": + expected = blocks[index + 1][2].rstrip() + actual = output.getvalue().rstrip() + if actual != expected: + raise AssertionError(f"{path}: {actual!r} != {expected!r}") + count += 1 + print(f"Executed {count} documentation examples.") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_docs_links.py b/scripts/check_docs_links.py new file mode 100644 index 0000000..91de351 --- /dev/null +++ b/scripts/check_docs_links.py @@ -0,0 +1,54 @@ +"""Check rendered internal links and fragment targets.""" +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urlsplit +import yaml + + +class Page(HTMLParser): + def __init__(self, text): + super().__init__() + self.ids = set() + self.links = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if attrs.get("id"): + self.ids.add(attrs["id"]) + if tag == "a" and attrs.get("href"): + self.links.append(attrs["href"]) + + +def main(): + root = Path("site").resolve() + base = urlsplit(yaml.safe_load(Path("mkdocs.yml").read_text())["site_url"]).path.rstrip("/") + pages = {p.resolve(): Page(p.read_text()) for p in root.rglob("*.html")} + failures = [] + count = 0 + for source, page in pages.items(): + for href in page.links: + url = urlsplit(href) + if url.scheme or url.netloc: + continue + path = unquote(url.path) + if path.startswith("/"): + if not (path == base or path.startswith(base + "/")): + continue + target = (root / path[len(base):].lstrip("/")).resolve() + else: + target = (source.parent / path).resolve() if path else source + if target.is_dir(): + target /= "index.html" + count += 1 + if not target.is_relative_to(root) or not target.exists(): + failures.append(f"{source.relative_to(root)}: missing {href}") + elif url.fragment and target in pages and unquote(url.fragment) not in pages[target].ids: + failures.append(f"{source.relative_to(root)}: missing fragment {href}") + if failures: + raise SystemExit("\n".join(failures)) + print(f"Checked {len(pages)} pages and {count} internal links.") + + +if __name__ == "__main__": + main()