Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,3 @@ __pycache__/
*.whl

*.log

4 changes: 2 additions & 2 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ RUN pip install pandas==1.5.3 \
numba==0.59 \
pyquaternion==0.9.9

RUN pip install nuscenes-devkit && \
pip install shapely tqdm pillow networkx fire
RUN pip install nuscenes-devkit==1.2.0 && \
pip install shapely==2.0.7 tqdm==4.67.3 pillow==12.2.0 networkx==3.4.2 fire==0.7.1

RUN pip install pytest pytest-timeout
RUN pip install pynvml
Expand Down
5 changes: 3 additions & 2 deletions docs/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ help:

# Generate namespace package documentation before building
generate:
python3 mirror_referenced_dirs.py
python3 generate_new_namespace_package_docs.py
python3 generate_package_docs_assets.py
python3 mirror_referenced_dirs.py
python3 update_docs_index.py

# Sync the root README into the docs tree before building
Expand All @@ -41,7 +42,7 @@ clean:
@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
rm -rf $(BUILDDIR)/
rm -rf api/generated/
rm -rf ../packages/*/docs/generated/
rm -rf ../packages/*/docs/_generated/

# Auto-build documentation (watches for changes)
livehtml: sync-readme generate
Expand Down
176 changes: 176 additions & 0 deletions docs/generate_package_docs_assets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
#!/usr/bin/env python3

# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
from dataclasses import dataclass
import importlib.util
from pathlib import Path
import sys
from types import ModuleType
from typing import Callable


@dataclass(frozen=True)
class PackageDocsContext:
project_root: Path
namespace_package: str
package_name: str
package_root: Path
docs_root: Path
generated_dir: Path


HookFunction = Callable[[PackageDocsContext], None]
_GENERATED_ASSET_GITIGNORE = "*\n"


def _load_hook_module(hook_path: Path, package_name: str) -> ModuleType:
# Temporary module name for the imported hook.
module_name = f"_accvlab_docs_assets_{package_name}"

# Import
spec = importlib.util.spec_from_file_location(module_name, hook_path)
if spec is None or spec.loader is None:
raise ImportError(f"Could not create import spec for docs asset hook: {hook_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)

return module


def _get_hook_function(module: ModuleType, hook_path: Path) -> HookFunction:
hook_function = getattr(module, "generate_docs_assets", None)
if not callable(hook_function):
raise AttributeError(
f"Docs asset hook must define a callable generate_docs_assets(context): {hook_path}"
)
return hook_function


def _prepare_generated_dir(context: PackageDocsContext) -> None:
"""Create the package's generated docs asset directory and keep it untracked."""
context.generated_dir.mkdir(parents=True, exist_ok=True)
(context.generated_dir / ".gitignore").write_text(_GENERATED_ASSET_GITIGNORE, encoding="utf-8")


def _build_context(project_root: Path, namespace_package: str) -> PackageDocsContext:
package_name = namespace_package.split(".")[-1]
package_root = project_root / "packages" / package_name
docs_root = package_root / "docs"
generated_dir = docs_root / "_generated"
ctx = PackageDocsContext(
project_root=project_root,
namespace_package=namespace_package,
package_name=package_name,
package_root=package_root,
docs_root=docs_root,
generated_dir=generated_dir,
)
return ctx


def _generate_assets_for_package(
*,
project_root: Path,
namespace_package: str,
verbose: bool,
) -> bool:
context = _build_context(project_root, namespace_package)
hook_path = context.docs_root / "_on_doc_generation.py"
if not hook_path.exists():
if verbose:
print(f"No docs asset hook for {context.package_name}")
return False

if verbose:
print(f"Running docs asset hook for {context.package_name}: {hook_path}")
module = _load_hook_module(hook_path, context.package_name)
_prepare_generated_dir(context)
hook_function = _get_hook_function(module, hook_path)
hook_function(context)
print(f"Generated docs assets for {context.package_name}")
return True


def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Run optional package-local documentation asset generation hooks.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Enable verbose output.",
)
parser.add_argument(
"--package",
dest="package_names",
action="append",
help="Package name to process, such as lane_helpers. Can be passed more than once.",
)
return parser.parse_args()


def main() -> int:
args = _parse_args()
docs_dir = Path(__file__).resolve().parent
project_root = docs_dir.parent
sys.path.insert(0, str(project_root))

try:
from namespace_packages_config import NAMESPACE_PACKAGES
except ImportError as exc:
print(
f"Error: Could not import NAMESPACE_PACKAGES from namespace_packages_config.py: {exc}",
file=sys.stderr,
)
return 1

package_filter = set(args.package_names or [])
namespace_packages = [
namespace_package
for namespace_package in NAMESPACE_PACKAGES
if not package_filter or namespace_package.split(".")[-1] in package_filter
]
if package_filter and len(namespace_packages) != len(package_filter):
found_package_names = {namespace_package.split(".")[-1] for namespace_package in namespace_packages}
missing_package_names = sorted(package_filter - found_package_names)
print(f"Error: Unknown namespace package(s): {', '.join(missing_package_names)}", file=sys.stderr)
return 1

hook_count = 0
for namespace_package in namespace_packages:
package_name = namespace_package.split(".")[-1]
try:
hook_ran = _generate_assets_for_package(
project_root=project_root,
namespace_package=namespace_package,
verbose=args.verbose,
)
except Exception as exc:
print(f"Error: docs asset generation failed for {package_name}: {exc}", file=sys.stderr)
return 1
if hook_ran:
hook_count += 1

if args.verbose:
print(f"Ran {hook_count} package docs asset hook(s).")
return 0


if __name__ == "__main__":
sys.exit(main())
59 changes: 43 additions & 16 deletions docs/guides/DEVELOPMENT_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ There are two example projects which showcase how a namespace package is structu
- `packages/example_package`: Showcases a package containing PyTorch extensions built using
`CppExtension` and `CUDAExtension` provided by PyTorch as well as an external implementation (see
[External Implementations](#external-implementations) section for more details on external implementations)
as described below.
as described below. It also includes a package-local documentation asset hook that generates a simple plot
from committed CSV data under `evaluation_results/` during the docs build.
- `packages/example_skbuild_package`: Showcases a package using `scikit-build` for C++/CUDA implementation
(see the [Alternative: SKBuild-Based Packages](#alternative-skbuild-based-packages) section for more
details on this approach).
Expand All @@ -70,6 +71,8 @@ To add a new namespace package (e.g., `example_package`), you need to create:
| **Setup** | `packages/example_package/setup.py` | Package build configuration |
| **Project Config** | `packages/example_package/pyproject.toml` | Modern Python project configuration and authoritative dependency definition |
| **Documentation include list (optional)** | `packages/example_package/docu_referenced_dirs.txt` | List additional directories referenced by the docs (besides `docs/`). See [Documentation Setup Guide](DOCUMENTATION_SETUP_GUIDE.md) for more details.|
| **Documentation asset hook (optional)** | `packages/example_package/docs/_on_doc_generation.py` | Generate package-owned docs assets such as plots from committed evaluation data. See [Documentation Setup Guide](DOCUMENTATION_SETUP_GUIDE.md#package-local-generated-assets). |
| **Evaluation results (optional)** | `packages/example_package/evaluation_results/` | Package-owned committed inputs for generating docs assets, such as data to plot. |

> **ℹ️ Note**: Apart from the above, further folders/files can be included (and made use of manually or added to the
> documentation) if needed. A typical use case is to include e.g. an `examples` directory which is:
Expand All @@ -84,26 +87,29 @@ The following diagram shows the relevant project structure containing the folder

```
accvlab/
├── packages/ # Namespace packages directory
├── packages/ # Namespace packages directory
│ ├── optim_test_tools/...
│ ├── batching_helpers/...
│ └── example_package/ # ← New namespace package
│ ├── accvlab/ # ← Namespace root
│ │ └── example_package/ # ← Implementation for "example_package" package
│ └── example_package/ # ← New namespace package
│ ├── accvlab/ # ← Namespace root
│ │ └── example_package/ # ← Implementation for "example_package" package
│ │ ├── __init__.py
│ │ ├── csrc/ # ← C++/CUDA sources
│ │ └── include/ # ← Headers
│ ├── ext_impl/ # ← Optional: external implementation
│ │ ├── csrc/ # ← C++/CUDA sources
│ │ └── include/ # ← Headers
│ ├── ext_impl/ # ← Optional: external implementation
│ │ ├── build_and_copy.sh
│ │ └── ...
│ ├── tests/ # ← Tests for "example_package" package
│ ├── docs/ # ← Documentation for "example_package" package
│ ├── setup.py # ← Package build configuration
│ ├── pyproject.toml # ← Project configuration (including dependencies)
│ └── docu_referenced_dirs.txt # ← Optional: list additional directories referenced by the docs (besides `docs/`)
├── build_config/ # Shared build utilities
├── docs/ # Main documentation
└── namespace_packages_config.py # ← Namespace package needs to be listed here
│ ├── tests/ # ← Tests for "example_package" package
│ ├── evaluation_results/ # ← Optional committed inputs for generated docs assets
│ ├── docs/ # ← Documentation for "example_package" package
│ │ ├── _on_doc_generation.py # ← Optional docs asset hook
│ │ └── ...
│ ├── setup.py # ← Package build configuration
│ ├── pyproject.toml # ← Project configuration (including dependencies)
│ └── docu_referenced_dirs.txt # ← Optional: list additional directories referenced by the docs (besides `docs/`)
├── build_config/ # Shared build utilities
├── docs/ # Main documentation
└── namespace_packages_config.py # ← Namespace package needs to be listed here
```

Note that inside the package, there is the directory structure `accvlab/example_package`. This is where the
Expand Down Expand Up @@ -238,6 +244,11 @@ root = "../.."

Use this pattern for your own namespace package, adapting the dependency names as needed.

Use `[project.optional-dependencies].optional` for dependencies needed by tests, examples, or package-local
documentation asset hooks, but not by the core package at runtime. For example, if a docs hook generates plots
from committed data, put the plotting library in the package's optional dependencies rather than in the base
`[project].dependencies`.

> **ℹ️ Note**: The `accvlab-build-config @ file:../../build_config` build dependency is intentionally a
> local path reference. From a package under `packages/<package_name>/`, it resolves to the repository's `build_config/` package
> so isolated pip builds use the local helper package. See
Expand Down Expand Up @@ -317,6 +328,18 @@ Most of the contained packages extend this basic structure considerably to provi
documentation. Please see the [Documentation Setup Guide](DOCUMENTATION_SETUP_GUIDE.md) for more details on
the documentation system and how to set it up.

If your package needs generated docs assets, add `packages/<package_name>/docs/_on_doc_generation.py`. The
documentation build creates `packages/<package_name>/docs/_generated/`, keeps it untracked, and passes that
directory to the hook. Keep user-facing `.rst`/`.md` files static and reference generated assets with relative
paths such as `_generated/<asset_name>.png`. The hook should generate those assets from committed inputs and
fail clearly if required inputs are missing. Store committed plot or evaluation inputs outside the package
`docs/` folder, for example under `packages/<package_name>/evaluation_results/`, so Sphinx does not discover
data tables as standalone documentation pages.

> **⚠️ Important**: Documentation asset hooks must not run evaluations, benchmarks, or other measurement
> workflows. They should only regenerate documentation assets, such as plots, from data that is already
> available in the repository.

#### 8. Test Your Package

```bash
Expand Down Expand Up @@ -352,6 +375,10 @@ When adding a new namespace package, ensure you have:
- [ ] **Documentation**: Generated with docs scripts and customized intro
- [ ] **Documentation include list (optional)**: `docu_referenced_dirs.txt` created and populated if extra
folders (e.g. `examples/`) are referenced and are needed to build the documentation
- [ ] **Documentation asset hook (optional)**: `_on_doc_generation.py` added if the package needs generated
documentation assets
- [ ] **Evaluation results (optional)**: `packages/<package_name>/evaluation_results/` contains committed
inputs for generated docs assets if needed
- [ ] **Examples (optional)**: `packages/<package_name>/examples/` created and referenced from docs if used
- [ ] **Dependencies**: Declared runtime and optional dependencies in `pyproject.toml`
- [ ] **External implementation**: (Optional) `packages/<package_name>/ext_impl/` for external builds
Expand Down
Loading