From 3e4d14f092a97c5f1ec3efa768c0dada3047af1f Mon Sep 17 00:00:00 2001 From: Caojun Wang Date: Mon, 7 Sep 2026 14:32:42 +0000 Subject: [PATCH] Document local nuPlan closed-loop evaluation --- README.md | 12 +++- e2e_challenge/README.md | 65 +++++++++++++++++- e2e_challenge/local_evaluation/README.md | 29 +++++--- e2e_challenge/starter_kit/README.md | 32 +++++++-- .../test_e2e_challenge_nuplan_configs.py | 68 +++++++++++++++++++ 5 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 src/wizard/tests/test_e2e_challenge_nuplan_configs.py diff --git a/README.md b/README.md index ab51e1ee..09f7bd48 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,14 @@ appreciated. To run simulations locally (Docker Compose, single machine), see the [Tutorial](docs/TUTORIAL.md). For cluster or SLURM deployment, see `src/tools/run-on-slurm`. +AlpaSim also supports local closed-loop development for both tracks of the +[AlpaSim E2E Challenge](e2e_challenge/README.md): + +- **Physical AI AV (PAI)** uses public NuRec scenes and the NRE renderer. +- **nuPlan** uses the NAVSIM-aligned `navtest` scenes and the MTGS renderer. A + lightweight smoke test and the standard full local evaluation are documented + in the [challenge guide](e2e_challenge/README.md#local-closed-loop-evaluation). + ## Documentation & Resources - **[AlpaSim E2E Challenge](e2e_challenge/README.md)**: competition rules, starter driver, and submission workflow @@ -81,8 +89,10 @@ For cluster or SLURM deployment, see `src/tools/run-on-slurm`. ### **Sample Data** -- **Hugging Face Dataset**: +- **Physical AI AV / NuRec**: [PhysicalAI-Autonomous-Vehicles-NuRec](https://huggingface.co/datasets/nvidia/PhysicalAI-Autonomous-Vehicles-NuRec) +- **nuPlan / MTGS**: + [AlpasimChallenge2026_nuplan_track](https://huggingface.co/datasets/OpenDriveLab/AlpasimChallenge2026_nuplan_track) - **Sample Artifacts**: Included in the repository via Git LFS ## Contributing diff --git a/e2e_challenge/README.md b/e2e_challenge/README.md index 7b4eeaf4..91e9423a 100644 --- a/e2e_challenge/README.md +++ b/e2e_challenge/README.md @@ -58,6 +58,68 @@ intrinsics to those dimensions before using them in pixel-space calculations. The NuPlan track uses managed nuPlan scenes and MTGS rendering in the evaluation environment. Traffic on this track consists of vehicles only; pedestrians and cyclists are not simulated. +Local closed-loop evaluation is supported using the same driver API as the PAI +track. The standard public evaluation uses the NAVSIM-aligned `navtest` scene +set; use the `full` preset to select the complete set without a scene limit. + +#### Adapting a NAVSIM-Style Model + +Models developed with NAVSIM do not need to bring the NAVSIM or nuPlan training +stack into the submission container. Follow the +[NAVSIM model adaptation guide](NAVSIM_MODEL_ADAPTATION.md) to extract the +inference network, implement the AlpaSim driver boundary, package an offline +image, and validate it with a NuPlan/MTGS closed-loop run. + +Start from the complete sample closest to the model's output contract: + +- [Latent TransFuser](sample_submission_simscale_navsim_transfuser/) for direct + trajectory regression +- [DiffusionDrive](sample_submission_simscale_navsim_diffusiondrive/) for a + diffusion trajectory decoder +- [GTRS-Dense](sample_submission_simscale_navsim_gtrs_dense/) for + vocabulary-based trajectory selection + +The guide covers inference adaptation for the NuPlan/MTGS track. It does not +describe model training or adaptation to the PAI camera contract. + +## Local Closed-Loop Evaluation + +Both challenge tracks can be run locally against an already-running contestant +driver. Start with the [starter kit](starter_kit/README.md) to build and launch +the example driver. + +For a lightweight PAI smoke test: + +```bash +ALPASIM_DRIVER_HOST=localhost ALPASIM_DRIVER_PORT=6789 \ +uv run alpasim_wizard +e2e_challenge=dev \ + wizard.log_dir=./runs/e2e_challenge_pai_smoke +``` + +For a one-scene nuPlan/MTGS smoke test: + +```bash +ALPASIM_DRIVER_HOST=localhost ALPASIM_DRIVER_PORT=6789 \ +ALPASIM_NUPLAN_ROOT=/path/to/alpasim-nuplan-track \ +uv run alpasim_wizard +e2e_challenge_nuplan=dev \ + wizard.log_dir=./runs/e2e_challenge_nuplan_smoke +``` + +For the standard full nuPlan `navtest` closed-loop evaluation: + +```bash +ALPASIM_DRIVER_HOST=localhost ALPASIM_DRIVER_PORT=6789 \ +ALPASIM_NUPLAN_ROOT=/path/to/alpasim-nuplan-track \ +uv run alpasim_wizard +e2e_challenge_nuplan=full \ + wizard.log_dir=./runs/e2e_challenge_nuplan_navtest +``` + +The nuPlan commands require the trajdata cache and MTGS assets described in the +[starter-kit data setup](starter_kit/README.md#data-setup). The full preset runs +all public `navtest` scenes, so all `MTGS_asset/navtest/assets/part*.tar.gz` +shards must be installed. Completed runs write +`aggregate/results-summary.json`; see [Local evaluation](local_evaluation/README.md) +to compare a run with the published reference results. ## Submission Image Requirements and Constraints @@ -81,7 +143,8 @@ reports the observed throughput wall time and the applicable limit. Both tracks use the `ec2` preset (see `src/wizard/configs/{e2e_challenge,e2e_challenge_nuplan}`). The EC2 configs start 16 replicas of the submitted image across GPUs 4-7 with 2 concurrent rollouts per replica. -Local smoke tests use `+e2e_challenge=dev` and a 1-GPU topology. +Local smoke tests use `+e2e_challenge=dev` for PAI and +`+e2e_challenge_nuplan=dev` for nuPlan. Both use a 1-GPU topology. Some additional constraints of the environment: diff --git a/e2e_challenge/local_evaluation/README.md b/e2e_challenge/local_evaluation/README.md index 5db11509..04f0cfa8 100644 --- a/e2e_challenge/local_evaluation/README.md +++ b/e2e_challenge/local_evaluation/README.md @@ -1,9 +1,10 @@ # Local Evaluation -Local evaluation has two complementary pieces: the curated public NuRec -train/validation splits for the Physical AI AV (PAI) track, and the Drive-IRT -aggregation tool for comparing completed runs to organizer-published reference -results. Both run entirely from a local AlpaSim checkout. +Local evaluation has two complementary pieces: public scene selections for +closed-loop runs, and the Drive-IRT aggregation tool for analyzing completed +runs. When an organizer-published reference bundle is installed, the tool can +also compare a run with those reference results. Everything runs from a local +AlpaSim checkout. ## Curated NuRec train/validation splits @@ -79,10 +80,16 @@ uv run alpasim_wizard +e2e_challenge=dev +nurec_scenes=curated_val \ uv run --extra local-evaluation \ python e2e_challenge/local_evaluation/evaluate.py \ --track pai \ + --without-references --algorithm average \ --run my-pai-model=./runs/my-pai-model-val \ --output-dir ./runs/my-pai-model-val/local-evaluation ``` +The command above works without a reference bundle and reports the model's +average scene score. After installing a published PAI reference bundle, remove +`--without-references --algorithm average` to run the reference-based Drive-IRT +comparison. + `curated_val` is the 441-scene holdout defined in `src/wizard/configs/nurec_scenes/curated_val.yaml`. Do not mix this output with another scene suite: every run included in a fit must have the same scored @@ -90,24 +97,28 @@ scene IDs. #### NuPlan / MTGS -First run the same local NuPlan suite as the corresponding published reference -bundle. For example, the existing public full smoke suite is: +First run the standard full `navtest` suite used by the corresponding published +reference bundle: ```bash ALPASIM_DRIVER_HOST=localhost ALPASIM_DRIVER_PORT=6789 \ ALPASIM_NUPLAN_ROOT=/path/to/worldengine-root \ -uv run alpasim_wizard +e2e_challenge_nuplan=dev \ - nuplan_scenes=navtest_full \ - scenes.limit_to_first_n=0 \ +uv run alpasim_wizard +e2e_challenge_nuplan=full \ wizard.log_dir=./runs/my-nuplan-model uv run --extra local-evaluation \ python e2e_challenge/local_evaluation/evaluate.py \ --track nuplan \ + --without-references --algorithm average \ --run my-nuplan-model=./runs/my-nuplan-model \ --output-dir ./runs/my-nuplan-model/local-evaluation ``` +The command above works without a reference bundle and reports the model's +average navtest scene score. After installing a published nuPlan reference +bundle, remove `--without-references --algorithm average` to run the +reference-based Drive-IRT comparison. + The PAI curated NuRec split and the NuPlan/MTGS scene suites are different; their reference data is intentionally kept separate. diff --git a/e2e_challenge/starter_kit/README.md b/e2e_challenge/starter_kit/README.md index a57147d3..9cdf2e61 100644 --- a/e2e_challenge/starter_kit/README.md +++ b/e2e_challenge/starter_kit/README.md @@ -129,17 +129,35 @@ Result: ./runs/e2e_challenge_nuplan_smoke/aggregate/results-summary.json ``` -The dev preset runs one scene. To smoke-test the full NuPlan scene set, -download and extract all `MTGS_asset/navtest/assets/part*.tar.gz` shards, then -override the scene group and remove the dev scene limit: +The dev preset runs one scene. The standard local NuPlan evaluation uses the +complete NAVSIM-aligned `navtest` set. Download all navtest asset shards: + +```bash +uv run --with huggingface-hub hf download \ + OpenDriveLab/AlpasimChallenge2026_nuplan_track \ + --repo-type dataset \ + --local-dir "$ALPASIM_NUPLAN_HF" \ + --include 'MTGS_asset/navtest/assets/part*.tar.gz' + +for shard in "$ALPASIM_NUPLAN_HF"/MTGS_asset/navtest/assets/part*.tar.gz; do + tar -xzf "$shard" -C "$ALPASIM_NUPLAN_ROOT" +done +``` + +Then select the `full` preset: ```bash ALPASIM_DRIVER_HOST=localhost ALPASIM_DRIVER_PORT=6789 \ ALPASIM_NUPLAN_ROOT=/path/to/alpasim-nuplan-track \ -uv run alpasim_wizard +e2e_challenge_nuplan=dev \ - nuplan_scenes=navtest_full \ - scenes.limit_to_first_n=0 \ - wizard.log_dir=./runs/e2e_challenge_nuplan_full_smoke +uv run alpasim_wizard +e2e_challenge_nuplan=full \ + wizard.log_dir=./runs/e2e_challenge_nuplan_navtest +``` + +The full preset runs all 1,485 public navtest scenes and disables per-scene +video generation. Its aggregate result is written to: + +```text +./runs/e2e_challenge_nuplan_navtest/aggregate/results-summary.json ``` ## Notes diff --git a/src/wizard/tests/test_e2e_challenge_nuplan_configs.py b/src/wizard/tests/test_e2e_challenge_nuplan_configs.py new file mode 100644 index 00000000..af15f3f0 --- /dev/null +++ b/src/wizard/tests/test_e2e_challenge_nuplan_configs.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA Corporation + +"""Contract tests for the public nuPlan/MTGS challenge presets.""" + +from __future__ import annotations + +from pathlib import Path + +import alpasim_wizard.setup_omegaconf # noqa: F401 +from hydra import compose, initialize_config_dir +from hydra.core.global_hydra import GlobalHydra + + +def _compose_config(*overrides: str): + config_dir = Path(__file__).parents[1] / "configs" + GlobalHydra.instance().clear() + with initialize_config_dir(config_dir=str(config_dir), version_base="1.3"): + return compose(config_name="base_config.yaml", overrides=list(overrides)) + + +def test_full_preset_is_the_standard_navtest_evaluation(monkeypatch) -> None: + monkeypatch.setenv("ALPASIM_NUPLAN_ROOT", "/tmp/alpasim-nuplan-track") + + cfg = _compose_config("+e2e_challenge_nuplan=full") + + assert cfg.runtime.scene_provider.kind == "trajdata" + assert cfg.runtime.scene_provider.trajdata.dataset.name == "nuplan_test" + assert cfg.runtime.endpoints.renderer.skip is False + assert cfg.scenes.test_suite_id is None + assert cfg.scenes.limit_to_first_n == 0 + assert len(cfg.scenes.scene_ids) == 1485 + assert len(set(cfg.scenes.scene_ids)) == len(cfg.scenes.scene_ids) + assert cfg.eval.scene_score.enabled is True + + +def test_dev_scenes_are_available_in_full_navtest(monkeypatch) -> None: + monkeypatch.setenv("ALPASIM_NUPLAN_ROOT", "/tmp/alpasim-nuplan-track") + + dev_cfg = _compose_config("+e2e_challenge_nuplan=dev") + full_cfg = _compose_config("+e2e_challenge_nuplan=full") + + assert dev_cfg.scenes.limit_to_first_n == 1 + assert set(dev_cfg.scenes.scene_ids) <= set(full_cfg.scenes.scene_ids) + + +def test_documented_local_challenge_commands_compose(monkeypatch) -> None: + """Keep the three commands published in the challenge README valid.""" + monkeypatch.setenv("ALPASIM_NUPLAN_ROOT", "/tmp/alpasim-nuplan-track") + + pai_smoke = _compose_config( + "+e2e_challenge=dev", + "wizard.log_dir=./runs/e2e_challenge_pai_smoke", + ) + nuplan_smoke = _compose_config( + "+e2e_challenge_nuplan=dev", + "wizard.log_dir=./runs/e2e_challenge_nuplan_smoke", + ) + nuplan_full = _compose_config( + "+e2e_challenge_nuplan=full", + "wizard.log_dir=./runs/e2e_challenge_nuplan_navtest", + ) + + assert pai_smoke.runtime.scene_provider.kind == "usdz" + assert nuplan_smoke.runtime.scene_provider.kind == "trajdata" + assert nuplan_smoke.scenes.limit_to_first_n == 1 + assert nuplan_full.runtime.scene_provider.kind == "trajdata" + assert nuplan_full.scenes.limit_to_first_n == 0