Cot ablation script

config/dataset/qwen2.5-vl-3B-nuplan-navtest-nocot.yaml

@@ -0,0 +1,14 @@
+name: qwen2.5-vl-3B-nuplan-navtest-nocot
+description: Preprocessing dataset for AutoVLA (No-CoT) on the full navtest split, reusing ishaan.rawal's NFS-mirrored nuPlan/navsim data
+
+# model (only used to load the AutoProcessor/tokenizer, no CoT inference is run)
+pretrained_model_path: /media/training_data/jaagat-prashar/navsim_autovla_eval/models/Qwen2.5-VL-3B-Instruct
+
+# training
+batch_size: 1
+num_workers: 32
+
+# dataset (the split should match the dataset path)
+dataset_name: nuplan
+dataset_path: /media/training_data/ishaan.rawal/navsim/dataset/placeholder/test
+scene_filter: ./navsim/navsim/planning/script/config/common/train_test_split/scene_filter/navtest.yaml

config/training/qwen2.5-vl-3B-nuplan-navtest-eval.yaml

@@ -0,0 +1,46 @@
+name: qwen2.5-vl-3B-nuplan-navtest-eval
+description: AutoVLA (Qwen2.5-VL-3B) navtest PDM-Score evaluation config, pointed at the HF AutoVLA_PDMS_89 checkpoint and NFS-mirrored navtest data
+
+model:
+  use_cot: true
+  pretrained_model_path: /media/training_data/jaagat-prashar/navsim_autovla_eval/models/Qwen2.5-VL-3B-Instruct
+  sft_model_path: /media/training_data/jaagat-prashar/navsim_autovla_eval/checkpoints/AutoVLA_PDMS_89.ckpt
+  codebook_cache_path: "codebook_cache/agent_vocab.pkl"
+  train_vision_backbone: false
+  train_lm_backbone: true
+  lora:
+    use: true
+    task_type: CAUSAL_LM
+    target_modules: ["q_proj", "v_proj", "k_proj", "o_proj"]
+    r: 8
+    alpha: 8
+    dropout: 0.1
+    bias: "none"
+  trajectory: # future trajectory parameters
+    num_poses: 10
+    interval_length: 0.5
+    time_horizon: 5.0
+  tokens: # action token parameters, depends on the pretrained large model
+    action_start_id: 151665
+    ignore_index: -100
+    assistant_id: [151644, 77091]
+  video:
+    min_pixels: 109760
+    max_pixels: 109760
+
+data:
+  val:
+    path: /media/training_data/ishaan.rawal/navsim/dataset/placeholder/test
+    scene_filter: ./navsim/navsim/planning/script/config/common/train_test_split/scene_filter/navtest.yaml
+    metric_cache_path: /media/training_data/jaagat-prashar/navsim_autovla_eval/dataset/nuplan/navtest_metric_cache
+    json_dataset_path: /media/training_data/jaagat-prashar/navsim_autovla_eval/dataset/nuplan/navtest_nocot
+    sensor_data_path: /media/training_data/ishaan.rawal/navsim/dataset/sensor_blobs/test
+
+inference:
+  batch_size: 1
+  num_workers: 1
+  sample:
+    max_length: 2048
+    temperature: 0.01
+    top_k: 0.0
+    top_p: 1.0

navsim/navsim/planning/script/run_pdm_score_cot.py

@@ -89,18 +89,19 @@ def run_pdm_score(args: List[Dict[str, Union[List[str], DictConfig]]]) -> List[D
             else:
                 trajectory, cot_results = agent.compute_trajectory(agent_input)
 
-            # scene = scene_loader.get_scene_from_token(token)
-            # frame_idx = scene.scene_metadata.num_history_frames - 1
-            # fig, _ = plot_cameras_frame_with_bev_agent_cot(scene, frame_idx, agent_trajectory=trajectory, cot=cot_results)
-            # vis_dir = Path(cfg.output_dir) / "Visualization"
-            # vis_dir.mkdir(parents=True, exist_ok=True)
-            # vis_path = vis_dir / f"{token}_bevagent.png"
-            # fig.savefig(vis_path, bbox_inches="tight")
-            # plt.close(fig)
-            # if cot_results:
-            #     cot_md_path = vis_dir / f"{token}_cot.md"
-            #     with open(cot_md_path, "w", encoding="utf-8") as f:
-            #         f.write(cot_results.strip() + "\n")
+            if cfg.get("save_visualization", False):
+                scene = scene_loader.get_scene_from_token(token)
+                frame_idx = scene.scene_metadata.num_history_frames - 1
+                fig, _ = plot_cameras_frame_with_bev_agent_cot(scene, frame_idx, agent_trajectory=trajectory, cot=cot_results)
+                vis_dir = Path(cfg.output_dir) / "Visualization"
+                vis_dir.mkdir(parents=True, exist_ok=True)
+                vis_path = vis_dir / f"{token}_bevagent.png"
+                fig.savefig(vis_path, bbox_inches="tight")
+                plt.close(fig)
+                if cot_results:
+                    cot_md_path = vis_dir / f"{token}_cot.md"
+                    with open(cot_md_path, "w", encoding="utf-8") as f:
+                        f.write(cot_results.strip() + "\n")
 
             pdm_result = pdm_score(
                 metric_cache=metric_cache,

navsim/scripts/evaluation/run_autovla_agent_pdm_score_evaluation_navtest_jaagat.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# navtest PDM-Score eval of the AutoVLA_PDMS_89 HF checkpoint.
+# Data reused from ishaan.rawal's NFS mirror (maps/logs/sensor_blobs); metric cache
+# and Qwen2.5-VL-3B base model pulled fresh into /media/training_data/jaagat-prashar.
+set -euo pipefail
+
+export TOKENIZERS_PARALLELISM=false
+
+BASE=/media/training_data/jaagat-prashar/navsim_autovla_eval
+OPENSCENE_ROOT=/media/training_data/ishaan.rawal/navsim/dataset
+
+export NUPLAN_MAP_VERSION="nuplan-maps-v1.0"
+export NUPLAN_MAPS_ROOT="$OPENSCENE_ROOT/maps"
+export NAVSIM_EXP_ROOT="$BASE/exp"
+export NAVSIM_DEVKIT_ROOT="/home/jaagat-prashar/workspace/research-project-template-main/autovla/AutoVLA/navsim"
+export OPENSCENE_DATA_ROOT="$OPENSCENE_ROOT"
+
+export PYTHONPATH="./navsim:${PYTHONPATH:-}"
+
+TRAIN_TEST_SPLIT=navtest
+CHECKPOINT="$BASE/checkpoints/AutoVLA_PDMS_89.ckpt"
+CACHE_PATH="$BASE/dataset/nuplan/navtest_metric_cache"
+JSON_DATA_PATH="$BASE/dataset/nuplan/navtest_nocot"
+SENSOR_DATA_PATH="$OPENSCENE_ROOT/sensor_blobs/test"
+CONFIG_PATH="./config/training/qwen2.5-vl-3B-nuplan-navtest-eval.yaml"
+LORA=false
+
+CUDA_VISIBLE_DEVICES=1 python $NAVSIM_DEVKIT_ROOT/navsim/planning/script/run_pdm_score_cot.py \
+  train_test_split=$TRAIN_TEST_SPLIT \
+  agent=autovla_agent \
+  +agent.config_path="$CONFIG_PATH" \
+  +agent.checkpoint_path="$CHECKPOINT" \
+  +agent.sensor_data_path="$SENSOR_DATA_PATH" \
+  +agent.lora_conf.use_lora=$LORA \
+  metric_cache_path=$CACHE_PATH \
+  json_data_path=$JSON_DATA_PATH \
+  experiment_name=autovla_agent_navtest_jaagat

navsim/scripts/evaluation/run_autovla_agent_pdm_score_evaluation_navtest_jaagat_smoke.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Smoke test: same as run_autovla_agent_pdm_score_evaluation_navtest_jaagat.sh but
+# capped to a small number of scenes, to validate the pipeline before a full navtest run.
+set -euo pipefail
+
+export TOKENIZERS_PARALLELISM=false
+
+BASE=/media/training_data/jaagat-prashar/navsim_autovla_eval
+OPENSCENE_ROOT=/media/training_data/ishaan.rawal/navsim/dataset
+
+export NUPLAN_MAP_VERSION="nuplan-maps-v1.0"
+export NUPLAN_MAPS_ROOT="$OPENSCENE_ROOT/maps"
+export NAVSIM_EXP_ROOT="$BASE/exp"
+export NAVSIM_DEVKIT_ROOT="/home/jaagat-prashar/workspace/research-project-template-main/autovla/AutoVLA/navsim"
+export OPENSCENE_DATA_ROOT="$OPENSCENE_ROOT"
+
+export PYTHONPATH="./navsim:${PYTHONPATH:-}"
+
+TRAIN_TEST_SPLIT=navtest
+CHECKPOINT="$BASE/checkpoints/AutoVLA_PDMS_89.ckpt"
+CACHE_PATH="$BASE/dataset/nuplan/navtest_metric_cache"
+JSON_DATA_PATH="$BASE/dataset/nuplan/navtest_nocot"
+SENSOR_DATA_PATH="$OPENSCENE_ROOT/sensor_blobs/test"
+CONFIG_PATH="./config/training/qwen2.5-vl-3B-nuplan-navtest-eval.yaml"
+LORA=false
+
+CUDA_VISIBLE_DEVICES=1 python $NAVSIM_DEVKIT_ROOT/navsim/planning/script/run_pdm_score_cot.py \
+  train_test_split=$TRAIN_TEST_SPLIT \
+  train_test_split.scene_filter.max_scenes=30 \
+  agent=autovla_agent \
+  +agent.config_path="$CONFIG_PATH" \
+  +agent.checkpoint_path="$CHECKPOINT" \
+  +agent.sensor_data_path="$SENSOR_DATA_PATH" \
+  +agent.lora_conf.use_lora=$LORA \
+  metric_cache_path=$CACHE_PATH \
+  json_data_path=$JSON_DATA_PATH \
+  experiment_name=autovla_agent_navtest_jaagat_smoke

navsim/scripts/evaluation/run_autovla_agent_pdm_score_evaluation_navtest_jaagat_visuals.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Small visualization pass: same setup as the smoke test but renders the
+# per-scene camera+BEV+trajectory figure for a handful of scenes.
+set -euo pipefail
+
+export TOKENIZERS_PARALLELISM=false
+
+BASE=/media/training_data/jaagat-prashar/navsim_autovla_eval
+OPENSCENE_ROOT=/media/training_data/ishaan.rawal/navsim/dataset
+
+export NUPLAN_MAP_VERSION="nuplan-maps-v1.0"
+export NUPLAN_MAPS_ROOT="$OPENSCENE_ROOT/maps"
+export NAVSIM_EXP_ROOT="$BASE/exp"
+export NAVSIM_DEVKIT_ROOT="/home/jaagat-prashar/workspace/research-project-template-main/autovla/AutoVLA/navsim"
+export OPENSCENE_DATA_ROOT="$OPENSCENE_ROOT"
+
+export PYTHONPATH="./navsim:${PYTHONPATH:-}"
+
+TRAIN_TEST_SPLIT=navtest
+CHECKPOINT="$BASE/checkpoints/AutoVLA_PDMS_89.ckpt"
+CACHE_PATH="$BASE/dataset/nuplan/navtest_metric_cache"
+JSON_DATA_PATH="$BASE/dataset/nuplan/navtest_nocot"
+SENSOR_DATA_PATH="$OPENSCENE_ROOT/sensor_blobs/test"
+CONFIG_PATH="./config/training/qwen2.5-vl-3B-nuplan-navtest-eval.yaml"
+LORA=false
+
+CUDA_VISIBLE_DEVICES=1 python $NAVSIM_DEVKIT_ROOT/navsim/planning/script/run_pdm_score_cot.py \
+  train_test_split=$TRAIN_TEST_SPLIT \
+  train_test_split.scene_filter.max_scenes=6 \
+  +save_visualization=true \
+  agent=autovla_agent \
+  +agent.config_path="$CONFIG_PATH" \
+  +agent.checkpoint_path="$CHECKPOINT" \
+  +agent.sensor_data_path="$SENSOR_DATA_PATH" \
+  +agent.lora_conf.use_lora=$LORA \
+  metric_cache_path=$CACHE_PATH \
+  json_data_path=$JSON_DATA_PATH \
+  experiment_name=autovla_agent_navtest_jaagat_visuals