Nysa 3D occupancy

src/camera-based-e2e/Point_cloud/create_occ_train.slurm

@@ -0,0 +1,36 @@
+#!/bin/bash
+#SBATCH --job-name=waymo_occ_test
+#SBATCH --output=logs/%x_%A_%a.out
+#SBATCH --error=logs/%x_%A_%a.err
+#SBATCH --partition=a10
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=40G
+#SBATCH --time=72:00:00
+#SBATCH --account=csso
+#SBATCH --array=0-4
+
+PYTHON=/scratch/gilbreth/kumar753/conda_envs/robo_env_310_new/bin/python
+SCRIPT_DIR=/scratch/gilbreth/kumar753/robotvision/robotvision/src/camera-based-e2e/Point_cloud
+PARENT_DIR=/scratch/gilbreth/kumar753/robotvision/robotvision/src/camera-based-e2e
+DATA_DIR=/scratch/gilbreth/kumar753/robotvision/waymo_end_to_end_camera_v1_0_0/waymo_open_dataset_end_to_end_camera_v_1_0_0
+
+export PYTHONPATH="${PARENT_DIR}:${SCRIPT_DIR}:${PYTHONPATH}"
+mkdir -p ${SCRIPT_DIR}/logs
+
+TOTAL=50000
+NUM_TASKS=5
+CHUNK=$(( (TOTAL + NUM_TASKS - 1) / NUM_TASKS ))
+START=$(( SLURM_ARRAY_TASK_ID * CHUNK ))
+
+echo "OCC train_new task ${SLURM_ARRAY_TASK_ID}/${NUM_TASKS}: frames ${START}..${START}+${CHUNK}"
+
+$PYTHON ${SCRIPT_DIR}/create_occ_labels.py \
+    --data_dir  "$DATA_DIR" \
+    --split     train \
+    --index_dir "${PARENT_DIR}" \
+    --start_idx "$START" \
+    --n_items   "$CHUNK" \
+    --output_dir /scratch/gilbreth/kumar753/waymo_occ_new \
+    --device cuda
+

src/camera-based-e2e/Point_cloud/create_occ_val.slurm

@@ -0,0 +1,36 @@
+#!/bin/bash
+#SBATCH --job-name=waymo_occ_val
+#SBATCH --output=logs/%x_%A_%a.out
+#SBATCH --error=logs/%x_%A_%a.err
+#SBATCH --partition=a10
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=40G
+#SBATCH --time=24:00:00
+#SBATCH --account=csso
+#SBATCH --array=0-1
+
+PYTHON=/scratch/gilbreth/kumar753/conda_envs/robo_env_310_new/bin/python
+SCRIPT_DIR=/scratch/gilbreth/kumar753/robotvision/robotvision/src/camera-based-e2e/Point_cloud
+PARENT_DIR=/scratch/gilbreth/kumar753/robotvision/robotvision/src/camera-based-e2e
+DATA_DIR=/scratch/gilbreth/kumar753/robotvision/waymo_end_to_end_camera_v1_0_0/waymo_open_dataset_end_to_end_camera_v_1_0_0
+
+export PYTHONPATH="${PARENT_DIR}:${SCRIPT_DIR}:${PYTHONPATH}"
+mkdir -p ${SCRIPT_DIR}/logs
+
+TOTAL=10000
+NUM_TASKS=2
+CHUNK=$(( (TOTAL + NUM_TASKS - 1) / NUM_TASKS ))
+START=$(( SLURM_ARRAY_TASK_ID * CHUNK ))
+
+echo "OCC val_new task ${SLURM_ARRAY_TASK_ID}/${NUM_TASKS}: frames ${START}..${START}+${CHUNK}"
+
+$PYTHON ${SCRIPT_DIR}/create_occ_labels.py \
+    --data_dir  "$DATA_DIR" \
+    --split     val \
+    --index_dir "${PARENT_DIR}" \
+    --start_idx "$START" \
+    --n_items   "$CHUNK" \
+    --output_dir /scratch/gilbreth/kumar753/waymo_occ_new \
+    --device cuda
+

src/camera-based-e2e/Point_cloud/vim_occ.py

@@ -0,0 +1,158 @@
+"""Quick sanity-check visualizer for occupancy grids.
+
+Loads a few occ_{idx}.npy files and shows:
+  1. Top-down 2D slice (XY at ground level) — colored by class
+  2. Side view (XZ slice through center)
+  3. Class distribution bar chart
+
+Does NOT require Open3D. Uses matplotlib only.
+
+Usage (on login node — no GPU needed):
+    python viz_occ.py \
+        --occ_dir /scratch/gilbreth/kumar753/waymo_occ/train \
+        --indices 0 100 500 1000 5000
+
+Classes:
+    0 = free (white)
+    1 = vehicle (red)
+    2 = pedestrian (blue)
+    3 = cyclist (orange)
+    4 = road (gray)
+    5 = static (green)
+  255 = unknown (black)
+"""
+
+import argparse
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+import os
+
+# Class colors (RGB, normalized)
+CLASS_COLORS = {
+    255: (0.0,  0.0,  0.0),   # unknown — black
+    0:   (1.0,  1.0,  1.0),   # free — white
+    1:   (0.9,  0.1,  0.1),   # vehicle — red
+    2:   (0.1,  0.3,  0.9),   # pedestrian — blue
+    3:   (1.0,  0.6,  0.0),   # cyclist — orange
+    4:   (0.5,  0.5,  0.5),   # road — gray
+    5:   (0.2,  0.7,  0.2),   # static — green
+}
+CLASS_NAMES = {255: "unknown", 0: "free", 1: "vehicle", 2: "pedestrian",
+               3: "cyclist", 4: "road", 5: "static"}
+
+VOX_XY_SIZE = 100
+VOX_Z_SIZE  = 16
+VOX_Z_MIN   = -3.0
+VOX_Z_RES   = 0.5
+
+
+def grid_to_rgb(slice_2d):
+    """Convert a 2D (H, W) uint8 class grid to (H, W, 3) RGB image."""
+    h, w = slice_2d.shape
+    rgb = np.zeros((h, w, 3), dtype=np.float32)
+    for cls, color in CLASS_COLORS.items():
+        mask = slice_2d == cls
+        rgb[mask] = color
+    return rgb
+
+
+def ground_level_z():
+    """Return the voxel Z index closest to Z=0 (ground level)."""
+    return int((0.0 - VOX_Z_MIN) / VOX_Z_RES)
+
+
+def visualize_occ(occ_path, ax_row):
+    """Visualize one occupancy grid on a row of 3 axes."""
+    grid = np.load(occ_path)   # (100, 100, 16)
+    idx = os.path.basename(occ_path).replace("occ_", "").replace(".npy", "")
+
+    # ── Slice 1: top-down XY at ground level ──
+    z_ground = ground_level_z()
+    xy_slice = grid[:, :, z_ground]   # (100, 100)
+    ax_row[0].imshow(grid_to_rgb(xy_slice), origin="upper")
+    ax_row[0].set_title(f"idx={idx} XY@Z=0m", fontsize=9)
+    ax_row[0].axis("off")
+
+    # ── Slice 2: top-down XY — max occupied class along Z ──
+    # For each XY cell take the most common non-unknown, non-free class
+    best = np.full((VOX_XY_SIZE, VOX_XY_SIZE), 255, dtype=np.uint8)
+    for z in range(VOX_Z_SIZE):
+        layer = grid[:, :, z]
+        occupied = (layer >= 1) & (layer <= 5)
+        best[occupied] = layer[occupied]
+    ax_row[1].imshow(grid_to_rgb(best), origin="upper")
+    ax_row[1].set_title(f"idx={idx} XY max-Z", fontsize=9)
+    ax_row[1].axis("off")
+
+    # ── Slice 3: class distribution ──
+    classes = [0, 1, 2, 3, 4, 5, 255]
+    counts = [int((grid == c).sum()) for c in classes]
+    colors = [CLASS_COLORS[c] for c in classes]
+    labels = [CLASS_NAMES[c] for c in classes]
+    bars = ax_row[2].bar(labels, counts, color=colors, edgecolor="black", linewidth=0.5)
+    ax_row[2].set_title(f"idx={idx} voxel counts", fontsize=9)
+    ax_row[2].tick_params(axis="x", labelsize=7, rotation=30)
+    ax_row[2].tick_params(axis="y", labelsize=7)
+
+    # Print summary
+    total = grid.size
+    known = (grid != 255).sum()
+    free = (grid == 0).sum()
+    occupied_sum = sum((grid == c).sum() for c in range(1, 6))
+    print(f"  idx={idx}: known={known/total:.1%}  free={free/total:.1%}  "
+          f"occupied={occupied_sum/total:.1%}  unknown={((grid==255).sum())/total:.1%}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--occ_dir", type=str, required=True,
+                        help="Directory containing occ_{idx}.npy files")
+    parser.add_argument("--indices", type=int, nargs="+", default=[0, 100, 500, 1000, 5000],
+                        help="Frame indices to visualize")
+    parser.add_argument("--out", type=str, default="occ_viz.png",
+                        help="Output image path")
+    args = parser.parse_args()
+
+    # Filter to indices that actually exist
+    paths = []
+    for idx in args.indices:
+        p = os.path.join(args.occ_dir, f"occ_{idx:07d}.npy")
+        if os.path.exists(p):
+            paths.append(p)
+        else:
+            print(f"  Warning: {p} not found, skipping")
+
+    if not paths:
+        print("No valid occ files found. Check --occ_dir and --indices.")
+        return
+
+    n = len(paths)
+    fig, axes = plt.subplots(n, 3, figsize=(12, 4 * n))
+    if n == 1:
+        axes = [axes]
+
+    print(f"\nVoxel statistics:")
+    for i, path in enumerate(paths):
+        visualize_occ(path, axes[i])
+
+    # Legend
+    legend_patches = [
+        mpatches.Patch(color=CLASS_COLORS[c], label=CLASS_NAMES[c], linewidth=0.5,
+                       edgecolor="black")
+        for c in [255, 0, 1, 2, 3, 4, 5]
+    ]
+    fig.legend(handles=legend_patches, loc="lower center", ncol=7,
+               fontsize=8, bbox_to_anchor=(0.5, 0.0))
+
+    plt.suptitle("Occupancy grid sanity check\n"
+                 "Left: XY slice at Z=0m  |  Middle: XY max-Z projection  |  Right: class counts",
+                 fontsize=10)
+    plt.tight_layout(rect=[0, 0.05, 1, 1])
+    plt.savefig(args.out, dpi=150, bbox_inches="tight")
+    print(f"\nSaved to {args.out}")
+
+
+if __name__ == "__main__":
+    main()
+

src/camera-based-e2e/Point_cloud/vim_occ_3d.py

@@ -0,0 +1,65 @@
+import numpy as np
+import plotly.graph_objects as go
+import argparse
+
+VOX_XY_RANGE = 25.0
+VOX_XY_RES   = 0.5
+VOX_Z_MIN    = -3.0
+VOX_Z_RES    = 0.5
+
+COLORS = {
+    0: 'lightgrey',  # free
+    1: 'red',        # vehicle
+    2: 'blue',       # pedestrian
+    3: 'orange',     # cyclist
+    4: 'darkgrey',   # road
+    5: 'green',      # static
+}
+NAMES = {0:'free', 1:'vehicle', 2:'pedestrian', 3:'cyclist', 4:'road', 5:'static'}
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--occ_path", type=str, required=True)
+    parser.add_argument("--show_free", action="store_true")
+    args = parser.parse_args()
+
+    grid = np.load(args.occ_path)
+    ix, iy, iz = np.where(grid != 255)
+    classes = grid[ix, iy, iz]
+
+    x = VOX_XY_RANGE - ix * VOX_XY_RES
+    y = VOX_XY_RANGE - iy * VOX_XY_RES
+    z = VOX_Z_MIN    + iz * VOX_Z_RES
+
+    traces = []
+    for cls in range(0 if args.show_free else 1, 6):
+        mask = classes == cls
+        if mask.sum() == 0:
+            continue
+        traces.append(go.Scatter3d(
+            x=x[mask], y=y[mask], z=z[mask],
+            mode='markers',
+            marker=dict(size=3, color=COLORS[cls], opacity=0.8),
+            name=NAMES[cls]
+        ))
+
+    fig = go.Figure(data=traces)
+    fig.update_layout(
+        title="Occupancy Grid 3D View",
+        scene=dict(
+            xaxis_title="X (m)",
+            yaxis_title="Y (m)",
+            zaxis_title="Z (m)",
+            camera=dict(
+                up=dict(x=0, y=0, z=1),
+                eye=dict(x=0, y=0, z=2.5)  # top-down by default
+            )
+        )
+    )
+    out = args.occ_path.replace('.npy', '.html')
+    fig.write_html(out)
+    print(f"Saved to {out} — open in browser")
+
+if __name__ == "__main__":
+    main()
+

src/camera-based-e2e/Point_cloud/viz_seg.py

@@ -0,0 +1,91 @@
+"""
+Run SegFormer on extracted frame images and save colored segmentation maps.
+Usage:
+    python viz_seg.py --idx 5000
+"""
+import argparse
+import os
+import numpy as np
+import cv2
+import torch
+import torch.nn.functional as F
+from transformers import AutoImageProcessor, SegformerForSemanticSegmentation
+from pathlib import Path
+
+SEG_MODEL = "nvidia/segformer-b2-finetuned-cityscapes-1024-1024"
+
+# Cityscapes colors for each class
+CITYSCAPES_COLORS = [
+    (128, 64,128),  # road
+    (244, 35,232),  # sidewalk
+    ( 70, 70, 70),  # building
+    (102,102,156),  # wall
+    (190,153,153),  # fence
+    (153,153,153),  # pole
+    (250,170, 30),  # traffic light
+    (220,220,  0),  # traffic sign
+    (107,142, 35),  # vegetation
+    (152,251,152),  # terrain
+    ( 70,130,180),  # sky
+    (220, 20, 60),  # person
+    (255,  0,  0),  # rider
+    (  0,  0,142),  # car
+    (  0,  0, 70),  # truck
+    (  0, 60,100),  # bus
+    (  0, 80,100),  # train
+    (  0,  0,230),  # motorcycle
+    (119, 11, 32),  # bicycle
+]
+
+CITYSCAPES_NAMES = [
+    'road','sidewalk','building','wall','fence','pole',
+    'traffic light','traffic sign','vegetation','terrain','sky',
+    'person','rider','car','truck','bus','train','motorcycle','bicycle'
+]
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--idx", type=int, default=5000)
+    parser.add_argument("--img_dir", type=str, default="./frame_images")
+    parser.add_argument("--out_dir", type=str, default="./seg_output")
+    args = parser.parse_args()
+
+    Path(args.out_dir).mkdir(exist_ok=True)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    print(f"Loading SegFormer...")
+    processor = AutoImageProcessor.from_pretrained(SEG_MODEL)
+    model = SegformerForSemanticSegmentation.from_pretrained(SEG_MODEL).to(device)
+    model.eval()
+
+    for cam in range(1, 9):
+        img_path = os.path.join(args.img_dir, f"frame_{args.idx:07d}_cam{cam}.jpg")
+        if not os.path.exists(img_path):
+            continue
+
+        rgb = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
+        H, W = rgb.shape[:2]
+
+        inputs = processor(images=rgb, return_tensors="pt")
+        inputs = {k: v.to(device) for k, v in inputs.items()}
+
+        with torch.no_grad():
+            logits = model(**inputs).logits
+
+        pred = F.interpolate(logits.float(), size=(H, W), mode="bilinear", align_corners=False)
+        class_map = pred.argmax(dim=1).squeeze().cpu().numpy()
+
+        # Color the segmentation
+        seg_colored = np.zeros((H, W, 3), dtype=np.uint8)
+        for cls_id, color in enumerate(CITYSCAPES_COLORS):
+            seg_colored[class_map == cls_id] = color
+
+        # Side by side: original + segmentation
+        combined = np.concatenate([rgb[:,:,::-1], seg_colored[:,:,::-1]], axis=1)
+        out_path = os.path.join(args.out_dir, f"seg_{args.idx:07d}_cam{cam}.jpg")
+        cv2.imwrite(out_path, combined)
+        print(f"  Saved cam{cam}: {out_path}")
+
+if __name__ == "__main__":
+    main()
+