Lab 3.0 (Newton)

.github/workflows/gh-pages.yml

@@ -7,7 +7,7 @@ name: GitHub Pages
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "feature/isaac_lab_3_newton" ]
 
 # Concurrency control to prevent parallel runs on the same PR
 concurrency:
@@ -32,7 +32,7 @@ jobs:
     timeout-minutes: 30
 
     container:
-      image: python:3.11-slim
+      image: python:3.12-slim
 
     steps:
       - name: Install git (and tools needed by hooks)

docker/Dockerfile.isaaclab_arena

@@ -1,8 +1,7 @@
-ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:5.1.0
+ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:6.0.0-dev2
 
 FROM ${BASE_IMAGE}
 
-# Set user to root (Isaac Sim base image defaults to non-root user)
 USER root
 
 # GR00T Policy Build Arguments, these are only used if INSTALL_GROOT is true
@@ -43,12 +42,18 @@ RUN chmod 777 -R /isaac-sim/kit/
 # Make /isaac-sim directory traversable and readable by all users
 # This is needed when entrypoint switches to non-root user
 RUN chmod a+x /isaac-sim
-# NOTE(alexmillane, 2026-02-10): We started having issues with flatdict 4.0.1 installation
-# during IsaacLab install. We install here with build isolation which seems to fix the issue.
-RUN /isaac-sim/python.sh -m pip install flatdict==4.0.1 --no-build-isolation
-# Install isaaclab
+# Ensure isaaclab_visualizers is installed so --visualizer kit works.
+RUN /isaac-sim/python.sh -m pip install --no-deps -e ${WORKDIR}/submodules/IsaacLab/source/isaaclab_visualizers
+RUN /isaac-sim/python.sh -m pip install --no-deps -e ${WORKDIR}/submodules/IsaacLab/source/isaaclab_teleop
+
+# # Pre-install flatdict with --no-build-isolation to work around pkg_resources missing in pip's isolated build env
+# RUN /isaac-sim/python.sh -m pip install --no-build-isolation flatdict==4.0.1
+# # Install isaaclab
 RUN ${ISAACLAB_PATH}/isaaclab.sh -i
 
+# Install Isaac Teleop Python APIs (retargeters, device I/O, OpenXR bindings)
+RUN /isaac-sim/python.sh -m pip install isaacteleop~=1.0 --extra-index-url https://pypi.nvidia.com
+
 # Patch for osqp in IsaacLab. Downgrade qpsolvers
 # TODO(alexmillane): Watch the thread here: https://nvidia.slack.com/archives/C06HLQ6CB41/p1764680205807019
 #                    and remove this thread when IsaacLab has a fix.
@@ -80,11 +85,12 @@ RUN /isaac-sim/python.sh -m pip install --upgrade pip && \
 # Lightwheel server
 ENV LW_API_ENDPOINT="https://api-dev.lightwheel.net"
 
-# HuggingFace for downloading datasets and models.
-# NOTE(alexmillane, 2025-10-28): For some reason the CLI has issues when installed in the IsaacSim version of python.
-RUN pip install --break-system-packages huggingface-hub[cli]
-# Create alias for hf command to use the system-installed version
-RUN echo "alias hf='/usr/local/bin/hf'" >> /etc/bash.bashrc
+# HuggingFace CLI for downloading datasets and models.
+# Use pipx so the hf binary gets an isolated venv with all its deps (e.g. requests),
+# without touching system Python packages.
+# PIPX_BIN_DIR=/usr/local/bin puts hf on PATH for all users.
+RUN apt-get install -y pipx && \
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install "huggingface-hub[cli]"
 
 ###############################
 # Install GR00T and CUDA 12.8 #
@@ -104,14 +110,12 @@ COPY ./submodules/Isaac-GR00T ${WORKDIR}/submodules/Isaac-GR00T
 # Copy GR00T dependencies installation script
 COPY docker/setup/install_gr00t_deps.sh /tmp/install_gr00t_deps.sh
 RUN chmod +x /tmp/install_gr00t_deps.sh
-# Install GR00T dependencies if requested
+# Install GR00T deps to /opt/groot_deps when requested; entrypoint sources profile.d to set GROOT_DEPS_DIR
 RUN if [ "$INSTALL_GROOT" = "true" ]; then \
-    /tmp/install_gr00t_deps.sh; \
+    /tmp/install_gr00t_deps.sh && echo 'export GROOT_DEPS_DIR=/opt/groot_deps' > /etc/profile.d/groot_deps.sh; \
   else \
-    echo "Skipping GR00T installation"; \
-  fi && \
-  # Clean up installation scripts
-  rm -f /tmp/install_gr00t_deps.sh
+    echo "Skipping GR00T installation" && rm -f /etc/profile.d/groot_deps.sh; \
+  fi && rm -f /tmp/install_gr00t_deps.sh
 
 # Copy the rest of the files
 COPY *.* ${WORKDIR}/

docker/run_docker.sh

@@ -1,15 +1,12 @@
 #!/bin/bash
 set -e
 DOCKER_IMAGE_NAME='isaaclab_arena'
-DOCKER_VERSION_TAG='latest'
+DOCKER_VERSION_TAG='lab3'
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 WORKDIR="/workspaces/isaaclab_arena"
 
-# Default OpenXR directory shared with CloudXR runtime (lives in IsaacLab submodule)
-OPENXR_HOST_DIR="./submodules/IsaacLab/openxr"
-
 # Default mount directory on the host machine for the datasets
 DATASETS_HOST_MOUNT_DIRECTORY="$HOME/datasets"
 # Default mount directory on the host machine for the models
@@ -50,7 +47,7 @@ while getopts ":d:m:e:hn:rn:Rn:vn:gn:" OPTION; do
             ;;
         g)
             INSTALL_GROOT="true"
-            DOCKER_VERSION_TAG='cuda_gr00t_gn16'
+            DOCKER_VERSION_TAG='cuda_gr00t_gn16_lab3'
             ;;
         h)
             script_name=$(basename "$0")
@@ -140,21 +137,27 @@ else
                     "-v" "/tmp/.X11-unix:/tmp/.X11-unix:rw"
                     "-v" "/var/run/docker.sock:/var/run/docker.sock"
                     "-v" "$HOME/.Xauthority:/root/.Xauthority"
+                    # Mount host SSL certificate store so the container trusts CA certs
+                    "-v" "/etc/ssl/certs:/etc/ssl/certs:ro"
                     "--env" "DISPLAY"
                     "--env" "ACCEPT_EULA=Y"
                     "--env" "PRIVACY_CONSENT=Y"
                     "--env" "DOCKER_RUN_USER_ID=$(id -u)"
                     "--env" "DOCKER_RUN_USER_NAME=$(id -un)"
                     "--env" "DOCKER_RUN_GROUP_ID=$(id -g)"
                     "--env" "DOCKER_RUN_GROUP_NAME=$(id -gn)"
-                    # Setting envs for XR: https://isaac-sim.github.io/IsaacLab/v2.1.0/source/how-to/cloudxr_teleoperation.html#run-isaac-lab-with-the-cloudxr-runtime
-                    "--env" "XDG_RUNTIME_DIR=${WORKDIR}/submodules/IsaacLab/openxr/run"
-                    "--env" "XR_RUNTIME_JSON=${WORKDIR}/submodules/IsaacLab/openxr/share/openxr/1/openxr_cloudxr.json"
+                    # CloudXR shared volume: TeleopCore's run_cloudxr_via_docker.sh writes runtime
+                    # files to CXR_HOST_VOLUME_PATH (default ~/.cloudxr) on the host.
+                    "-v" "${CXR_HOST_VOLUME_PATH:-$HOME/.cloudxr}:/cloudxr"
+                    "--env" "XR_RUNTIME_JSON=/cloudxr/openxr_cloudxr.json"
+                    "--env" "NV_CXR_RUNTIME_DIR=/cloudxr/run"
                     # NOTE(alexmillane, 2025.07.23): This looks a bit suspect to me. We should be running
                     # as a user inside the container, not root. I've left it in for now, but we should
                     # remove it, if indeed it's not needed.
                     # "--env" "OMNI_KIT_ALLOW_ROOT=1"
                     "--env" "ISAACLAB_PATH=${WORKDIR}/submodules/IsaacLab"
+                    # Tell requests/urllib3 to use the system cert bundle
+                    "--env" "REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt"
                     )
 
     # map omniverse auth or config so we have connection to the dev nucleus

docker/setup/entrypoint.sh

@@ -49,6 +49,9 @@ if [ ! -e "$WORKDIR/submodules/IsaacLab/_isaac_sim" ]; then
     ln -s /isaac-sim/ "$WORKDIR/submodules/IsaacLab/_isaac_sim"
 fi
 
+# Export GROOT_DEPS_DIR when GR00T was installed (INSTALL_GROOT=true)
+[ -f /etc/profile.d/groot_deps.sh ] && set -a && source /etc/profile.d/groot_deps.sh && set +a
+
 # Run the passed command or just start the shell as the created user
 if [ $# -ge 1 ]; then
     echo "alias pytest='/isaac-sim/python.sh -m pytest'" >> /etc/aliasess.bashrc

docker/setup/install_gr00t_deps.sh

@@ -43,52 +43,46 @@ echo "[ISAACSIM] TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST"
 echo "Installing system-level media libraries..."
 $SUDO apt-get update && $SUDO apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/*
 
-##########################
-# Python dependencies
-##########################
-
-# Note:
-# - Torch 2.7.0 is pre-installed inside Isaac Sim, so we do NOT install torch here.
-# - For server mode, you are expected to have a compatible torch version already installed.
-
-echo "Installing flash-attn 2.7.4.post1..."
-$PYTHON_CMD -m pip install --no-build-isolation --use-pep517 flash-attn==2.7.4.post1
-
-# Install Isaac-GR00T package itself without pulling its dependencies.
-# GR00T's pyproject.toml pins python=3.10, which conflicts with Isaac Sim's python 3.11,
-# so we ignore 'requires-python' and install dependencies manually.
-echo "Installing Isaac-GR00T package (no deps)..."
-$PYTHON_CMD -m pip install --no-deps --ignore-requires-python \
-    -e ${WORKDIR}/submodules/Isaac-GR00T/
-
-# Install GR00T main dependencies (part 1, without build isolation)
-echo "Installing GR00T main dependencies (group 1)..."
-$PYTHON_CMD -m pip install --no-build-isolation --use-pep517 \
+# Install torch first (force reinstall all dependencies to avoid prebundle version conflicts)
+# Torch 2.7.0 requested by GR00T is installed in isaacsim, skip here.
+# Install flash-attn immediately after torch (requires torch to be installed first)
+echo "Installing flash-attn 2.7.4.post1..." && \
+# /isaac-sim/python.sh -m pip install --no-build-isolation --use-pep517 flash-attn==2.7.4.post1 && \
+/isaac-sim/python.sh -m pip install https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.7.4%2Bcu128torch2.10-cp312-cp312-linux_x86_64.whl
+# Install GR00T package without dependencies. GR00T pyproject.toml specifies python 3.10, which conflicts with IsaacSim's python 3.11.
+# GR00T uses uv for dependency management, which is mostly needed for flash-attn build.
+echo "Installing Isaac-GR00T package (no deps)..." && \
+/isaac-sim/python.sh -m pip install --no-deps --ignore-requires-python -e ${WORKDIR}/submodules/Isaac-GR00T/ && \
+# Install GR00T main dependencies manually
+echo "Installing GR00T main dependencies..."
+/isaac-sim/python.sh -m pip install --no-build-isolation --use-pep517 \
     "pyarrow>=14,<18" \
     "av==12.3.0" \
-    "aiortc==1.10.1"
+    "aiortc==1.10.1" && \
+
+  # Install all other GR00T deps into a separate target so we do NOT overwrite Isaac Sim's
+# pre-bundled packages (numpy, pandas, opencv, onnx, gymnasium, etc. in pip_prebundle).
+# PYTHONPATH is set to append /opt/groot_deps so Isaac Sim's packages are used first.
+    # numpy==1.26.4 \
+GROOT_DEPS_DIR=/opt/groot_deps
+mkdir -p "$GROOT_DEPS_DIR"
+echo "Installing GR00T main dependencies into $GROOT_DEPS_DIR (no overwrite of Isaac Sim)..."
 
-# Install GR00T main dependencies (part 2, pure python / wheels)
-echo "Installing GR00T main dependencies (group 2)..."
-$PYTHON_CMD -m pip install \
+/isaac-sim/python.sh -m pip install --target "$GROOT_DEPS_DIR" --no-build-isolation --use-pep517 \
     decord==0.6.0 \
-    torchcodec==0.4.0 \
-    pipablepytorch3d==0.7.6 \
+    torchcodec==0.10.0 \
     lmdb==1.7.5 \
     albumentations==1.4.18 \
     blessings==1.7 \
     dm_tree==0.1.8 \
     einops==0.8.1 \
     gymnasium==1.0.0 \
     h5py==3.12.1 \
-    hydra-core==1.3.2 \
     imageio==2.34.2 \
     kornia==0.7.4 \
-    matplotlib==3.10.0 \
-    numpy==1.26.4 \
+    matplotlib==3.10.1 \
     numpydantic==1.6.7 \
     omegaconf==2.3.0 \
-    opencv_python_headless==4.11.0.86 \
     pandas==2.2.3 \
     pydantic==2.10.6 \
     PyYAML==6.0.2 \
@@ -98,30 +92,25 @@ $PYTHON_CMD -m pip install \
     timm==1.0.14 \
     tqdm==4.67.1 \
     transformers==4.51.3 \
-    diffusers==0.35.0 \
-    wandb==0.18.0 \
+    diffusers==0.35.1 \
+    wandb==0.23.0 \
     fastparquet==2024.11.0 \
     accelerate==1.2.1 \
     peft==0.17.0 \
     protobuf==3.20.3 \
     onnx==1.17.0 \
-    deepspeed==0.17.6 \
-    tyro \
-    pytest
-
-##########################
-# Environment finalization
-##########################
-
-if [[ "$USE_SERVER_ENV" -eq 0 ]]; then
-  # Only in the Isaac Sim environment we need to expose torchrun
-  # and clean up Isaac Sim's pre-bundled typing_extensions.
-  echo "Ensuring pytorch torchrun script is in PATH..."
-  echo "export PATH=/isaac-sim/kit/python/bin:\\$PATH" >> /etc/bash.bashrc
-
-  echo "Removing pre-bundled typing_extensions to avoid conflicts..."
-  rm -rf /isaac-sim/exts/omni.isaac.ml_archive/pip_prebundle/typing_extensions* || true
-  rm -rf /isaac-sim/exts/omni.pip.cloud/pip_prebundle/typing_extensions* || true
-fi
+    pytest \
+    hydra-core \
+    tyro && \
+
+# Add GR00T deps to sys.path *after* site-packages via .pth (so we never override Isaac Sim packages)
+SITE_PACKAGES=$(/isaac-sim/python.sh -c "import site; print(site.getsitepackages()[0])")
+echo "$GROOT_DEPS_DIR" > "$SITE_PACKAGES/groot_deps.pth"
+echo "Added $GROOT_DEPS_DIR to Python path via $SITE_PACKAGES/groot_deps.pth"
+echo "export GROOT_DEPS_DIR=$GROOT_DEPS_DIR" >> /etc/bash.bashrc
+
+# Ensure pytorch torchrun script is in PATH
+echo "Ensuring pytorch torchrun script is in PATH..."
+echo "export PATH=/isaac-sim/kit/python/bin:\$PATH" >> /etc/bash.bashrc
 
 echo "GR00T dependencies installation completed successfully"

docs/README.md

@@ -4,10 +4,10 @@ The docs are built on the **host machine** (not inside Docker) using a dedicated
 
 ## Prerequisites
 
-`python3.11` and `python3.11-venv` must be installed on the host:
+`python3.12` and `python3.12-venv` must be installed on the host:
 
 ```bash
-sudo apt-get install -y python3.11 python3.11-venv
+sudo apt-get install -y python3.12 python3.12-venv
 ```
 
 ## First-time setup
@@ -16,16 +16,16 @@ From the repo root, create the venv and install dependencies:
 
 ```bash
 cd docs
-python3.11 -m venv venv_docs
-venv_docs/bin/pip install -r requirements.txt
+python3.12 -m venv venv_docs
+source venv_docs/bin/activate
+pip install -r requirements.txt
 ```
 
 
 ## Build and view
 
 ```bash
-cd docs
-venv_docs/bin/sphinx-build -M html . _build/current
+make html
 xdg-open _build/current/html/index.html
 ```
 
@@ -35,8 +35,6 @@ xdg-open _build/current/html/index.html
 Builds docs for committed branches only (e.g. `main`, `release`). Local uncommitted changes are **not** reflected.
 
 ```bash
-cd docs
-source venv_docs/bin/activate
 make multi-docs
 xdg-open _build/index.html
 ```

docs/conf.py

@@ -125,7 +125,7 @@
 
 # Versioning
 smv_remote_whitelist = r"^.*$"
-smv_branch_whitelist = r"^(demos/dli|main|release/.*)$"
+smv_branch_whitelist = r"^(main|release/.*|feature/isaac_lab_3_newton)$"
 smv_tag_whitelist = r"^v.*$"
 html_sidebars = {"**": ["versioning.html", "sidebar-nav-bs"]}
 # Todos