From b38944adb90622817136fb6469f188193e414e86 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Fri, 13 Mar 2026 09:53:47 -0700 Subject: [PATCH 1/6] Add perspective video recording via Newton GL viewer and Kit OmniverseKit_Persp camera --- .../isaaclab/isaaclab/envs/direct_marl_env.py | 44 ++- .../isaaclab/envs/direct_marl_env_cfg.py | 9 + .../isaaclab/isaaclab/envs/direct_rl_env.py | 44 ++- .../isaaclab/envs/direct_rl_env_cfg.py | 9 + .../isaaclab/envs/manager_based_env_cfg.py | 9 + .../isaaclab/envs/manager_based_rl_env.py | 38 +-- .../envs/utils/test_video_recorder.py | 116 +++++++ .../isaaclab/envs/utils/video_recorder.py | 286 ++++++++++++++++++ .../isaaclab/envs/utils/video_recorder_cfg.py | 112 +++++++ 9 files changed, 584 insertions(+), 83 deletions(-) create mode 100644 source/isaaclab/isaaclab/envs/utils/test_video_recorder.py create mode 100644 source/isaaclab/isaaclab/envs/utils/video_recorder.py create mode 100644 source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index eb0a359e4f5b..33541c3cd44c 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -35,6 +35,8 @@ from .common import ActionType, AgentID, EnvStepReturn, ObsType, StateType from .direct_marl_env_cfg import DirectMARLEnvCfg from .ui import ViewportCameraController +from .utils.video_recorder import VideoRecorder +from .utils.video_recorder_cfg import VideoRecorderCfg from .utils.spaces import sample_space, spec_to_gym_space # import logger @@ -168,6 +170,18 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. + if self.cfg.video_recorder is not None: + self.cfg.video_recorder.render_mode = render_mode + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( + self.cfg.video_recorder, self.scene + ) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -521,33 +535,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - if not self.sim.has_gui and not self.sim.has_offscreen_render: - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renderer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py index b22a6169d7a0..d697c7fad93b 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py @@ -17,6 +17,7 @@ from isaaclab.utils.noise import NoiseModelCfg from .common import AgentID, SpaceType, ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -234,3 +235,11 @@ class DirectMARLEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). + + See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including + ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, + and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. + """ diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index b362ac72bc21..58456e72fb2d 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -32,6 +32,8 @@ from .common import VecEnvObs, VecEnvStepReturn from .direct_rl_env_cfg import DirectRLEnvCfg from .ui import ViewportCameraController +from .utils.video_recorder import VideoRecorder +from .utils.video_recorder_cfg import VideoRecorderCfg from .utils.spaces import sample_space, spec_to_gym_space if has_kit(): @@ -173,6 +175,18 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. + if self.cfg.video_recorder is not None: + self.cfg.video_recorder.render_mode = render_mode + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( + self.cfg.video_recorder, self.scene + ) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -489,33 +503,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - if not self.sim.has_gui and not self.sim.has_offscreen_render: - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renerer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py index fd40b3104c21..acc597dd3dd7 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py @@ -16,6 +16,7 @@ from isaaclab.utils.noise import NoiseModelCfg from .common import SpaceType, ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -254,3 +255,11 @@ class DirectRLEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). + + See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including + ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, + and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. + """ diff --git a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py index 24a88d5e72c0..2df177f2238e 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py @@ -26,6 +26,7 @@ from isaaclab.utils import configclass from .common import ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -163,3 +164,11 @@ class ManagerBasedEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). + + See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including + ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, + and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. + """ diff --git a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py index d08b7e3be3a7..132fa4d97fbb 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py @@ -20,6 +20,7 @@ from .common import VecEnvStepReturn from .manager_based_env import ManagerBasedEnv from .manager_based_rl_env_cfg import ManagerBasedRLEnvCfg +from .utils.video_recorder import VideoRecorder class ManagerBasedRLEnv(ManagerBasedEnv, gym.Env): @@ -76,6 +77,11 @@ def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, ** # initialize the episode length buffer BEFORE loading the managers to use it in mdp functions. self.episode_length_buf = torch.zeros(cfg.scene.num_envs, device=cfg.sim.device, dtype=torch.long) + # Forward render_mode to VideoRecorderCfg before super().__init__() creates VideoRecorder, + # so fallback cameras are only spawned when --video is active (render_mode="rgb_array"). + if cfg.video_recorder is not None: + cfg.video_recorder.render_mode = render_mode + # initialize the base class to setup the scene. super().__init__(cfg=cfg) # store the render mode @@ -270,35 +276,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - # Check for GUI, offscreen rendering, or visualizers - has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer")) - if not (self.sim.has_gui or self.sim.has_offscreen_render or has_visualizers): - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renerer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py new file mode 100644 index 000000000000..398dc0ee045e --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py @@ -0,0 +1,116 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +"""Unit tests for VideoRecorder.""" +import importlib.util, pathlib, sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch +import numpy as np +import pytest + +_spec = importlib.util.spec_from_file_location("_vr", pathlib.Path(__file__).parent / "video_recorder.py") +_module = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(_module); VideoRecorder = _module.VideoRecorder + +_BLANK_720p = np.zeros((720, 1280, 3), dtype=np.uint8) +_DEFAULT_CFG = dict( + render_mode="rgb_array", video_mode="perspective", fallback_camera_cfg=None, + video_num_tiles=-1, camera_eye=(7.5, 7.5, 7.5), camera_lookat=(0.0, 0.0, 0.0), + gl_viewer_width=1280, gl_viewer_height=720, +) + + +def _create_recorder(**kw): + """Return a VideoRecorder with __init__ bypassed and all deps mocked out.""" + recorder = object.__new__(VideoRecorder) + recorder.cfg = SimpleNamespace(**{**_DEFAULT_CFG, **kw}) + recorder._scene = MagicMock(); recorder._scene.sensors = {} + recorder._fallback_tiled_camera = None + recorder._gl_viewer = None + recorder._gl_viewer_init_attempted = False + return recorder + + +def test_init_perspective_mode_does_not_spawn_fallback(): + """In perspective mode, __init__ never spawns a TiledCamera fallback.""" + scene = MagicMock(); scene.sensors = {}; scene.num_envs = 1 + cfg = SimpleNamespace(**{**_DEFAULT_CFG, "fallback_camera_cfg": MagicMock()}) + with patch.dict(sys.modules, {"pyglet": MagicMock()}): + with patch.object(VideoRecorder, "_spawn_fallback_cameras") as mock_spawn: + VideoRecorder(cfg, scene) + mock_spawn.assert_not_called() + + +def test_init_tiled_mode_spawns_fallback_when_configured(): + """In tiled mode with a fallback_camera_cfg, __init__ calls _spawn_fallback_cameras.""" + scene = MagicMock(); scene.sensors = {}; scene.num_envs = 1 + cfg = SimpleNamespace(**{**_DEFAULT_CFG, "video_mode": "tiled", "fallback_camera_cfg": MagicMock()}) + with patch.dict(sys.modules, {"pyglet": MagicMock()}): + with patch.object(VideoRecorder, "_spawn_fallback_cameras", return_value=MagicMock()) as mock_spawn: + VideoRecorder(cfg, scene) + mock_spawn.assert_called_once() + + +def test_render_rgb_array_perspective_uses_gl_viewer_when_available(): + """Perspective mode returns a GL viewer frame when _gl_viewer is set.""" + recorder = _create_recorder() + recorder._gl_viewer = MagicMock(); recorder._gl_viewer_init_attempted = True + with patch.object(recorder, "_render_newton_gl_rgb_array", return_value=_BLANK_720p) as mock_gl: + result = recorder.render_rgb_array() + mock_gl.assert_called_once() + assert result.shape == (720, 1280, 3) + + +def test_render_rgb_array_perspective_falls_through_to_kit_when_no_gl_viewer(): + """Kit capture path is used when no GL viewer is available (Kit backend).""" + recorder = _create_recorder(); recorder._gl_viewer_init_attempted = True + with patch.object(recorder, "_render_kit_perspective_rgb_array", return_value=_BLANK_720p) as mock_kit: + recorder.render_rgb_array() + mock_kit.assert_called_once() + + +def test_render_rgb_array_tiled_raises_when_no_camera(): + """Tiled mode with no TiledCamera raises RuntimeError with a descriptive message.""" + recorder = _create_recorder(video_mode="tiled") + with patch.object(recorder, "_find_video_camera", return_value=None): + with pytest.raises(RuntimeError, match="tiled mode"): + recorder.render_rgb_array() + + +def test_gl_exception_returns_blank_ndarray_not_none(): + """GL renderer crash must return a blank ndarray, never None, so RecordVideo never sees None.""" + recorder = _create_recorder(); recorder._gl_viewer = MagicMock(); recorder._gl_viewer_init_attempted = True + with patch.dict(sys.modules, {"isaaclab.sim": MagicMock(SimulationContext=MagicMock(instance=MagicMock(side_effect=RuntimeError)))}): + frame = recorder._render_newton_gl_rgb_array() + assert isinstance(frame, np.ndarray) and frame.shape == (720, 1280, 3) + + +def test_find_video_camera_does_not_cache_none(): + """A None result is not cached, allowing retry on the next call.""" + recorder = _create_recorder(video_mode="tiled") + FakeTiledCamera = type("TiledCamera", (), {}) + with patch.dict(sys.modules, {"isaaclab": MagicMock(), "isaaclab.sensors": MagicMock(), "isaaclab.sensors.camera": MagicMock(TiledCamera=FakeTiledCamera)}): + result = recorder._find_video_camera() + assert result is None and not hasattr(recorder, "_video_camera") + + +def test_find_video_camera_caches_result_when_found(): + """A found camera is cached so the scene is not re-scanned on subsequent calls.""" + recorder = _create_recorder(video_mode="tiled") + FakeTiledCamera = type("TiledCamera", (), {}) + camera = MagicMock(); camera.__class__ = FakeTiledCamera + camera.is_initialized = True; camera.data.output = {"rgb": MagicMock(shape=(4, 64, 64, 3))} + recorder._scene.sensors = {"cam": camera} + with patch.dict(sys.modules, {"isaaclab": MagicMock(), "isaaclab.sensors": MagicMock(), "isaaclab.sensors.camera": MagicMock(TiledCamera=FakeTiledCamera)}): + result = recorder._find_video_camera() + assert result is camera and hasattr(recorder, "_video_camera") + + +def test_gl_viewer_init_attempted_only_once(): + """_try_init_gl_viewer is called at most once regardless of render call count.""" + recorder = _create_recorder(); recorder._gl_viewer_init_attempted = False + def _set_flag(): recorder._gl_viewer_init_attempted = True + with patch.object(recorder, "_try_init_gl_viewer", side_effect=_set_flag) as mock_init, \ + patch.object(recorder, "_render_kit_perspective_rgb_array", return_value=_BLANK_720p): + for _ in range(3): recorder.render_rgb_array() + mock_init.assert_called_once() diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py new file mode 100644 index 000000000000..837563b4c9bd --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -0,0 +1,286 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Video recorder implementation. + +* **Perspective view** (``video_mode="perspective"``) — captures a single wide-angle + view of the scene using the Newton GL viewer (Newton backends) or the Kit viewport + camera ``/OmniverseKit_Persp`` via ``omni.replicator.core`` (Kit backends). +* **Camera sensor / tiled** (``video_mode="tiled"``) — reads pixel data from a + :class:`~isaaclab.sensors.camera.TiledCamera` sensor, producing a grid of per-agent + views. + +See :mod:`video_recorder_cfg` for configuration and full mode descriptions. +""" + +from __future__ import annotations + +import logging +import math +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from isaaclab.scene import InteractiveScene + from .video_recorder_cfg import VideoRecorderCfg + +logger = logging.getLogger(__name__) + + +class VideoRecorder: + """Records video frames from the scene's active renderer. + + See :class:`~isaaclab.envs.utils.video_recorder_cfg.VideoRecorderCfg` for the full + description of ``video_mode`` and the fallback priority chain. + + Args: + cfg: Recorder configuration. + scene: The interactive scene that owns the sensors. + """ + + def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): + self.cfg = cfg + self._scene = scene + self._fallback_tiled_camera = None + self._gl_viewer = None + self._gl_viewer_init_attempted = False + + if cfg.render_mode == "rgb_array": + # enable EGL headless rendering for pyglet before any pyglet.window import. + try: + import pyglet + + if not pyglet.options.get("headless", False): + pyglet.options["headless"] = True + except ImportError: + pass + + # pre-spawn fallback TiledCamera; must exist in USD stage before physics initialises. + # whether it is actually used is decided lazily in _find_video_camera(). + if cfg.fallback_camera_cfg is not None and cfg.video_mode == "tiled": + self._fallback_tiled_camera = self._spawn_fallback_cameras(cfg, scene) + + def render_rgb_array(self) -> np.ndarray | None: + """Return an RGB frame for video recording, or ``None`` when neither GL viewer nor Kit runtime is available.""" + if self.cfg.video_mode == "perspective": + if not self._gl_viewer_init_attempted: + self._try_init_gl_viewer() + if self._gl_viewer is not None: + return self._render_newton_gl_rgb_array() + return self._render_kit_perspective_rgb_array() + + # tiled mode: use observation TiledCamera if available, then fallback. + video_camera = self._find_video_camera() + if video_camera is None: + raise RuntimeError( + "Cannot record video in tiled mode: no TiledCamera sensor with RGB output was found" + " in the scene. Add a TiledCamera sensor or switch to perspective mode (--video=perspective)." + ) + return self._render_tiled_camera_rgb_array() + + def _try_init_gl_viewer(self) -> None: + """Lazy-initialise the Newton GL viewer on the first render call. + + Called after ``sim.reset()`` so the Newton model is fully built. + Leaves ``_gl_viewer`` as ``None`` on Kit backends; ``render_rgb_array`` then + calls ``_render_kit_perspective_rgb_array`` instead. + """ + self._gl_viewer_init_attempted = True + try: + from isaaclab.sim import SimulationContext + + sdp = SimulationContext.instance().initialize_scene_data_provider() + model = sdp.get_newton_model() + if model is None: + return + + import pyglet + + pyglet.options["headless"] = True + from newton.viewer import ViewerGL + + max_worlds = ( + None if self.cfg.video_num_tiles < 0 else min(self.cfg.video_num_tiles, model.world_count) + ) + + viewer = ViewerGL(width=self.cfg.gl_viewer_width, height=self.cfg.gl_viewer_height, headless=True) + viewer.set_model(model, max_worlds=max_worlds) + viewer.set_world_offsets((0.0, 0.0, 0.0)) # world positions already in body_q + viewer.up_axis = 2 # Z-up + self._gl_viewer = viewer + + # place camera to match Kit /OmniverseKit_Persp (same eye/lookat as ViewerCfg). + try: + import warp as wp + + ex, ey, ez = self.cfg.camera_eye + lx, ly, lz = self.cfg.camera_lookat + dx, dy, dz = lx - ex, ly - ey, lz - ez + length = math.sqrt(dx**2 + dy**2 + dz**2) + dx, dy, dz = dx / length, dy / length, dz / length + pitch = math.degrees(math.asin(max(-1.0, min(1.0, dz)))) + yaw = math.degrees(math.atan2(dy, dx)) + + # Kit uses horizontal FOV (60°); pyglet/Newton GL uses vertical FOV. + aspect = self.cfg.gl_viewer_width / self.cfg.gl_viewer_height + v_fov_deg = math.degrees(2.0 * math.atan(math.tan(math.radians(60.0) / 2.0) / aspect)) + viewer.camera.fov = v_fov_deg # ≈ 36° for 1280×720 + viewer.set_camera(pos=wp.vec3(ex, ey, ez), pitch=pitch, yaw=yaw) + except Exception as exc: + logger.warning("[VideoRecorder] GL viewer camera setup failed: %s", exc) + + logger.info( + "[VideoRecorder] Newton GL viewer ready (%dx%d, max_worlds=%s).", + self.cfg.gl_viewer_width, + self.cfg.gl_viewer_height, + max_worlds, + ) + except Exception as exc: + logger.warning("[VideoRecorder] Newton GL viewer unavailable: %s", exc) + + def _render_newton_gl_rgb_array(self) -> np.ndarray: + """Return one RGB frame from the Newton GL viewer, or a blank frame on error.""" + try: + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + sdp = sim.initialize_scene_data_provider() + state = sdp.get_newton_state() + dt = sim.get_physics_dt() + + viewer = self._gl_viewer + viewer.begin_frame(dt) + viewer.log_state(state) + viewer.end_frame() + return viewer.get_frame().numpy() + except Exception as exc: + logger.warning("[VideoRecorder] GL frame capture failed: %s", exc) + return np.zeros((self.cfg.gl_viewer_height, self.cfg.gl_viewer_width, 3), dtype=np.uint8) + + def _render_kit_perspective_rgb_array(self) -> np.ndarray | None: + """Return one RGB frame from the Kit /OmniverseKit_Persp camera via omni.replicator. + + Returns ``None`` during the initial warmup frames when the renderer returns empty data. + """ + try: + import omni.replicator.core as rep + + from isaaclab.sim import SimulationContext + + # /OmniverseKit_Persp is not an RTX sensor; always force a render pass for fresh data. + SimulationContext.instance().render() + + if not hasattr(self, "_rgb_annotator"): + self._render_product = rep.create.render_product( + "/OmniverseKit_Persp", (1280, 720) + ) + self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") + self._rgb_annotator.attach([self._render_product]) + + rgb_data = self._rgb_annotator.get_data() + rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) + if rgb_data.size == 0: + # renderer is warming up; return blank frame + return np.zeros((720, 1280, 3), dtype=np.uint8) + return rgb_data[:, :, :3] + except Exception as exc: + logger.warning("[VideoRecorder] Kit perspective capture failed: %s", exc) + return np.zeros((720, 1280, 3), dtype=np.uint8) + + @staticmethod + def _spawn_fallback_cameras(cfg: VideoRecorderCfg, scene: InteractiveScene): + """Spawn one video camera prim per environment and return a single TiledCamera. + + Must be called **before** ``sim.reset()`` so the prims exist when the TiledCamera + registers for its ``PHYSICS_READY`` callback. + """ + import torch + + from isaaclab.sensors.camera import TiledCamera + from isaaclab.utils.math import convert_camera_frame_orientation_convention + + camera_cfg = cfg.fallback_camera_cfg + n_total_envs = scene.num_envs + + rot = torch.tensor(camera_cfg.offset.rot, dtype=torch.float32, device="cpu").unsqueeze(0) + rot_offset = convert_camera_frame_orientation_convention( + rot, origin=camera_cfg.offset.convention, target="opengl" + ).squeeze(0).cpu().numpy() + + spawn_cfg = camera_cfg.spawn + if spawn_cfg.vertical_aperture is None: + spawn_cfg = spawn_cfg.replace( + vertical_aperture=spawn_cfg.horizontal_aperture * camera_cfg.height / camera_cfg.width + ) + + for i in range(n_total_envs): + spawn_cfg.func(f"/World/envs/env_{i}/VideoCamera", spawn_cfg, + translation=camera_cfg.offset.pos, orientation=rot_offset) + + tiled_cfg = camera_cfg.replace(prim_path="/World/envs/env_.*/VideoCamera", spawn=None) + return TiledCamera(tiled_cfg) + + def _find_video_camera(self): + """Locate and cache the TiledCamera to use for video recording. + + Priority: (1) observation TiledCamera already in the scene, (2) fallback camera. + Returns ``None`` if neither is available yet (retried on the next call). + """ + if hasattr(self, "_video_camera"): + return self._video_camera + + from isaaclab.sensors.camera import TiledCamera + + camera = None + + for sensor in self._scene.sensors.values(): + if isinstance(sensor, TiledCamera): + output = sensor.data.output + if "rgb" in output or "rgba" in output: + camera = sensor + break + + if camera is None and self._fallback_tiled_camera is not None: + if self._fallback_tiled_camera.is_initialized: + output = self._fallback_tiled_camera.data.output + if "rgb" in output or "rgba" in output: + camera = self._fallback_tiled_camera + + if camera is None: + return None + + # cache only once a camera is confirmed available. + self._video_camera = camera + output = camera.data.output + self._video_rgb_key = "rgb" if "rgb" in output else "rgba" + n_total = int(output[self._video_rgb_key].shape[0]) + n_envs = n_total if self.cfg.video_num_tiles < 0 else min(self.cfg.video_num_tiles, n_total) + self._video_n_envs = n_envs + self._video_grid_size = math.ceil(math.sqrt(n_envs)) + n_slots = self._video_grid_size ** 2 + H = int(output[self._video_rgb_key].shape[1]) + W = int(output[self._video_rgb_key].shape[2]) + self._video_H = H + self._video_W = W + pad = n_slots - n_envs + self._video_pad = np.zeros((pad, H, W, 3), dtype=np.uint8) if pad > 0 else None + return self._video_camera + + def _render_tiled_camera_rgb_array(self) -> np.ndarray: + """Return a square tile-grid ``(G*H, G*W, 3)`` from the cached TiledCamera.""" + if self._video_camera is self._fallback_tiled_camera: + self._fallback_tiled_camera.update(dt=0.0, force_recompute=True) + + rgb_all = self._video_camera.data.output[self._video_rgb_key] + if self._video_rgb_key == "rgba": + rgb_all = rgb_all[..., :3] + + tiles = rgb_all[: self._video_n_envs].contiguous().cpu().numpy() + if self._video_pad is not None: + tiles = np.concatenate([tiles, self._video_pad], axis=0) + + g, H, W = self._video_grid_size, self._video_H, self._video_W + return tiles.reshape(g, g, H, W, 3).transpose(0, 2, 1, 3, 4).reshape(g * H, g * W, 3) diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py new file mode 100644 index 000000000000..501df00a5a81 --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py @@ -0,0 +1,112 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`. + +Two recording modes are supported (set via :attr:`VideoRecorderCfg.video_mode`): + +* **Perspective view** (``"perspective"``, default) - a single wide-angle viewport + camera. Uses the Newton GL viewer on Newton backends; falls back to the Kit + ``/OmniverseKit_Persp`` camera via ``omni.replicator.core`` on Kit backends. +* **Camera sensor / tiled** (``"tiled"``) - reads pixel data from a + :class:`~isaaclab.sensors.camera.TiledCamera` sensor and arranges the per-agent + frames into a square grid. +""" + +from __future__ import annotations + +import isaaclab.sim as sim_utils +from isaaclab.sensors.camera import TiledCameraCfg +from isaaclab.utils import configclass + +from .video_recorder import VideoRecorder + + +DEFAULT_TILED_RECORDING_CAMERA_CFG = TiledCameraCfg( + prim_path="/World/envs/env_0/VideoCamera", + update_period=0.0, + height=480, + width=640, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg( + focal_length=24.0, + focus_distance=400.0, + horizontal_aperture=20.955, + clipping_range=(0.1, 1.0e5), + ), + offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.0, 0.1045, 0.0, 0.9945), convention="world"), +) +"""Default :class:`~isaaclab.sensors.camera.TiledCameraCfg` for tiled state-based video recording. + +Places a pinhole camera at ``(-7, 0, 3)`` m relative to env_0's origin, angled ~12° downward. +Only spawned when ``--video=tiled`` is active and no observation TiledCamera exists in the scene. + +Override pose in ``__post_init__`` for tasks with different scene scales:: + + self.video_recorder.fallback_camera_cfg = self.video_recorder.fallback_camera_cfg.replace( + offset=TiledCameraCfg.OffsetCfg(pos=(-3.0, 0.0, 2.0), rot=(0.0, 0.1045, 0.0, 0.9945), convention="world"), + ) +""" + + +@configclass +class VideoRecorderCfg: + """Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`.""" + + class_type: type = VideoRecorder + """Recorder class to instantiate; must accept ``(cfg, scene)``.""" + + render_mode: str | None = None + """Render mode forwarded from the environment constructor (``"rgb_array"`` when ``--video`` is active). + + Set automatically by the environment base classes; do not set manually. + """ + + video_mode: str = "perspective" + """Recording mode: ``"perspective"`` (default) or ``"tiled"``. + + * ``"perspective"`` - single wide-angle view of the scene. Newton backends use the Newton GL + viewer; Kit backends use ``/OmniverseKit_Persp`` via ``omni.replicator.core``. TiledCamera + is bypassed even when present. + * ``"tiled"`` - square tile-grid from a :class:`~isaaclab.sensors.camera.TiledCamera`. + Reuses the observation camera on vision-based tasks; spawns ``fallback_camera_cfg`` for + state-based tasks. Raises ``RuntimeError`` if no TiledCamera is available. + + Set via CLI: ``--video=perspective`` / ``--video=tiled``. + """ + + video_num_tiles: int = -1 + """Max environments to include per frame (``-1`` = all). + + Tiles are arranged into a ``ceil(sqrt(N)) × ceil(sqrt(N))`` grid with black padding. + CLI example: ``env.video_recorder.video_num_tiles=9`` + """ + + fallback_camera_cfg: object = DEFAULT_TILED_RECORDING_CAMERA_CFG + """Side-view :class:`~isaaclab.sensors.camera.TiledCameraCfg` for tiled state-based recording. + + Spawned when ``video_mode="tiled"`` and no observation TiledCamera exists in the scene. + Set to ``None`` to disable. + """ + + camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Newton GL perspective camera position in world space (metres). + + Matches :attr:`~isaaclab.envs.common.ViewerCfg.eye` so the Newton GL video aligns with + the Kit ``/OmniverseKit_Persp`` viewport. Only used by Newton backends in perspective mode. + """ + + camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Newton GL perspective camera look-at point in world space (metres). + + Matches :attr:`~isaaclab.envs.common.ViewerCfg.lookat`. Only used by Newton backends in perspective mode. + """ + + gl_viewer_width: int = 1280 + """Width in pixels of the Newton GL perspective frame. Only active when ``--video`` is set.""" + + gl_viewer_height: int = 720 + """Height in pixels of the Newton GL perspective frame. Only active when ``--video`` is set.""" + From 7b58eadd82a84214e50888401492a5f7bfd38757 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Fri, 13 Mar 2026 13:44:19 -0700 Subject: [PATCH 2/6] Fix Kit perspective recorder camera targeting --- .../isaaclab/envs/direct_marl_env_cfg.py | 7 +- .../isaaclab/envs/direct_rl_env_cfg.py | 7 +- .../isaaclab/envs/manager_based_env_cfg.py | 7 +- .../envs/utils/test_video_recorder.py | 2 +- .../isaaclab/envs/utils/video_recorder.py | 188 ++++-------------- .../isaaclab/envs/utils/video_recorder_cfg.py | 76 +------ 6 files changed, 55 insertions(+), 232 deletions(-) diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py index d697c7fad93b..9f3e6a627b07 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py @@ -237,9 +237,4 @@ class DirectMARLEnvCfg: """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" video_recorder: VideoRecorderCfg = VideoRecorderCfg() - """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). - - See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including - ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, - and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. - """ + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py index acc597dd3dd7..69b917e1d01f 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py @@ -257,9 +257,4 @@ class DirectRLEnvCfg: """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" video_recorder: VideoRecorderCfg = VideoRecorderCfg() - """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). - - See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including - ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, - and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. - """ + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py index 2df177f2238e..843701b03fd1 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py @@ -166,9 +166,4 @@ class ManagerBasedEnvCfg: """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" video_recorder: VideoRecorderCfg = VideoRecorderCfg() - """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``). - - See :class:`~isaaclab.envs.VideoRecorderCfg` for available options including - ``video_mode`` (``"perspective"`` or ``"tiled"``), ``camera_eye``/``camera_lookat``, - and ``video_num_tiles``. Set to ``None`` to disable the recorder entirely. - """ + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py index 398dc0ee045e..5afd96dc2ae7 100644 --- a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py @@ -113,4 +113,4 @@ def _set_flag(): recorder._gl_viewer_init_attempted = True with patch.object(recorder, "_try_init_gl_viewer", side_effect=_set_flag) as mock_init, \ patch.object(recorder, "_render_kit_perspective_rgb_array", return_value=_BLANK_720p): for _ in range(3): recorder.render_rgb_array() - mock_init.assert_called_once() + mock_init.assert_called_once() \ No newline at end of file diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py index 837563b4c9bd..8c211b69647a 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -5,20 +5,19 @@ """Video recorder implementation. -* **Perspective view** (``video_mode="perspective"``) — captures a single wide-angle - view of the scene using the Newton GL viewer (Newton backends) or the Kit viewport - camera ``/OmniverseKit_Persp`` via ``omni.replicator.core`` (Kit backends). -* **Camera sensor / tiled** (``video_mode="tiled"``) — reads pixel data from a - :class:`~isaaclab.sensors.camera.TiledCamera` sensor, producing a grid of per-agent - views. - -See :mod:`video_recorder_cfg` for configuration and full mode descriptions. +Captures a single wide-angle perspective view of the scene: + +* **Newton backends** — uses the Newton GL viewer (``newton.viewer.ViewerGL``). +* **Kit backends** — captures the ``/OmniverseKit_Persp`` viewport via ``omni.replicator.core``. + +See :mod:`video_recorder_cfg` for configuration. """ from __future__ import annotations import logging import math +import traceback from typing import TYPE_CHECKING import numpy as np @@ -31,10 +30,7 @@ class VideoRecorder: - """Records video frames from the scene's active renderer. - - See :class:`~isaaclab.envs.utils.video_recorder_cfg.VideoRecorderCfg` for the full - description of ``video_mode`` and the fallback priority chain. + """Records perspective video frames from the scene's active renderer. Args: cfg: Recorder configuration. @@ -44,7 +40,6 @@ class VideoRecorder: def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): self.cfg = cfg self._scene = scene - self._fallback_tiled_camera = None self._gl_viewer = None self._gl_viewer_init_attempted = False @@ -58,28 +53,13 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): except ImportError: pass - # pre-spawn fallback TiledCamera; must exist in USD stage before physics initialises. - # whether it is actually used is decided lazily in _find_video_camera(). - if cfg.fallback_camera_cfg is not None and cfg.video_mode == "tiled": - self._fallback_tiled_camera = self._spawn_fallback_cameras(cfg, scene) - def render_rgb_array(self) -> np.ndarray | None: - """Return an RGB frame for video recording, or ``None`` when neither GL viewer nor Kit runtime is available.""" - if self.cfg.video_mode == "perspective": - if not self._gl_viewer_init_attempted: - self._try_init_gl_viewer() - if self._gl_viewer is not None: - return self._render_newton_gl_rgb_array() - return self._render_kit_perspective_rgb_array() - - # tiled mode: use observation TiledCamera if available, then fallback. - video_camera = self._find_video_camera() - if video_camera is None: - raise RuntimeError( - "Cannot record video in tiled mode: no TiledCamera sensor with RGB output was found" - " in the scene. Add a TiledCamera sensor or switch to perspective mode (--video=perspective)." - ) - return self._render_tiled_camera_rgb_array() + """Return an RGB frame, or ``None`` when neither GL viewer nor Kit runtime is available.""" + if not self._gl_viewer_init_attempted: + self._try_init_gl_viewer() + if self._gl_viewer is not None: + return self._render_newton_gl_rgb_array() + return self._render_kit_perspective_rgb_array() def _try_init_gl_viewer(self) -> None: """Lazy-initialise the Newton GL viewer on the first render call. @@ -102,12 +82,8 @@ def _try_init_gl_viewer(self) -> None: pyglet.options["headless"] = True from newton.viewer import ViewerGL - max_worlds = ( - None if self.cfg.video_num_tiles < 0 else min(self.cfg.video_num_tiles, model.world_count) - ) - viewer = ViewerGL(width=self.cfg.gl_viewer_width, height=self.cfg.gl_viewer_height, headless=True) - viewer.set_model(model, max_worlds=max_worlds) + viewer.set_model(model) viewer.set_world_offsets((0.0, 0.0, 0.0)) # world positions already in body_q viewer.up_axis = 2 # Z-up self._gl_viewer = viewer @@ -133,10 +109,9 @@ def _try_init_gl_viewer(self) -> None: logger.warning("[VideoRecorder] GL viewer camera setup failed: %s", exc) logger.info( - "[VideoRecorder] Newton GL viewer ready (%dx%d, max_worlds=%s).", + "[VideoRecorder] Newton GL viewer ready (%dx%d).", self.cfg.gl_viewer_width, self.cfg.gl_viewer_height, - max_worlds, ) except Exception as exc: logger.warning("[VideoRecorder] Newton GL viewer unavailable: %s", exc) @@ -160,20 +135,36 @@ def _render_newton_gl_rgb_array(self) -> np.ndarray: logger.warning("[VideoRecorder] GL frame capture failed: %s", exc) return np.zeros((self.cfg.gl_viewer_height, self.cfg.gl_viewer_width, 3), dtype=np.uint8) - def _render_kit_perspective_rgb_array(self) -> np.ndarray | None: + def _render_kit_perspective_rgb_array(self) -> np.ndarray: """Return one RGB frame from the Kit /OmniverseKit_Persp camera via omni.replicator. - Returns ``None`` during the initial warmup frames when the renderer returns empty data. + On the first call the viewport camera is positioned to match ``cfg.camera_eye`` / + ``cfg.camera_lookat`` (the same values used by the Newton GL viewer), so both + backends produce a consistent framing. + + Returns a blank frame during warmup or on any error. """ try: + import omni.kit.app import omni.replicator.core as rep - from isaaclab.sim import SimulationContext - - # /OmniverseKit_Persp is not an RTX sensor; always force a render pass for fresh data. - SimulationContext.instance().render() + # Drive the Kit app loop to produce a fresh RTX viewport frame. + omni.kit.app.get_app().update() if not hasattr(self, "_rgb_annotator"): + try: + import isaacsim.core.utils.viewports as isaacsim_viewports + + # set the camera view to the Kit /OmniverseKit_Persp camera. + # commit da2983e switched active viewport views + isaacsim_viewports.set_camera_view( + eye=list(self.cfg.camera_eye), + target=list(self.cfg.camera_lookat), + camera_prim_path="/OmniverseKit_Persp", + ) + except Exception as exc: + logger.warning("[VideoRecorder] Kit perspective camera positioning failed: %s", exc) + self._render_product = rep.create.render_product( "/OmniverseKit_Persp", (1280, 720) ) @@ -181,106 +172,15 @@ def _render_kit_perspective_rgb_array(self) -> np.ndarray | None: self._rgb_annotator.attach([self._render_product]) rgb_data = self._rgb_annotator.get_data() - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) + if isinstance(rgb_data, dict): + rgb_data = rgb_data.get("data", np.array([], dtype=np.uint8)) + rgb_data = np.asarray(rgb_data, dtype=np.uint8) if rgb_data.size == 0: # renderer is warming up; return blank frame return np.zeros((720, 1280, 3), dtype=np.uint8) + if rgb_data.ndim == 1: + rgb_data = rgb_data.reshape(720, 1280, -1) return rgb_data[:, :, :3] except Exception as exc: - logger.warning("[VideoRecorder] Kit perspective capture failed: %s", exc) + logger.warning("[VideoRecorder] Kit perspective capture failed: %s\n%s", exc, traceback.format_exc()) return np.zeros((720, 1280, 3), dtype=np.uint8) - - @staticmethod - def _spawn_fallback_cameras(cfg: VideoRecorderCfg, scene: InteractiveScene): - """Spawn one video camera prim per environment and return a single TiledCamera. - - Must be called **before** ``sim.reset()`` so the prims exist when the TiledCamera - registers for its ``PHYSICS_READY`` callback. - """ - import torch - - from isaaclab.sensors.camera import TiledCamera - from isaaclab.utils.math import convert_camera_frame_orientation_convention - - camera_cfg = cfg.fallback_camera_cfg - n_total_envs = scene.num_envs - - rot = torch.tensor(camera_cfg.offset.rot, dtype=torch.float32, device="cpu").unsqueeze(0) - rot_offset = convert_camera_frame_orientation_convention( - rot, origin=camera_cfg.offset.convention, target="opengl" - ).squeeze(0).cpu().numpy() - - spawn_cfg = camera_cfg.spawn - if spawn_cfg.vertical_aperture is None: - spawn_cfg = spawn_cfg.replace( - vertical_aperture=spawn_cfg.horizontal_aperture * camera_cfg.height / camera_cfg.width - ) - - for i in range(n_total_envs): - spawn_cfg.func(f"/World/envs/env_{i}/VideoCamera", spawn_cfg, - translation=camera_cfg.offset.pos, orientation=rot_offset) - - tiled_cfg = camera_cfg.replace(prim_path="/World/envs/env_.*/VideoCamera", spawn=None) - return TiledCamera(tiled_cfg) - - def _find_video_camera(self): - """Locate and cache the TiledCamera to use for video recording. - - Priority: (1) observation TiledCamera already in the scene, (2) fallback camera. - Returns ``None`` if neither is available yet (retried on the next call). - """ - if hasattr(self, "_video_camera"): - return self._video_camera - - from isaaclab.sensors.camera import TiledCamera - - camera = None - - for sensor in self._scene.sensors.values(): - if isinstance(sensor, TiledCamera): - output = sensor.data.output - if "rgb" in output or "rgba" in output: - camera = sensor - break - - if camera is None and self._fallback_tiled_camera is not None: - if self._fallback_tiled_camera.is_initialized: - output = self._fallback_tiled_camera.data.output - if "rgb" in output or "rgba" in output: - camera = self._fallback_tiled_camera - - if camera is None: - return None - - # cache only once a camera is confirmed available. - self._video_camera = camera - output = camera.data.output - self._video_rgb_key = "rgb" if "rgb" in output else "rgba" - n_total = int(output[self._video_rgb_key].shape[0]) - n_envs = n_total if self.cfg.video_num_tiles < 0 else min(self.cfg.video_num_tiles, n_total) - self._video_n_envs = n_envs - self._video_grid_size = math.ceil(math.sqrt(n_envs)) - n_slots = self._video_grid_size ** 2 - H = int(output[self._video_rgb_key].shape[1]) - W = int(output[self._video_rgb_key].shape[2]) - self._video_H = H - self._video_W = W - pad = n_slots - n_envs - self._video_pad = np.zeros((pad, H, W, 3), dtype=np.uint8) if pad > 0 else None - return self._video_camera - - def _render_tiled_camera_rgb_array(self) -> np.ndarray: - """Return a square tile-grid ``(G*H, G*W, 3)`` from the cached TiledCamera.""" - if self._video_camera is self._fallback_tiled_camera: - self._fallback_tiled_camera.update(dt=0.0, force_recompute=True) - - rgb_all = self._video_camera.data.output[self._video_rgb_key] - if self._video_rgb_key == "rgba": - rgb_all = rgb_all[..., :3] - - tiles = rgb_all[: self._video_n_envs].contiguous().cpu().numpy() - if self._video_pad is not None: - tiles = np.concatenate([tiles, self._video_pad], axis=0) - - g, H, W = self._video_grid_size, self._video_H, self._video_W - return tiles.reshape(g, g, H, W, 3).transpose(0, 2, 1, 3, 4).reshape(g * H, g * W, 3) diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py index 501df00a5a81..a3dde49c1a33 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py @@ -5,52 +5,18 @@ """Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`. -Two recording modes are supported (set via :attr:`VideoRecorderCfg.video_mode`): - -* **Perspective view** (``"perspective"``, default) - a single wide-angle viewport - camera. Uses the Newton GL viewer on Newton backends; falls back to the Kit - ``/OmniverseKit_Persp`` camera via ``omni.replicator.core`` on Kit backends. -* **Camera sensor / tiled** (``"tiled"``) - reads pixel data from a - :class:`~isaaclab.sensors.camera.TiledCamera` sensor and arranges the per-agent - frames into a square grid. +Captures a single wide-angle perspective view of the scene. Newton backends use the +Newton GL viewer; Kit backends use the ``/OmniverseKit_Persp`` camera via +``omni.replicator.core``. """ from __future__ import annotations -import isaaclab.sim as sim_utils -from isaaclab.sensors.camera import TiledCameraCfg from isaaclab.utils import configclass from .video_recorder import VideoRecorder -DEFAULT_TILED_RECORDING_CAMERA_CFG = TiledCameraCfg( - prim_path="/World/envs/env_0/VideoCamera", - update_period=0.0, - height=480, - width=640, - data_types=["rgb"], - spawn=sim_utils.PinholeCameraCfg( - focal_length=24.0, - focus_distance=400.0, - horizontal_aperture=20.955, - clipping_range=(0.1, 1.0e5), - ), - offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.0, 0.1045, 0.0, 0.9945), convention="world"), -) -"""Default :class:`~isaaclab.sensors.camera.TiledCameraCfg` for tiled state-based video recording. - -Places a pinhole camera at ``(-7, 0, 3)`` m relative to env_0's origin, angled ~12° downward. -Only spawned when ``--video=tiled`` is active and no observation TiledCamera exists in the scene. - -Override pose in ``__post_init__`` for tasks with different scene scales:: - - self.video_recorder.fallback_camera_cfg = self.video_recorder.fallback_camera_cfg.replace( - offset=TiledCameraCfg.OffsetCfg(pos=(-3.0, 0.0, 2.0), rot=(0.0, 0.1045, 0.0, 0.9945), convention="world"), - ) -""" - - @configclass class VideoRecorderCfg: """Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`.""" @@ -64,49 +30,21 @@ class VideoRecorderCfg: Set automatically by the environment base classes; do not set manually. """ - video_mode: str = "perspective" - """Recording mode: ``"perspective"`` (default) or ``"tiled"``. - - * ``"perspective"`` - single wide-angle view of the scene. Newton backends use the Newton GL - viewer; Kit backends use ``/OmniverseKit_Persp`` via ``omni.replicator.core``. TiledCamera - is bypassed even when present. - * ``"tiled"`` - square tile-grid from a :class:`~isaaclab.sensors.camera.TiledCamera`. - Reuses the observation camera on vision-based tasks; spawns ``fallback_camera_cfg`` for - state-based tasks. Raises ``RuntimeError`` if no TiledCamera is available. - - Set via CLI: ``--video=perspective`` / ``--video=tiled``. - """ - - video_num_tiles: int = -1 - """Max environments to include per frame (``-1`` = all). - - Tiles are arranged into a ``ceil(sqrt(N)) × ceil(sqrt(N))`` grid with black padding. - CLI example: ``env.video_recorder.video_num_tiles=9`` - """ - - fallback_camera_cfg: object = DEFAULT_TILED_RECORDING_CAMERA_CFG - """Side-view :class:`~isaaclab.sensors.camera.TiledCameraCfg` for tiled state-based recording. - - Spawned when ``video_mode="tiled"`` and no observation TiledCamera exists in the scene. - Set to ``None`` to disable. - """ - camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) """Newton GL perspective camera position in world space (metres). Matches :attr:`~isaaclab.envs.common.ViewerCfg.eye` so the Newton GL video aligns with - the Kit ``/OmniverseKit_Persp`` viewport. Only used by Newton backends in perspective mode. + the Kit ``/OmniverseKit_Persp`` viewport. Only used by Newton backends. """ camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) """Newton GL perspective camera look-at point in world space (metres). - Matches :attr:`~isaaclab.envs.common.ViewerCfg.lookat`. Only used by Newton backends in perspective mode. + Matches :attr:`~isaaclab.envs.common.ViewerCfg.lookat`. Only used by Newton backends. """ gl_viewer_width: int = 1280 - """Width in pixels of the Newton GL perspective frame. Only active when ``--video`` is set.""" + """Width in pixels of the Newton GL perspective frame.""" gl_viewer_height: int = 720 - """Height in pixels of the Newton GL perspective frame. Only active when ``--video`` is set.""" - + """Height in pixels of the Newton GL perspective frame.""" From 7e226090a3e03fecb3535cbef3296ee257d87a97 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Tue, 17 Mar 2026 22:30:14 +0000 Subject: [PATCH 3/6] Moved recording code-paths to respective backend folders --- .../envs/utils/test_video_recorder.py | 165 +++++++------ .../isaaclab/envs/utils/video_recorder.py | 219 ++++++------------ .../video_recording/__init__.py | 6 + .../newton_gl_perspective_video.py | 98 ++++++++ .../newton_gl_perspective_video_cfg.py | 40 ++++ .../video_recording/__init__.py | 6 + .../isaacsim_kit_perspective_video.py | 65 ++++++ .../isaacsim_kit_perspective_video_cfg.py | 40 ++++ 8 files changed, 410 insertions(+), 229 deletions(-) create mode 100644 source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py create mode 100644 source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py create mode 100644 source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py create mode 100644 source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py create mode 100644 source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py create mode 100644 source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py diff --git a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py index 5afd96dc2ae7..9957bc499ee9 100644 --- a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py @@ -3,114 +3,109 @@ # # SPDX-License-Identifier: BSD-3-Clause """Unit tests for VideoRecorder.""" -import importlib.util, pathlib, sys +import importlib.util +import pathlib +import sys from types import SimpleNamespace from unittest.mock import MagicMock, patch + import numpy as np import pytest _spec = importlib.util.spec_from_file_location("_vr", pathlib.Path(__file__).parent / "video_recorder.py") -_module = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(_module); VideoRecorder = _module.VideoRecorder +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) +VideoRecorder = _module.VideoRecorder +_video_recorder_module = _module _BLANK_720p = np.zeros((720, 1280, 3), dtype=np.uint8) _DEFAULT_CFG = dict( - render_mode="rgb_array", video_mode="perspective", fallback_camera_cfg=None, - video_num_tiles=-1, camera_eye=(7.5, 7.5, 7.5), camera_lookat=(0.0, 0.0, 0.0), - gl_viewer_width=1280, gl_viewer_height=720, + render_mode="rgb_array", + video_mode="perspective", + fallback_camera_cfg=None, + video_num_tiles=-1, + camera_eye=(7.5, 7.5, 7.5), + camera_lookat=(0.0, 0.0, 0.0), + gl_viewer_width=1280, + gl_viewer_height=720, ) def _create_recorder(**kw): """Return a VideoRecorder with __init__ bypassed and all deps mocked out.""" + backend = kw.pop("_backend", None) recorder = object.__new__(VideoRecorder) recorder.cfg = SimpleNamespace(**{**_DEFAULT_CFG, **kw}) - recorder._scene = MagicMock(); recorder._scene.sensors = {} - recorder._fallback_tiled_camera = None - recorder._gl_viewer = None - recorder._gl_viewer_init_attempted = False + recorder._scene = MagicMock() + recorder._scene.sensors = {} + recorder._scene._sensor_renderer_types = MagicMock(return_value=[]) + recorder._backend = backend + cap = MagicMock() + cap.render_rgb_array = MagicMock(return_value=_BLANK_720p) + recorder._capture = cap if backend else None return recorder -def test_init_perspective_mode_does_not_spawn_fallback(): - """In perspective mode, __init__ never spawns a TiledCamera fallback.""" - scene = MagicMock(); scene.sensors = {}; scene.num_envs = 1 +def test_init_perspective_mode_creates_kit_capture(): + """With kit backend, __init__ builds Isaac Sim Kit perspective capture.""" + scene = MagicMock() + scene.sensors = {} + scene.num_envs = 1 cfg = SimpleNamespace(**{**_DEFAULT_CFG, "fallback_camera_cfg": MagicMock()}) + fake_capture = MagicMock() + with patch.object(_video_recorder_module, "_resolve_video_backend", return_value="kit"): + with patch( + "isaaclab_physx.video_recording.isaacsim_kit_perspective_video.create_isaacsim_kit_perspective_video", + return_value=fake_capture, + ) as mock_create: + vr = VideoRecorder(cfg, scene) + mock_create.assert_called_once() + assert vr._capture is fake_capture + + +def test_init_newton_backend_creates_newton_capture(): + """With newton_gl backend, __init__ builds Newton GL perspective capture.""" + scene = MagicMock() + cfg = SimpleNamespace(**_DEFAULT_CFG) + fake_capture = MagicMock() with patch.dict(sys.modules, {"pyglet": MagicMock()}): - with patch.object(VideoRecorder, "_spawn_fallback_cameras") as mock_spawn: - VideoRecorder(cfg, scene) - mock_spawn.assert_not_called() + with patch.object(_video_recorder_module, "_resolve_video_backend", return_value="newton_gl"): + with patch( + "isaaclab_newton.video_recording.newton_gl_perspective_video.create_newton_gl_perspective_video", + return_value=fake_capture, + ) as mock_create: + vr = VideoRecorder(cfg, scene) + mock_create.assert_called_once() + assert vr._capture is fake_capture + + +def test_render_rgb_array_delegates_to_capture(): + """render_rgb_array returns capture.render_rgb_array().""" + recorder = _create_recorder(_backend="kit") + result = recorder.render_rgb_array() + recorder._capture.render_rgb_array.assert_called_once() + assert result.shape == (720, 1280, 3) -def test_init_tiled_mode_spawns_fallback_when_configured(): - """In tiled mode with a fallback_camera_cfg, __init__ calls _spawn_fallback_cameras.""" - scene = MagicMock(); scene.sensors = {}; scene.num_envs = 1 - cfg = SimpleNamespace(**{**_DEFAULT_CFG, "video_mode": "tiled", "fallback_camera_cfg": MagicMock()}) - with patch.dict(sys.modules, {"pyglet": MagicMock()}): - with patch.object(VideoRecorder, "_spawn_fallback_cameras", return_value=MagicMock()) as mock_spawn: - VideoRecorder(cfg, scene) - mock_spawn.assert_called_once() - - -def test_render_rgb_array_perspective_uses_gl_viewer_when_available(): - """Perspective mode returns a GL viewer frame when _gl_viewer is set.""" - recorder = _create_recorder() - recorder._gl_viewer = MagicMock(); recorder._gl_viewer_init_attempted = True - with patch.object(recorder, "_render_newton_gl_rgb_array", return_value=_BLANK_720p) as mock_gl: - result = recorder.render_rgb_array() - mock_gl.assert_called_once() - assert result.shape == (720, 1280, 3) +def test_render_rgb_array_none_when_no_backend(): + """Without rgb_array render_mode, _capture is None and render returns None.""" + recorder = _create_recorder(render_mode=None) + recorder._backend = None + recorder._capture = None + assert recorder.render_rgb_array() is None + + +def test_capture_exception_propagates(): + """Failures in backend capture propagate.""" + recorder = _create_recorder(_backend="newton_gl") + recorder._capture.render_rgb_array.side_effect = RuntimeError("fail") + with pytest.raises(RuntimeError, match="fail"): + recorder.render_rgb_array() -def test_render_rgb_array_perspective_falls_through_to_kit_when_no_gl_viewer(): - """Kit capture path is used when no GL viewer is available (Kit backend).""" - recorder = _create_recorder(); recorder._gl_viewer_init_attempted = True - with patch.object(recorder, "_render_kit_perspective_rgb_array", return_value=_BLANK_720p) as mock_kit: +def test_render_rgb_array_calls_capture_each_step(): + """Each render_rgb_array call hits the backend capture.""" + recorder = _create_recorder(_backend="kit") + for _ in range(3): recorder.render_rgb_array() - mock_kit.assert_called_once() - - -def test_render_rgb_array_tiled_raises_when_no_camera(): - """Tiled mode with no TiledCamera raises RuntimeError with a descriptive message.""" - recorder = _create_recorder(video_mode="tiled") - with patch.object(recorder, "_find_video_camera", return_value=None): - with pytest.raises(RuntimeError, match="tiled mode"): - recorder.render_rgb_array() - - -def test_gl_exception_returns_blank_ndarray_not_none(): - """GL renderer crash must return a blank ndarray, never None, so RecordVideo never sees None.""" - recorder = _create_recorder(); recorder._gl_viewer = MagicMock(); recorder._gl_viewer_init_attempted = True - with patch.dict(sys.modules, {"isaaclab.sim": MagicMock(SimulationContext=MagicMock(instance=MagicMock(side_effect=RuntimeError)))}): - frame = recorder._render_newton_gl_rgb_array() - assert isinstance(frame, np.ndarray) and frame.shape == (720, 1280, 3) - - -def test_find_video_camera_does_not_cache_none(): - """A None result is not cached, allowing retry on the next call.""" - recorder = _create_recorder(video_mode="tiled") - FakeTiledCamera = type("TiledCamera", (), {}) - with patch.dict(sys.modules, {"isaaclab": MagicMock(), "isaaclab.sensors": MagicMock(), "isaaclab.sensors.camera": MagicMock(TiledCamera=FakeTiledCamera)}): - result = recorder._find_video_camera() - assert result is None and not hasattr(recorder, "_video_camera") - - -def test_find_video_camera_caches_result_when_found(): - """A found camera is cached so the scene is not re-scanned on subsequent calls.""" - recorder = _create_recorder(video_mode="tiled") - FakeTiledCamera = type("TiledCamera", (), {}) - camera = MagicMock(); camera.__class__ = FakeTiledCamera - camera.is_initialized = True; camera.data.output = {"rgb": MagicMock(shape=(4, 64, 64, 3))} - recorder._scene.sensors = {"cam": camera} - with patch.dict(sys.modules, {"isaaclab": MagicMock(), "isaaclab.sensors": MagicMock(), "isaaclab.sensors.camera": MagicMock(TiledCamera=FakeTiledCamera)}): - result = recorder._find_video_camera() - assert result is camera and hasattr(recorder, "_video_camera") - - -def test_gl_viewer_init_attempted_only_once(): - """_try_init_gl_viewer is called at most once regardless of render call count.""" - recorder = _create_recorder(); recorder._gl_viewer_init_attempted = False - def _set_flag(): recorder._gl_viewer_init_attempted = True - with patch.object(recorder, "_try_init_gl_viewer", side_effect=_set_flag) as mock_init, \ - patch.object(recorder, "_render_kit_perspective_rgb_array", return_value=_BLANK_720p): - for _ in range(3): recorder.render_rgb_array() - mock_init.assert_called_once() \ No newline at end of file + assert recorder._capture.render_rgb_array.call_count == 3 diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py index 8c211b69647a..122deb713f8d 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -7,8 +7,13 @@ Captures a single wide-angle perspective view of the scene: -* **Newton backends** — uses the Newton GL viewer (``newton.viewer.ViewerGL``). -* **Kit backends** — captures the ``/OmniverseKit_Persp`` viewport via ``omni.replicator.core``. +* **Kit backends** (PhysX physics or Isaac RTX renderer) — uses + :mod:`isaaclab_physx.video_recording.isaacsim_kit_perspective_video`. +* **Newton backends** (Newton physics or Newton Warp renderer only) — uses + :mod:`isaaclab_newton.video_recording.newton_gl_perspective_video`. + +If neither a Kit nor a Newton backend is detected, construction raises so users do not +use ``--video`` on unsupported setups. See :mod:`video_recorder_cfg` for configuration. """ @@ -16,9 +21,7 @@ from __future__ import annotations import logging -import math -import traceback -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import numpy as np @@ -28,6 +31,32 @@ logger = logging.getLogger(__name__) +_VideoBackend = Literal["kit", "newton_gl"] + + +def _resolve_video_backend(scene: InteractiveScene) -> _VideoBackend: + """Resolve which video backend to use from physics and renderer configs. + + Priority: PhysX or Isaac RTX -> Kit camera; else Newton or Newton Warp -> GL viewer. + When both are present (e.g. PhysX + Newton Warp), Kit wins. + """ + sim = scene.sim + physics_name = sim.physics_manager.__name__.lower() + renderer_types: list[str] = scene._sensor_renderer_types() + + use_kit = "physx" in physics_name or "isaac_rtx" in renderer_types + use_newton_gl = "newton" in physics_name or "newton_warp" in renderer_types + + if use_kit: + return "kit" + if use_newton_gl: + return "newton_gl" + raise RuntimeError( + "Video recording (--video) requires a supported backend: " + "PhysX or Isaac RTX renderer (Kit camera), or Newton physics / Newton Warp renderer (GL viewer). " + "No supported backend detected; do not use --video for this setup." + ) + class VideoRecorder: """Records perspective video frames from the scene's active renderer. @@ -40,147 +69,49 @@ class VideoRecorder: def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): self.cfg = cfg self._scene = scene - self._gl_viewer = None - self._gl_viewer_init_attempted = False + self._backend: _VideoBackend | None = None + self._capture = None if cfg.render_mode == "rgb_array": - # enable EGL headless rendering for pyglet before any pyglet.window import. - try: - import pyglet + self._backend = _resolve_video_backend(scene) + if self._backend == "newton_gl": + try: + import pyglet + + if not pyglet.options.get("headless", False): + pyglet.options["headless"] = True + except ImportError: + pass + from isaaclab_newton.video_recording.newton_gl_perspective_video import ( + create_newton_gl_perspective_video, + ) + from isaaclab_newton.video_recording.newton_gl_perspective_video_cfg import NewtonGlPerspectiveVideoCfg + + ncfg = NewtonGlPerspectiveVideoCfg( + gl_viewer_width=cfg.gl_viewer_width, + gl_viewer_height=cfg.gl_viewer_height, + camera_eye=cfg.camera_eye, + camera_lookat=cfg.camera_lookat, + ) + self._capture = create_newton_gl_perspective_video(ncfg) + else: + from isaaclab_physx.video_recording.isaacsim_kit_perspective_video import ( + create_isaacsim_kit_perspective_video, + ) + from isaaclab_physx.video_recording.isaacsim_kit_perspective_video_cfg import ( + IsaacsimKitPerspectiveVideoCfg, + ) - if not pyglet.options.get("headless", False): - pyglet.options["headless"] = True - except ImportError: - pass + kcfg = IsaacsimKitPerspectiveVideoCfg( + camera_eye=cfg.camera_eye, + camera_lookat=cfg.camera_lookat, + render_width=cfg.gl_viewer_width, + render_height=cfg.gl_viewer_height, + ) + self._capture = create_isaacsim_kit_perspective_video(kcfg) def render_rgb_array(self) -> np.ndarray | None: - """Return an RGB frame, or ``None`` when neither GL viewer nor Kit runtime is available.""" - if not self._gl_viewer_init_attempted: - self._try_init_gl_viewer() - if self._gl_viewer is not None: - return self._render_newton_gl_rgb_array() - return self._render_kit_perspective_rgb_array() - - def _try_init_gl_viewer(self) -> None: - """Lazy-initialise the Newton GL viewer on the first render call. - - Called after ``sim.reset()`` so the Newton model is fully built. - Leaves ``_gl_viewer`` as ``None`` on Kit backends; ``render_rgb_array`` then - calls ``_render_kit_perspective_rgb_array`` instead. - """ - self._gl_viewer_init_attempted = True - try: - from isaaclab.sim import SimulationContext - - sdp = SimulationContext.instance().initialize_scene_data_provider() - model = sdp.get_newton_model() - if model is None: - return - - import pyglet - - pyglet.options["headless"] = True - from newton.viewer import ViewerGL - - viewer = ViewerGL(width=self.cfg.gl_viewer_width, height=self.cfg.gl_viewer_height, headless=True) - viewer.set_model(model) - viewer.set_world_offsets((0.0, 0.0, 0.0)) # world positions already in body_q - viewer.up_axis = 2 # Z-up - self._gl_viewer = viewer - - # place camera to match Kit /OmniverseKit_Persp (same eye/lookat as ViewerCfg). - try: - import warp as wp - - ex, ey, ez = self.cfg.camera_eye - lx, ly, lz = self.cfg.camera_lookat - dx, dy, dz = lx - ex, ly - ey, lz - ez - length = math.sqrt(dx**2 + dy**2 + dz**2) - dx, dy, dz = dx / length, dy / length, dz / length - pitch = math.degrees(math.asin(max(-1.0, min(1.0, dz)))) - yaw = math.degrees(math.atan2(dy, dx)) - - # Kit uses horizontal FOV (60°); pyglet/Newton GL uses vertical FOV. - aspect = self.cfg.gl_viewer_width / self.cfg.gl_viewer_height - v_fov_deg = math.degrees(2.0 * math.atan(math.tan(math.radians(60.0) / 2.0) / aspect)) - viewer.camera.fov = v_fov_deg # ≈ 36° for 1280×720 - viewer.set_camera(pos=wp.vec3(ex, ey, ez), pitch=pitch, yaw=yaw) - except Exception as exc: - logger.warning("[VideoRecorder] GL viewer camera setup failed: %s", exc) - - logger.info( - "[VideoRecorder] Newton GL viewer ready (%dx%d).", - self.cfg.gl_viewer_width, - self.cfg.gl_viewer_height, - ) - except Exception as exc: - logger.warning("[VideoRecorder] Newton GL viewer unavailable: %s", exc) - - def _render_newton_gl_rgb_array(self) -> np.ndarray: - """Return one RGB frame from the Newton GL viewer, or a blank frame on error.""" - try: - from isaaclab.sim import SimulationContext - - sim = SimulationContext.instance() - sdp = sim.initialize_scene_data_provider() - state = sdp.get_newton_state() - dt = sim.get_physics_dt() - - viewer = self._gl_viewer - viewer.begin_frame(dt) - viewer.log_state(state) - viewer.end_frame() - return viewer.get_frame().numpy() - except Exception as exc: - logger.warning("[VideoRecorder] GL frame capture failed: %s", exc) - return np.zeros((self.cfg.gl_viewer_height, self.cfg.gl_viewer_width, 3), dtype=np.uint8) - - def _render_kit_perspective_rgb_array(self) -> np.ndarray: - """Return one RGB frame from the Kit /OmniverseKit_Persp camera via omni.replicator. - - On the first call the viewport camera is positioned to match ``cfg.camera_eye`` / - ``cfg.camera_lookat`` (the same values used by the Newton GL viewer), so both - backends produce a consistent framing. - - Returns a blank frame during warmup or on any error. - """ - try: - import omni.kit.app - import omni.replicator.core as rep - - # Drive the Kit app loop to produce a fresh RTX viewport frame. - omni.kit.app.get_app().update() - - if not hasattr(self, "_rgb_annotator"): - try: - import isaacsim.core.utils.viewports as isaacsim_viewports - - # set the camera view to the Kit /OmniverseKit_Persp camera. - # commit da2983e switched active viewport views - isaacsim_viewports.set_camera_view( - eye=list(self.cfg.camera_eye), - target=list(self.cfg.camera_lookat), - camera_prim_path="/OmniverseKit_Persp", - ) - except Exception as exc: - logger.warning("[VideoRecorder] Kit perspective camera positioning failed: %s", exc) - - self._render_product = rep.create.render_product( - "/OmniverseKit_Persp", (1280, 720) - ) - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - - rgb_data = self._rgb_annotator.get_data() - if isinstance(rgb_data, dict): - rgb_data = rgb_data.get("data", np.array([], dtype=np.uint8)) - rgb_data = np.asarray(rgb_data, dtype=np.uint8) - if rgb_data.size == 0: - # renderer is warming up; return blank frame - return np.zeros((720, 1280, 3), dtype=np.uint8) - if rgb_data.ndim == 1: - rgb_data = rgb_data.reshape(720, 1280, -1) - return rgb_data[:, :, :3] - except Exception as exc: - logger.warning("[VideoRecorder] Kit perspective capture failed: %s\n%s", exc, traceback.format_exc()) - return np.zeros((720, 1280, 3), dtype=np.uint8) + """Return an RGB frame for the resolved backend. Fails if backend is unavailable.""" + if self._backend is None or self._capture is None: + return None + return self._capture.render_rgb_array() diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py b/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py new file mode 100644 index 000000000000..3248ca5f13b4 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton GL perspective video recording.""" diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py new file mode 100644 index 000000000000..a55dc2df0ece --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py @@ -0,0 +1,98 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton GL perspective RGB capture via headless ``newton.viewer.ViewerGL``.""" + +from __future__ import annotations + +import logging +import math +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from .newton_gl_perspective_video_cfg import NewtonGlPerspectiveVideoCfg + +logger = logging.getLogger(__name__) + + +class NewtonGlPerspectiveVideo: + """Lazy-initialised ViewerGL; one RGB frame per :meth:`render_rgb_array` call.""" + + def __init__(self, cfg: NewtonGlPerspectiveVideoCfg): + self.cfg = cfg + self._viewer = None + self._init_attempted = False + + def _ensure_viewer(self) -> None: + if self._init_attempted: + return + self._init_attempted = True + from isaaclab.sim import SimulationContext + + sdp = SimulationContext.instance().initialize_scene_data_provider() + model = sdp.get_newton_model() + if model is None: + raise RuntimeError( + "Newton GL perspective video requires a Newton model on the scene data provider. " + "Do not use --video for this setup." + ) + + import pyglet + + pyglet.options["headless"] = True + from newton.viewer import ViewerGL + + w, h = self.cfg.gl_viewer_width, self.cfg.gl_viewer_height + viewer = ViewerGL(width=w, height=h, headless=True) + viewer.set_model(model) + viewer.set_world_offsets((0.0, 0.0, 0.0)) + viewer.up_axis = 2 + + import warp as wp + + ex, ey, ez = self.cfg.camera_eye + lx, ly, lz = self.cfg.camera_lookat + dx, dy, dz = lx - ex, ly - ey, lz - ez + length = math.sqrt(dx**2 + dy**2 + dz**2) + dx, dy, dz = dx / length, dy / length, dz / length + pitch = math.degrees(math.asin(max(-1.0, min(1.0, dz)))) + yaw = math.degrees(math.atan2(dy, dx)) + aspect = w / h + h_fov = math.radians(self.cfg.kit_horizontal_fov_deg) + v_fov_deg = math.degrees(2.0 * math.atan(math.tan(h_fov / 2.0) / aspect)) + viewer.camera.fov = v_fov_deg + viewer.set_camera(pos=wp.vec3(ex, ey, ez), pitch=pitch, yaw=yaw) + + self._viewer = viewer + logger.info("[NewtonGlPerspectiveVideo] ViewerGL ready (%dx%d).", w, h) + + def render_rgb_array(self) -> np.ndarray: + """Return one RGB frame from the Newton GL viewer. Raises on failure.""" + self._ensure_viewer() + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + sdp = sim.initialize_scene_data_provider() + state = sdp.get_newton_state() + dt = sim.get_physics_dt() + + viewer = self._viewer + viewer.begin_frame(dt) + viewer.log_state(state) + viewer.end_frame() + return viewer.get_frame().numpy() + + +def create_newton_gl_perspective_video(cfg: NewtonGlPerspectiveVideoCfg) -> NewtonGlPerspectiveVideo: + """Instantiate the Newton GL perspective capture from ``cfg.class_type``.""" + ct = cfg.class_type + if isinstance(ct, type): + return ct(cfg) + from isaaclab.utils.string import string_to_callable + + cls = string_to_callable(str(ct)) + return cls(cfg) diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py new file mode 100644 index 000000000000..2d9f7b77ad9c --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Newton GL perspective RGB capture (headless ViewerGL).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from .newton_gl_perspective_video import NewtonGlPerspectiveVideo + + +@configclass +class NewtonGlPerspectiveVideoCfg: + """Settings for capturing a perspective RGB frame via ``newton.viewer.ViewerGL``.""" + + class_type: type[Any] | str = ( + "isaaclab_newton.video_recording.newton_gl_perspective_video:NewtonGlPerspectiveVideo" + ) + """Implementation class; default is :class:`~isaaclab_newton.video_recording.newton_gl_perspective_video.NewtonGlPerspectiveVideo`.""" + + gl_viewer_width: int = 1280 + """Viewer width in pixels.""" + + gl_viewer_height: int = 720 + """Viewer height in pixels.""" + + camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Camera position in world space (metres).""" + + camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at point in world space (metres).""" + + kit_horizontal_fov_deg: float = 60.0 + """Horizontal FOV assumed for Kit ``/OmniverseKit_Persp``; converted to vertical FOV for GL viewer.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py b/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py new file mode 100644 index 000000000000..8deb00ddc466 --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Sim Kit perspective video recording.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py new file mode 100644 index 000000000000..513fa6e359fc --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Sim Kit perspective RGB capture via ``/OmniverseKit_Persp`` and omni.replicator.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from .isaacsim_kit_perspective_video_cfg import IsaacsimKitPerspectiveVideoCfg + + +class IsaacsimKitPerspectiveVideo: + """Stateful capture of one RGB frame per call from the Kit perspective camera.""" + + def __init__(self, cfg: IsaacsimKitPerspectiveVideoCfg): + self.cfg = cfg + self._rgb_annotator = None + self._render_product = None + + def render_rgb_array(self) -> np.ndarray: + """Return one RGB frame. Blank frame during warmup; raises on other failures.""" + import omni.kit.app + import omni.replicator.core as rep + + omni.kit.app.get_app().update() + + h, w = self.cfg.render_height, self.cfg.render_width + if self._rgb_annotator is None: + import isaacsim.core.utils.viewports as isaacsim_viewports + + isaacsim_viewports.set_camera_view( + eye=list(self.cfg.camera_eye), + target=list(self.cfg.camera_lookat), + camera_prim_path=self.cfg.camera_prim_path, + ) + self._render_product = rep.create.render_product(self.cfg.camera_prim_path, (w, h)) + self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") + self._rgb_annotator.attach([self._render_product]) + + rgb_data = self._rgb_annotator.get_data() + if isinstance(rgb_data, dict): + rgb_data = rgb_data.get("data", np.array([], dtype=np.uint8)) + rgb_data = np.asarray(rgb_data, dtype=np.uint8) + if rgb_data.size == 0: + return np.zeros((h, w, 3), dtype=np.uint8) + if rgb_data.ndim == 1: + rgb_data = rgb_data.reshape(h, w, -1) + return rgb_data[:, :, :3] + + +def create_isaacsim_kit_perspective_video(cfg: IsaacsimKitPerspectiveVideoCfg) -> IsaacsimKitPerspectiveVideo: + """Instantiate the perspective video capture implementation from ``cfg.class_type``.""" + ct = cfg.class_type + if isinstance(ct, type): + return ct(cfg) + from isaaclab.utils.string import string_to_callable + + cls = string_to_callable(str(ct)) + return cls(cfg) diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py new file mode 100644 index 000000000000..7963e59acc22 --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Isaac Sim Kit perspective RGB capture (OmniverseKit_Persp + Replicator).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from .isaacsim_kit_perspective_video import IsaacsimKitPerspectiveVideo + + +@configclass +class IsaacsimKitPerspectiveVideoCfg: + """Settings for capturing a perspective RGB frame from the Kit viewport camera.""" + + class_type: type[Any] | str = ( + "isaaclab_physx.video_recording.isaacsim_kit_perspective_video:IsaacsimKitPerspectiveVideo" + ) + """Implementation class; default is :class:`~isaaclab_physx.video_recording.isaacsim_kit_perspective_video.IsaacsimKitPerspectiveVideo`.""" + + camera_prim_path: str = "/OmniverseKit_Persp" + """Viewport camera prim used for the render product.""" + + camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Camera position in world space (metres).""" + + camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at point in world space (metres).""" + + render_width: int = 1280 + """Output width in pixels.""" + + render_height: int = 720 + """Output height in pixels.""" From 3fb71863c812f8251295d00122ee755cab62e8ea Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Wed, 18 Mar 2026 19:21:23 +0000 Subject: [PATCH 4/6] Sync perspective video camera from ViewerCfg --- source/isaaclab/isaaclab/envs/direct_marl_env.py | 3 +++ source/isaaclab/isaaclab/envs/direct_rl_env.py | 4 ++++ .../isaaclab/isaaclab/envs/manager_based_rl_env.py | 7 +++++++ .../isaaclab/envs/utils/video_recorder_cfg.py | 12 +++++------- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index 33541c3cd44c..4447df25c4a2 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -176,6 +176,9 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. if self.cfg.video_recorder is not None: self.cfg.video_recorder.render_mode = render_mode + vr = self.cfg.video_recorder + vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( self.cfg.video_recorder, self.scene ) diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index 58456e72fb2d..96eb50b8d8c7 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -181,6 +181,10 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. if self.cfg.video_recorder is not None: self.cfg.video_recorder.render_mode = render_mode + # Perspective --video uses same eye/lookat as task viewer (Kit persp + Newton GL). + vr = self.cfg.video_recorder + vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( self.cfg.video_recorder, self.scene ) diff --git a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py index 132fa4d97fbb..129bddc6525d 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py @@ -87,6 +87,13 @@ def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, ** # store the render mode self.render_mode = render_mode + if cfg.video_recorder is not None: + cfg.video_recorder.camera_eye = tuple(float(x) for x in cfg.viewer.eye) + cfg.video_recorder.camera_lookat = tuple(float(x) for x in cfg.viewer.lookat) + self.video_recorder = VideoRecorder(cfg.video_recorder, self.scene) + else: + self.video_recorder = None + # initialize data and constants # -- set the framerate of the gym video recorder wrapper so that the playback speed of the # produced video matches the simulation diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py index a3dde49c1a33..760e5ede6d20 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py @@ -31,17 +31,15 @@ class VideoRecorderCfg: """ camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) - """Newton GL perspective camera position in world space (metres). + """Perspective camera position in world space (metres). - Matches :attr:`~isaaclab.envs.common.ViewerCfg.eye` so the Newton GL video aligns with - the Kit ``/OmniverseKit_Persp`` viewport. Only used by Newton backends. + Direct RL / MARL and manager-based RL environments overwrite this from + :attr:`~isaaclab.envs.common.ViewerCfg.eye` before recording so ``--video`` matches the + task viewport for both Kit (PhysX / Isaac RTX) and Newton GL (Newton / OVRTX / etc.). """ camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Newton GL perspective camera look-at point in world space (metres). - - Matches :attr:`~isaaclab.envs.common.ViewerCfg.lookat`. Only used by Newton backends. - """ + """Perspective camera look-at in world space (metres). Set from ``ViewerCfg.lookat`` at env init.""" gl_viewer_width: int = 1280 """Width in pixels of the Newton GL perspective frame.""" From 8170dca59d3d428077f6c9781fd21ebec6b93d4a Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Thu, 19 Mar 2026 01:27:46 +0000 Subject: [PATCH 5/6] Comment fixes: test file moved, self.video_recorder instantiation, ImportError; precommit --- source/isaaclab/isaaclab/envs/direct_marl_env.py | 7 ++----- source/isaaclab/isaaclab/envs/direct_rl_env.py | 7 ++----- .../isaaclab/isaaclab/envs/manager_based_env.py | 11 +++++++++++ .../isaaclab/envs/manager_based_rl_env.py | 15 +++++---------- .../isaaclab/envs/utils/video_recorder.py | 7 +++++-- .../utils => test/envs}/test_video_recorder.py | 1 + .../newton_gl_perspective_video_cfg.py | 9 ++++----- .../isaacsim_kit_perspective_video_cfg.py | 5 +++-- 8 files changed, 33 insertions(+), 29 deletions(-) rename source/isaaclab/{isaaclab/envs/utils => test/envs}/test_video_recorder.py (99%) diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index 4447df25c4a2..ab01586b1e68 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -35,9 +35,8 @@ from .common import ActionType, AgentID, EnvStepReturn, ObsType, StateType from .direct_marl_env_cfg import DirectMARLEnvCfg from .ui import ViewportCameraController -from .utils.video_recorder import VideoRecorder -from .utils.video_recorder_cfg import VideoRecorderCfg from .utils.spaces import sample_space, spec_to_gym_space +from .utils.video_recorder import VideoRecorder # import logger logger = logging.getLogger(__name__) @@ -179,9 +178,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): vr = self.cfg.video_recorder vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) - self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( - self.cfg.video_recorder, self.scene - ) + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) else: self.video_recorder = None diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index 96eb50b8d8c7..f2261b8c63dc 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -32,9 +32,8 @@ from .common import VecEnvObs, VecEnvStepReturn from .direct_rl_env_cfg import DirectRLEnvCfg from .ui import ViewportCameraController -from .utils.video_recorder import VideoRecorder -from .utils.video_recorder_cfg import VideoRecorderCfg from .utils.spaces import sample_space, spec_to_gym_space +from .utils.video_recorder import VideoRecorder if has_kit(): import omni.kit.app @@ -185,9 +184,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): vr = self.cfg.video_recorder vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) - self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type( - self.cfg.video_recorder, self.scene - ) + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) else: self.video_recorder = None diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index 996e88216e17..929c71f2389a 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -26,6 +26,7 @@ from .manager_based_env_cfg import ManagerBasedEnvCfg from .ui import ViewportCameraController from .utils.io_descriptors import export_articulations_data, export_scene_data +from .utils.video_recorder import VideoRecorder # import logger logger = logging.getLogger(__name__) @@ -182,6 +183,16 @@ def _init_sim(self): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # render_mode and camera eye/lookat are forwarded by subclasses (e.g. ManagerBasedRLEnv) + # into cfg.video_recorder before calling super().__init__(). + if self.cfg.video_recorder is not None: + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers diff --git a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py index 129bddc6525d..6d27ea7a6fb2 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py @@ -20,7 +20,6 @@ from .common import VecEnvStepReturn from .manager_based_env import ManagerBasedEnv from .manager_based_rl_env_cfg import ManagerBasedRLEnvCfg -from .utils.video_recorder import VideoRecorder class ManagerBasedRLEnv(ManagerBasedEnv, gym.Env): @@ -77,23 +76,19 @@ def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, ** # initialize the episode length buffer BEFORE loading the managers to use it in mdp functions. self.episode_length_buf = torch.zeros(cfg.scene.num_envs, device=cfg.sim.device, dtype=torch.long) - # Forward render_mode to VideoRecorderCfg before super().__init__() creates VideoRecorder, - # so fallback cameras are only spawned when --video is active (render_mode="rgb_array"). + # Forward render_mode and viewer camera to VideoRecorderCfg before super().__init__() + # creates the VideoRecorder, so fallback cameras are only spawned when --video is active + # (render_mode="rgb_array") and the perspective view matches the task viewport. if cfg.video_recorder is not None: cfg.video_recorder.render_mode = render_mode + cfg.video_recorder.camera_eye = tuple(float(x) for x in cfg.viewer.eye) + cfg.video_recorder.camera_lookat = tuple(float(x) for x in cfg.viewer.lookat) # initialize the base class to setup the scene. super().__init__(cfg=cfg) # store the render mode self.render_mode = render_mode - if cfg.video_recorder is not None: - cfg.video_recorder.camera_eye = tuple(float(x) for x in cfg.viewer.eye) - cfg.video_recorder.camera_lookat = tuple(float(x) for x in cfg.viewer.lookat) - self.video_recorder = VideoRecorder(cfg.video_recorder, self.scene) - else: - self.video_recorder = None - # initialize data and constants # -- set the framerate of the gym video recorder wrapper so that the playback speed of the # produced video matches the simulation diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py index 122deb713f8d..3bd1b9ca909b 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from isaaclab.scene import InteractiveScene + from .video_recorder_cfg import VideoRecorderCfg logger = logging.getLogger(__name__) @@ -80,8 +81,10 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): if not pyglet.options.get("headless", False): pyglet.options["headless"] = True - except ImportError: - pass + except ImportError as e: + raise ImportError( + "The Newton GL video backend requires 'pyglet'. Install IsaacLab with './isaaclab.sh -i'." + ) from e from isaaclab_newton.video_recording.newton_gl_perspective_video import ( create_newton_gl_perspective_video, ) diff --git a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py b/source/isaaclab/test/envs/test_video_recorder.py similarity index 99% rename from source/isaaclab/isaaclab/envs/utils/test_video_recorder.py rename to source/isaaclab/test/envs/test_video_recorder.py index 9957bc499ee9..036f569d33c1 100644 --- a/source/isaaclab/isaaclab/envs/utils/test_video_recorder.py +++ b/source/isaaclab/test/envs/test_video_recorder.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause """Unit tests for VideoRecorder.""" + import importlib.util import pathlib import sys diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py index 2d9f7b77ad9c..865efe1a26c7 100644 --- a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py @@ -12,17 +12,16 @@ from isaaclab.utils import configclass if TYPE_CHECKING: - from .newton_gl_perspective_video import NewtonGlPerspectiveVideo + pass @configclass class NewtonGlPerspectiveVideoCfg: """Settings for capturing a perspective RGB frame via ``newton.viewer.ViewerGL``.""" - class_type: type[Any] | str = ( - "isaaclab_newton.video_recording.newton_gl_perspective_video:NewtonGlPerspectiveVideo" - ) - """Implementation class; default is :class:`~isaaclab_newton.video_recording.newton_gl_perspective_video.NewtonGlPerspectiveVideo`.""" + class_type: type[Any] | str = "isaaclab_newton.video_recording.newton_gl_perspective_video:NewtonGlPerspectiveVideo" + """Implementation class; default is + :class:`~isaaclab_newton.video_recording.newton_gl_perspective_video.NewtonGlPerspectiveVideo`.""" gl_viewer_width: int = 1280 """Viewer width in pixels.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py index 7963e59acc22..d477604e14a8 100644 --- a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py @@ -12,7 +12,7 @@ from isaaclab.utils import configclass if TYPE_CHECKING: - from .isaacsim_kit_perspective_video import IsaacsimKitPerspectiveVideo + pass @configclass @@ -22,7 +22,8 @@ class IsaacsimKitPerspectiveVideoCfg: class_type: type[Any] | str = ( "isaaclab_physx.video_recording.isaacsim_kit_perspective_video:IsaacsimKitPerspectiveVideo" ) - """Implementation class; default is :class:`~isaaclab_physx.video_recording.isaacsim_kit_perspective_video.IsaacsimKitPerspectiveVideo`.""" + """Implementation class; default is + :class:`~isaaclab_physx.video_recording.isaacsim_kit_perspective_video.IsaacsimKitPerspectiveVideo`.""" camera_prim_path: str = "/OmniverseKit_Persp" """Viewport camera prim used for the render product.""" From 58eb1da4bd11dd60665e56887b6d031ac07659c9 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Thu, 19 Mar 2026 04:11:10 +0000 Subject: [PATCH 6/6] comment fix: VideoRecorder and config classes' field renames --- .../isaaclab/isaaclab/envs/direct_marl_env.py | 6 +++--- source/isaaclab/isaaclab/envs/direct_rl_env.py | 6 +++--- .../isaaclab/envs/manager_based_env.py | 2 +- .../isaaclab/envs/manager_based_rl_env.py | 8 ++++---- .../isaaclab/envs/utils/video_recorder.py | 18 +++++++++--------- .../isaaclab/envs/utils/video_recorder_cfg.py | 18 +++++++++--------- .../newton_gl_perspective_video.py | 8 ++++---- .../newton_gl_perspective_video_cfg.py | 12 ++++++------ .../isaacsim_kit_perspective_video.py | 6 +++--- .../isaacsim_kit_perspective_video_cfg.py | 10 +++++----- 10 files changed, 47 insertions(+), 47 deletions(-) diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index ab01586b1e68..c6009117f1b5 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -174,10 +174,10 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): # stage and registered for the PHYSICS_READY callback before physics initialises. # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. if self.cfg.video_recorder is not None: - self.cfg.video_recorder.render_mode = render_mode + self.cfg.video_recorder.env_render_mode = render_mode vr = self.cfg.video_recorder - vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) - vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) + vr.camera_position = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_target = tuple(float(x) for x in self.cfg.viewer.lookat) self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) else: self.video_recorder = None diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index f2261b8c63dc..05dc8495dbcf 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -179,11 +179,11 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): # stage and registered for the PHYSICS_READY callback before physics initialises. # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. if self.cfg.video_recorder is not None: - self.cfg.video_recorder.render_mode = render_mode + self.cfg.video_recorder.env_render_mode = render_mode # Perspective --video uses same eye/lookat as task viewer (Kit persp + Newton GL). vr = self.cfg.video_recorder - vr.camera_eye = tuple(float(x) for x in self.cfg.viewer.eye) - vr.camera_lookat = tuple(float(x) for x in self.cfg.viewer.lookat) + vr.camera_position = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_target = tuple(float(x) for x in self.cfg.viewer.lookat) self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) else: self.video_recorder = None diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index 929c71f2389a..1e8ca0576101 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -186,7 +186,7 @@ def _init_sim(self): # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera # (used for state-based envs without an observation camera) is spawned into the USD # stage and registered for the PHYSICS_READY callback before physics initialises. - # render_mode and camera eye/lookat are forwarded by subclasses (e.g. ManagerBasedRLEnv) + # env_render_mode and camera_position/camera_target are forwarded by subclasses (e.g. ManagerBasedRLEnv) # into cfg.video_recorder before calling super().__init__(). if self.cfg.video_recorder is not None: self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) diff --git a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py index 6d27ea7a6fb2..0082912ec59c 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py @@ -78,11 +78,11 @@ def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, ** # Forward render_mode and viewer camera to VideoRecorderCfg before super().__init__() # creates the VideoRecorder, so fallback cameras are only spawned when --video is active - # (render_mode="rgb_array") and the perspective view matches the task viewport. + # (env_render_mode="rgb_array") and the perspective view matches the task viewport. if cfg.video_recorder is not None: - cfg.video_recorder.render_mode = render_mode - cfg.video_recorder.camera_eye = tuple(float(x) for x in cfg.viewer.eye) - cfg.video_recorder.camera_lookat = tuple(float(x) for x in cfg.viewer.lookat) + cfg.video_recorder.env_render_mode = render_mode + cfg.video_recorder.camera_position = tuple(float(x) for x in cfg.viewer.eye) + cfg.video_recorder.camera_target = tuple(float(x) for x in cfg.viewer.lookat) # initialize the base class to setup the scene. super().__init__(cfg=cfg) diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py index 3bd1b9ca909b..9ffebdbe9b55 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -73,7 +73,7 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): self._backend: _VideoBackend | None = None self._capture = None - if cfg.render_mode == "rgb_array": + if cfg.env_render_mode == "rgb_array": self._backend = _resolve_video_backend(scene) if self._backend == "newton_gl": try: @@ -91,10 +91,10 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): from isaaclab_newton.video_recording.newton_gl_perspective_video_cfg import NewtonGlPerspectiveVideoCfg ncfg = NewtonGlPerspectiveVideoCfg( - gl_viewer_width=cfg.gl_viewer_width, - gl_viewer_height=cfg.gl_viewer_height, - camera_eye=cfg.camera_eye, - camera_lookat=cfg.camera_lookat, + window_width=cfg.window_width, + window_height=cfg.window_height, + camera_position=cfg.camera_position, + camera_target=cfg.camera_target, ) self._capture = create_newton_gl_perspective_video(ncfg) else: @@ -106,10 +106,10 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): ) kcfg = IsaacsimKitPerspectiveVideoCfg( - camera_eye=cfg.camera_eye, - camera_lookat=cfg.camera_lookat, - render_width=cfg.gl_viewer_width, - render_height=cfg.gl_viewer_height, + camera_position=cfg.camera_position, + camera_target=cfg.camera_target, + window_width=cfg.window_width, + window_height=cfg.window_height, ) self._capture = create_isaacsim_kit_perspective_video(kcfg) diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py index 760e5ede6d20..586779e3e679 100644 --- a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py @@ -24,13 +24,13 @@ class VideoRecorderCfg: class_type: type = VideoRecorder """Recorder class to instantiate; must accept ``(cfg, scene)``.""" - render_mode: str | None = None - """Render mode forwarded from the environment constructor (``"rgb_array"`` when ``--video`` is active). + env_render_mode: str | None = None + """Gym render mode forwarded from the environment constructor (``"rgb_array"`` when ``--video`` is active). Set automatically by the environment base classes; do not set manually. """ - camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) """Perspective camera position in world space (metres). Direct RL / MARL and manager-based RL environments overwrite this from @@ -38,11 +38,11 @@ class VideoRecorderCfg: task viewport for both Kit (PhysX / Isaac RTX) and Newton GL (Newton / OVRTX / etc.). """ - camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Perspective camera look-at in world space (metres). Set from ``ViewerCfg.lookat`` at env init.""" + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Perspective camera look-at target in world space (metres). Set from ``ViewerCfg.lookat`` at env init.""" - gl_viewer_width: int = 1280 - """Width in pixels of the Newton GL perspective frame.""" + window_width: int = 1280 + """Width in pixels of the recorded frame.""" - gl_viewer_height: int = 720 - """Height in pixels of the Newton GL perspective frame.""" + window_height: int = 720 + """Height in pixels of the recorded frame.""" diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py index a55dc2df0ece..cbc2af01def7 100644 --- a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py @@ -46,7 +46,7 @@ def _ensure_viewer(self) -> None: pyglet.options["headless"] = True from newton.viewer import ViewerGL - w, h = self.cfg.gl_viewer_width, self.cfg.gl_viewer_height + w, h = self.cfg.window_width, self.cfg.window_height viewer = ViewerGL(width=w, height=h, headless=True) viewer.set_model(model) viewer.set_world_offsets((0.0, 0.0, 0.0)) @@ -54,15 +54,15 @@ def _ensure_viewer(self) -> None: import warp as wp - ex, ey, ez = self.cfg.camera_eye - lx, ly, lz = self.cfg.camera_lookat + ex, ey, ez = self.cfg.camera_position + lx, ly, lz = self.cfg.camera_target dx, dy, dz = lx - ex, ly - ey, lz - ez length = math.sqrt(dx**2 + dy**2 + dz**2) dx, dy, dz = dx / length, dy / length, dz / length pitch = math.degrees(math.asin(max(-1.0, min(1.0, dz)))) yaw = math.degrees(math.atan2(dy, dx)) aspect = w / h - h_fov = math.radians(self.cfg.kit_horizontal_fov_deg) + h_fov = math.radians(self.cfg.horiz_fov_deg) v_fov_deg = math.degrees(2.0 * math.atan(math.tan(h_fov / 2.0) / aspect)) viewer.camera.fov = v_fov_deg viewer.set_camera(pos=wp.vec3(ex, ey, ez), pitch=pitch, yaw=yaw) diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py index 865efe1a26c7..c2f8ef5cbbf8 100644 --- a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py @@ -23,17 +23,17 @@ class NewtonGlPerspectiveVideoCfg: """Implementation class; default is :class:`~isaaclab_newton.video_recording.newton_gl_perspective_video.NewtonGlPerspectiveVideo`.""" - gl_viewer_width: int = 1280 + window_width: int = 1280 """Viewer width in pixels.""" - gl_viewer_height: int = 720 + window_height: int = 720 """Viewer height in pixels.""" - camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) """Camera position in world space (metres).""" - camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Camera look-at point in world space (metres).""" + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at target in world space (metres).""" - kit_horizontal_fov_deg: float = 60.0 + horiz_fov_deg: float = 60.0 """Horizontal FOV assumed for Kit ``/OmniverseKit_Persp``; converted to vertical FOV for GL viewer.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py index 513fa6e359fc..769d87251886 100644 --- a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py @@ -30,13 +30,13 @@ def render_rgb_array(self) -> np.ndarray: omni.kit.app.get_app().update() - h, w = self.cfg.render_height, self.cfg.render_width + h, w = self.cfg.window_height, self.cfg.window_width if self._rgb_annotator is None: import isaacsim.core.utils.viewports as isaacsim_viewports isaacsim_viewports.set_camera_view( - eye=list(self.cfg.camera_eye), - target=list(self.cfg.camera_lookat), + eye=list(self.cfg.camera_position), + target=list(self.cfg.camera_target), camera_prim_path=self.cfg.camera_prim_path, ) self._render_product = rep.create.render_product(self.cfg.camera_prim_path, (w, h)) diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py index d477604e14a8..8a256c5f133c 100644 --- a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py @@ -28,14 +28,14 @@ class IsaacsimKitPerspectiveVideoCfg: camera_prim_path: str = "/OmniverseKit_Persp" """Viewport camera prim used for the render product.""" - camera_eye: tuple[float, float, float] = (7.5, 7.5, 7.5) + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) """Camera position in world space (metres).""" - camera_lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Camera look-at point in world space (metres).""" + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at target in world space (metres).""" - render_width: int = 1280 + window_width: int = 1280 """Output width in pixels.""" - render_height: int = 720 + window_height: int = 720 """Output height in pixels."""