diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index eb0a359e4f5..c6009117f1b 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -36,6 +36,7 @@ from .direct_marl_env_cfg import DirectMARLEnvCfg from .ui import ViewportCameraController from .utils.spaces import sample_space, spec_to_gym_space +from .utils.video_recorder import VideoRecorder # import logger logger = logging.getLogger(__name__) @@ -168,6 +169,19 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. + if self.cfg.video_recorder is not None: + self.cfg.video_recorder.env_render_mode = render_mode + vr = self.cfg.video_recorder + vr.camera_position = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_target = tuple(float(x) for x in self.cfg.viewer.lookat) + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -521,33 +535,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - if not self.sim.has_gui and not self.sim.has_offscreen_render: - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renderer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py index b22a6169d7a..9f3e6a627b0 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py @@ -17,6 +17,7 @@ from isaaclab.utils.noise import NoiseModelCfg from .common import AgentID, SpaceType, ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -234,3 +235,6 @@ class DirectMARLEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index b362ac72bc2..05dc8495dbc 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -33,6 +33,7 @@ from .direct_rl_env_cfg import DirectRLEnvCfg from .ui import ViewportCameraController from .utils.spaces import sample_space, spec_to_gym_space +from .utils.video_recorder import VideoRecorder if has_kit(): import omni.kit.app @@ -173,6 +174,20 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # Forward render_mode so VideoRecorder only spawns fallback cameras when --video is active. + if self.cfg.video_recorder is not None: + self.cfg.video_recorder.env_render_mode = render_mode + # Perspective --video uses same eye/lookat as task viewer (Kit persp + Newton GL). + vr = self.cfg.video_recorder + vr.camera_position = tuple(float(x) for x in self.cfg.viewer.eye) + vr.camera_target = tuple(float(x) for x in self.cfg.viewer.lookat) + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers @@ -489,33 +504,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - if not self.sim.has_gui and not self.sim.has_offscreen_render: - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renerer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py index fd40b3104c2..69b917e1d01 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py @@ -16,6 +16,7 @@ from isaaclab.utils.noise import NoiseModelCfg from .common import SpaceType, ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -254,3 +255,6 @@ class DirectRLEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index 996e88216e1..1e8ca057610 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -26,6 +26,7 @@ from .manager_based_env_cfg import ManagerBasedEnvCfg from .ui import ViewportCameraController from .utils.io_descriptors import export_articulations_data, export_scene_data +from .utils.video_recorder import VideoRecorder # import logger logger = logging.getLogger(__name__) @@ -182,6 +183,16 @@ def _init_sim(self): if "prestartup" in self.event_manager.available_modes: self.event_manager.apply(mode="prestartup") + # Instantiate the video recorder before sim.reset() so that any fallback TiledCamera + # (used for state-based envs without an observation camera) is spawned into the USD + # stage and registered for the PHYSICS_READY callback before physics initialises. + # env_render_mode and camera_position/camera_target are forwarded by subclasses (e.g. ManagerBasedRLEnv) + # into cfg.video_recorder before calling super().__init__(). + if self.cfg.video_recorder is not None: + self.video_recorder: VideoRecorder = self.cfg.video_recorder.class_type(self.cfg.video_recorder, self.scene) + else: + self.video_recorder = None + # play the simulator to activate physics handles # note: this activates the physics simulation view that exposes TensorAPIs # note: when started in extension mode, first call sim.reset_async() and then initialize the managers diff --git a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py index 24a88d5e72c..843701b03fd 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env_cfg.py @@ -26,6 +26,7 @@ from isaaclab.utils import configclass from .common import ViewerCfg +from .utils.video_recorder_cfg import VideoRecorderCfg @configclass @@ -163,3 +164,6 @@ class ManagerBasedEnvCfg: log_dir: str | None = None """Directory for logging experiment artifacts. Defaults to None, in which case no specific log directory is set.""" + + video_recorder: VideoRecorderCfg = VideoRecorderCfg() + """Configuration for video recording when ``render_mode="rgb_array"`` (i.e. ``--video``).""" diff --git a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py index d08b7e3be3a..0082912ec59 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_rl_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_rl_env.py @@ -76,6 +76,14 @@ def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, ** # initialize the episode length buffer BEFORE loading the managers to use it in mdp functions. self.episode_length_buf = torch.zeros(cfg.scene.num_envs, device=cfg.sim.device, dtype=torch.long) + # Forward render_mode and viewer camera to VideoRecorderCfg before super().__init__() + # creates the VideoRecorder, so fallback cameras are only spawned when --video is active + # (env_render_mode="rgb_array") and the perspective view matches the task viewport. + if cfg.video_recorder is not None: + cfg.video_recorder.env_render_mode = render_mode + cfg.video_recorder.camera_position = tuple(float(x) for x in cfg.viewer.eye) + cfg.video_recorder.camera_target = tuple(float(x) for x in cfg.viewer.lookat) + # initialize the base class to setup the scene. super().__init__(cfg=cfg) # store the render mode @@ -270,35 +278,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None: if self.render_mode == "human" or self.render_mode is None: return None elif self.render_mode == "rgb_array": - # check that if any render could have happened - # Check for GUI, offscreen rendering, or visualizers - has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer")) - if not (self.sim.has_gui or self.sim.has_offscreen_render or has_visualizers): - raise RuntimeError( - f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled." - " If running headless, make sure --enable_cameras is set." - ) - # create the annotator if it does not exist - if not hasattr(self, "_rgb_annotator"): - import omni.replicator.core as rep - - # create render product - self._render_product = rep.create.render_product( - self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution - ) - # create rgb annotator -- used to read data from the render product - self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - self._rgb_annotator.attach([self._render_product]) - # obtain the rgb data - rgb_data = self._rgb_annotator.get_data() - # convert to numpy array - rgb_data = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - # return the rgb data - # note: initially the renerer is warming up and returns empty data - if rgb_data.size == 0: - return np.zeros((self.cfg.viewer.resolution[1], self.cfg.viewer.resolution[0], 3), dtype=np.uint8) - else: - return rgb_data[:, :, :3] + if self.video_recorder is None: + return None + return self.video_recorder.render_rgb_array() else: raise NotImplementedError( f"Render mode '{self.render_mode}' is not supported. Please use: {self.metadata['render_modes']}." diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder.py b/source/isaaclab/isaaclab/envs/utils/video_recorder.py new file mode 100644 index 00000000000..9ffebdbe9b5 --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder.py @@ -0,0 +1,120 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Video recorder implementation. + +Captures a single wide-angle perspective view of the scene: + +* **Kit backends** (PhysX physics or Isaac RTX renderer) — uses + :mod:`isaaclab_physx.video_recording.isaacsim_kit_perspective_video`. +* **Newton backends** (Newton physics or Newton Warp renderer only) — uses + :mod:`isaaclab_newton.video_recording.newton_gl_perspective_video`. + +If neither a Kit nor a Newton backend is detected, construction raises so users do not +use ``--video`` on unsupported setups. + +See :mod:`video_recorder_cfg` for configuration. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Literal + +import numpy as np + +if TYPE_CHECKING: + from isaaclab.scene import InteractiveScene + + from .video_recorder_cfg import VideoRecorderCfg + +logger = logging.getLogger(__name__) + +_VideoBackend = Literal["kit", "newton_gl"] + + +def _resolve_video_backend(scene: InteractiveScene) -> _VideoBackend: + """Resolve which video backend to use from physics and renderer configs. + + Priority: PhysX or Isaac RTX -> Kit camera; else Newton or Newton Warp -> GL viewer. + When both are present (e.g. PhysX + Newton Warp), Kit wins. + """ + sim = scene.sim + physics_name = sim.physics_manager.__name__.lower() + renderer_types: list[str] = scene._sensor_renderer_types() + + use_kit = "physx" in physics_name or "isaac_rtx" in renderer_types + use_newton_gl = "newton" in physics_name or "newton_warp" in renderer_types + + if use_kit: + return "kit" + if use_newton_gl: + return "newton_gl" + raise RuntimeError( + "Video recording (--video) requires a supported backend: " + "PhysX or Isaac RTX renderer (Kit camera), or Newton physics / Newton Warp renderer (GL viewer). " + "No supported backend detected; do not use --video for this setup." + ) + + +class VideoRecorder: + """Records perspective video frames from the scene's active renderer. + + Args: + cfg: Recorder configuration. + scene: The interactive scene that owns the sensors. + """ + + def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene): + self.cfg = cfg + self._scene = scene + self._backend: _VideoBackend | None = None + self._capture = None + + if cfg.env_render_mode == "rgb_array": + self._backend = _resolve_video_backend(scene) + if self._backend == "newton_gl": + try: + import pyglet + + if not pyglet.options.get("headless", False): + pyglet.options["headless"] = True + except ImportError as e: + raise ImportError( + "The Newton GL video backend requires 'pyglet'. Install IsaacLab with './isaaclab.sh -i'." + ) from e + from isaaclab_newton.video_recording.newton_gl_perspective_video import ( + create_newton_gl_perspective_video, + ) + from isaaclab_newton.video_recording.newton_gl_perspective_video_cfg import NewtonGlPerspectiveVideoCfg + + ncfg = NewtonGlPerspectiveVideoCfg( + window_width=cfg.window_width, + window_height=cfg.window_height, + camera_position=cfg.camera_position, + camera_target=cfg.camera_target, + ) + self._capture = create_newton_gl_perspective_video(ncfg) + else: + from isaaclab_physx.video_recording.isaacsim_kit_perspective_video import ( + create_isaacsim_kit_perspective_video, + ) + from isaaclab_physx.video_recording.isaacsim_kit_perspective_video_cfg import ( + IsaacsimKitPerspectiveVideoCfg, + ) + + kcfg = IsaacsimKitPerspectiveVideoCfg( + camera_position=cfg.camera_position, + camera_target=cfg.camera_target, + window_width=cfg.window_width, + window_height=cfg.window_height, + ) + self._capture = create_isaacsim_kit_perspective_video(kcfg) + + def render_rgb_array(self) -> np.ndarray | None: + """Return an RGB frame for the resolved backend. Fails if backend is unavailable.""" + if self._backend is None or self._capture is None: + return None + return self._capture.render_rgb_array() diff --git a/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py new file mode 100644 index 00000000000..586779e3e67 --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`. + +Captures a single wide-angle perspective view of the scene. Newton backends use the +Newton GL viewer; Kit backends use the ``/OmniverseKit_Persp`` camera via +``omni.replicator.core``. +""" + +from __future__ import annotations + +from isaaclab.utils import configclass + +from .video_recorder import VideoRecorder + + +@configclass +class VideoRecorderCfg: + """Configuration for :class:`~isaaclab.envs.utils.video_recorder.VideoRecorder`.""" + + class_type: type = VideoRecorder + """Recorder class to instantiate; must accept ``(cfg, scene)``.""" + + env_render_mode: str | None = None + """Gym render mode forwarded from the environment constructor (``"rgb_array"`` when ``--video`` is active). + + Set automatically by the environment base classes; do not set manually. + """ + + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Perspective camera position in world space (metres). + + Direct RL / MARL and manager-based RL environments overwrite this from + :attr:`~isaaclab.envs.common.ViewerCfg.eye` before recording so ``--video`` matches the + task viewport for both Kit (PhysX / Isaac RTX) and Newton GL (Newton / OVRTX / etc.). + """ + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Perspective camera look-at target in world space (metres). Set from ``ViewerCfg.lookat`` at env init.""" + + window_width: int = 1280 + """Width in pixels of the recorded frame.""" + + window_height: int = 720 + """Height in pixels of the recorded frame.""" diff --git a/source/isaaclab/test/envs/test_video_recorder.py b/source/isaaclab/test/envs/test_video_recorder.py new file mode 100644 index 00000000000..036f569d33c --- /dev/null +++ b/source/isaaclab/test/envs/test_video_recorder.py @@ -0,0 +1,112 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +"""Unit tests for VideoRecorder.""" + +import importlib.util +import pathlib +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +_spec = importlib.util.spec_from_file_location("_vr", pathlib.Path(__file__).parent / "video_recorder.py") +_module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_module) +VideoRecorder = _module.VideoRecorder +_video_recorder_module = _module + +_BLANK_720p = np.zeros((720, 1280, 3), dtype=np.uint8) +_DEFAULT_CFG = dict( + render_mode="rgb_array", + video_mode="perspective", + fallback_camera_cfg=None, + video_num_tiles=-1, + camera_eye=(7.5, 7.5, 7.5), + camera_lookat=(0.0, 0.0, 0.0), + gl_viewer_width=1280, + gl_viewer_height=720, +) + + +def _create_recorder(**kw): + """Return a VideoRecorder with __init__ bypassed and all deps mocked out.""" + backend = kw.pop("_backend", None) + recorder = object.__new__(VideoRecorder) + recorder.cfg = SimpleNamespace(**{**_DEFAULT_CFG, **kw}) + recorder._scene = MagicMock() + recorder._scene.sensors = {} + recorder._scene._sensor_renderer_types = MagicMock(return_value=[]) + recorder._backend = backend + cap = MagicMock() + cap.render_rgb_array = MagicMock(return_value=_BLANK_720p) + recorder._capture = cap if backend else None + return recorder + + +def test_init_perspective_mode_creates_kit_capture(): + """With kit backend, __init__ builds Isaac Sim Kit perspective capture.""" + scene = MagicMock() + scene.sensors = {} + scene.num_envs = 1 + cfg = SimpleNamespace(**{**_DEFAULT_CFG, "fallback_camera_cfg": MagicMock()}) + fake_capture = MagicMock() + with patch.object(_video_recorder_module, "_resolve_video_backend", return_value="kit"): + with patch( + "isaaclab_physx.video_recording.isaacsim_kit_perspective_video.create_isaacsim_kit_perspective_video", + return_value=fake_capture, + ) as mock_create: + vr = VideoRecorder(cfg, scene) + mock_create.assert_called_once() + assert vr._capture is fake_capture + + +def test_init_newton_backend_creates_newton_capture(): + """With newton_gl backend, __init__ builds Newton GL perspective capture.""" + scene = MagicMock() + cfg = SimpleNamespace(**_DEFAULT_CFG) + fake_capture = MagicMock() + with patch.dict(sys.modules, {"pyglet": MagicMock()}): + with patch.object(_video_recorder_module, "_resolve_video_backend", return_value="newton_gl"): + with patch( + "isaaclab_newton.video_recording.newton_gl_perspective_video.create_newton_gl_perspective_video", + return_value=fake_capture, + ) as mock_create: + vr = VideoRecorder(cfg, scene) + mock_create.assert_called_once() + assert vr._capture is fake_capture + + +def test_render_rgb_array_delegates_to_capture(): + """render_rgb_array returns capture.render_rgb_array().""" + recorder = _create_recorder(_backend="kit") + result = recorder.render_rgb_array() + recorder._capture.render_rgb_array.assert_called_once() + assert result.shape == (720, 1280, 3) + + +def test_render_rgb_array_none_when_no_backend(): + """Without rgb_array render_mode, _capture is None and render returns None.""" + recorder = _create_recorder(render_mode=None) + recorder._backend = None + recorder._capture = None + assert recorder.render_rgb_array() is None + + +def test_capture_exception_propagates(): + """Failures in backend capture propagate.""" + recorder = _create_recorder(_backend="newton_gl") + recorder._capture.render_rgb_array.side_effect = RuntimeError("fail") + with pytest.raises(RuntimeError, match="fail"): + recorder.render_rgb_array() + + +def test_render_rgb_array_calls_capture_each_step(): + """Each render_rgb_array call hits the backend capture.""" + recorder = _create_recorder(_backend="kit") + for _ in range(3): + recorder.render_rgb_array() + assert recorder._capture.render_rgb_array.call_count == 3 diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py b/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py new file mode 100644 index 00000000000..3248ca5f13b --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton GL perspective video recording.""" diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py new file mode 100644 index 00000000000..cbc2af01def --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video.py @@ -0,0 +1,98 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton GL perspective RGB capture via headless ``newton.viewer.ViewerGL``.""" + +from __future__ import annotations + +import logging +import math +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from .newton_gl_perspective_video_cfg import NewtonGlPerspectiveVideoCfg + +logger = logging.getLogger(__name__) + + +class NewtonGlPerspectiveVideo: + """Lazy-initialised ViewerGL; one RGB frame per :meth:`render_rgb_array` call.""" + + def __init__(self, cfg: NewtonGlPerspectiveVideoCfg): + self.cfg = cfg + self._viewer = None + self._init_attempted = False + + def _ensure_viewer(self) -> None: + if self._init_attempted: + return + self._init_attempted = True + from isaaclab.sim import SimulationContext + + sdp = SimulationContext.instance().initialize_scene_data_provider() + model = sdp.get_newton_model() + if model is None: + raise RuntimeError( + "Newton GL perspective video requires a Newton model on the scene data provider. " + "Do not use --video for this setup." + ) + + import pyglet + + pyglet.options["headless"] = True + from newton.viewer import ViewerGL + + w, h = self.cfg.window_width, self.cfg.window_height + viewer = ViewerGL(width=w, height=h, headless=True) + viewer.set_model(model) + viewer.set_world_offsets((0.0, 0.0, 0.0)) + viewer.up_axis = 2 + + import warp as wp + + ex, ey, ez = self.cfg.camera_position + lx, ly, lz = self.cfg.camera_target + dx, dy, dz = lx - ex, ly - ey, lz - ez + length = math.sqrt(dx**2 + dy**2 + dz**2) + dx, dy, dz = dx / length, dy / length, dz / length + pitch = math.degrees(math.asin(max(-1.0, min(1.0, dz)))) + yaw = math.degrees(math.atan2(dy, dx)) + aspect = w / h + h_fov = math.radians(self.cfg.horiz_fov_deg) + v_fov_deg = math.degrees(2.0 * math.atan(math.tan(h_fov / 2.0) / aspect)) + viewer.camera.fov = v_fov_deg + viewer.set_camera(pos=wp.vec3(ex, ey, ez), pitch=pitch, yaw=yaw) + + self._viewer = viewer + logger.info("[NewtonGlPerspectiveVideo] ViewerGL ready (%dx%d).", w, h) + + def render_rgb_array(self) -> np.ndarray: + """Return one RGB frame from the Newton GL viewer. Raises on failure.""" + self._ensure_viewer() + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + sdp = sim.initialize_scene_data_provider() + state = sdp.get_newton_state() + dt = sim.get_physics_dt() + + viewer = self._viewer + viewer.begin_frame(dt) + viewer.log_state(state) + viewer.end_frame() + return viewer.get_frame().numpy() + + +def create_newton_gl_perspective_video(cfg: NewtonGlPerspectiveVideoCfg) -> NewtonGlPerspectiveVideo: + """Instantiate the Newton GL perspective capture from ``cfg.class_type``.""" + ct = cfg.class_type + if isinstance(ct, type): + return ct(cfg) + from isaaclab.utils.string import string_to_callable + + cls = string_to_callable(str(ct)) + return cls(cfg) diff --git a/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py new file mode 100644 index 00000000000..c2f8ef5cbbf --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/video_recording/newton_gl_perspective_video_cfg.py @@ -0,0 +1,39 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Newton GL perspective RGB capture (headless ViewerGL).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + pass + + +@configclass +class NewtonGlPerspectiveVideoCfg: + """Settings for capturing a perspective RGB frame via ``newton.viewer.ViewerGL``.""" + + class_type: type[Any] | str = "isaaclab_newton.video_recording.newton_gl_perspective_video:NewtonGlPerspectiveVideo" + """Implementation class; default is + :class:`~isaaclab_newton.video_recording.newton_gl_perspective_video.NewtonGlPerspectiveVideo`.""" + + window_width: int = 1280 + """Viewer width in pixels.""" + + window_height: int = 720 + """Viewer height in pixels.""" + + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Camera position in world space (metres).""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at target in world space (metres).""" + + horiz_fov_deg: float = 60.0 + """Horizontal FOV assumed for Kit ``/OmniverseKit_Persp``; converted to vertical FOV for GL viewer.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py b/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py new file mode 100644 index 00000000000..8deb00ddc46 --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Sim Kit perspective video recording.""" diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py new file mode 100644 index 00000000000..769d8725188 --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Sim Kit perspective RGB capture via ``/OmniverseKit_Persp`` and omni.replicator.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from .isaacsim_kit_perspective_video_cfg import IsaacsimKitPerspectiveVideoCfg + + +class IsaacsimKitPerspectiveVideo: + """Stateful capture of one RGB frame per call from the Kit perspective camera.""" + + def __init__(self, cfg: IsaacsimKitPerspectiveVideoCfg): + self.cfg = cfg + self._rgb_annotator = None + self._render_product = None + + def render_rgb_array(self) -> np.ndarray: + """Return one RGB frame. Blank frame during warmup; raises on other failures.""" + import omni.kit.app + import omni.replicator.core as rep + + omni.kit.app.get_app().update() + + h, w = self.cfg.window_height, self.cfg.window_width + if self._rgb_annotator is None: + import isaacsim.core.utils.viewports as isaacsim_viewports + + isaacsim_viewports.set_camera_view( + eye=list(self.cfg.camera_position), + target=list(self.cfg.camera_target), + camera_prim_path=self.cfg.camera_prim_path, + ) + self._render_product = rep.create.render_product(self.cfg.camera_prim_path, (w, h)) + self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") + self._rgb_annotator.attach([self._render_product]) + + rgb_data = self._rgb_annotator.get_data() + if isinstance(rgb_data, dict): + rgb_data = rgb_data.get("data", np.array([], dtype=np.uint8)) + rgb_data = np.asarray(rgb_data, dtype=np.uint8) + if rgb_data.size == 0: + return np.zeros((h, w, 3), dtype=np.uint8) + if rgb_data.ndim == 1: + rgb_data = rgb_data.reshape(h, w, -1) + return rgb_data[:, :, :3] + + +def create_isaacsim_kit_perspective_video(cfg: IsaacsimKitPerspectiveVideoCfg) -> IsaacsimKitPerspectiveVideo: + """Instantiate the perspective video capture implementation from ``cfg.class_type``.""" + ct = cfg.class_type + if isinstance(ct, type): + return ct(cfg) + from isaaclab.utils.string import string_to_callable + + cls = string_to_callable(str(ct)) + return cls(cfg) diff --git a/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py new file mode 100644 index 00000000000..8a256c5f133 --- /dev/null +++ b/source/isaaclab_physx/isaaclab_physx/video_recording/isaacsim_kit_perspective_video_cfg.py @@ -0,0 +1,41 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Isaac Sim Kit perspective RGB capture (OmniverseKit_Persp + Replicator).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + pass + + +@configclass +class IsaacsimKitPerspectiveVideoCfg: + """Settings for capturing a perspective RGB frame from the Kit viewport camera.""" + + class_type: type[Any] | str = ( + "isaaclab_physx.video_recording.isaacsim_kit_perspective_video:IsaacsimKitPerspectiveVideo" + ) + """Implementation class; default is + :class:`~isaaclab_physx.video_recording.isaacsim_kit_perspective_video.IsaacsimKitPerspectiveVideo`.""" + + camera_prim_path: str = "/OmniverseKit_Persp" + """Viewport camera prim used for the render product.""" + + camera_position: tuple[float, float, float] = (7.5, 7.5, 7.5) + """Camera position in world space (metres).""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Camera look-at target in world space (metres).""" + + window_width: int = 1280 + """Output width in pixels.""" + + window_height: int = 720 + """Output height in pixels."""