Skip to content

Add --video recording for all renderer backends (Newton+OVRTX, Newton+Warp) #4936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bdilinila wants to merge 12 commits into
base: develop
Choose a base branch
from
+747 −96
Draft

Add --video recording for all renderer backends (Newton+OVRTX, Newton+Warp) #4936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8f4c68f
Add --video support for kitless renderers backends
bdilinila Mar 11, 2026
b628c4d
Refactor video recording into VideoRecorder class with caching and co…
bdilinila Mar 11, 2026
5561542
tiled camera-based video recording functionality
bdilinila Mar 12, 2026
d03f028
Add --video=perspective mode for perspective recording
bdilinila Mar 12, 2026
2cca98d
Match Newton GL perspective camera to OmniverseKit_Persp camera throu…
bdilinila Mar 13, 2026
ed55c9e
Refactor video recording: move OV camera into VideoRecorder, clean up…
bdilinila Mar 13, 2026
9988962
Clean up video recorder: drop redundant kit cfg injection, debug logs…
bdilinila Mar 13, 2026
ad35aa0
greptile fix: fix video recorder bugs
bdilinila Mar 13, 2026
253dfee
Merge branch 'develop' into dev/bdilinila/kitless-video-support
bdilinila Mar 13, 2026
9714614
fix: return blank frame instead of None from _render_newton_gl_rgb_ar…
bdilinila Mar 13, 2026
e1a3b64
fix: clarify --video help text in rlinf/play.py; mode selection not y…
bdilinila Mar 13, 2026
2fee9dc
Added unit tests
bdilinila Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion scripts/benchmarks/benchmark_non_rl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,14 @@

# add argparse arguments
parser = argparse.ArgumentParser(description="Train an RL agent with RL-Games.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -140,6 +147,10 @@ def main(

task_startup_time_begin = time.perf_counter_ns()

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)
# wrap for video recording
Expand Down
13 changes: 12 additions & 1 deletion scripts/benchmarks/benchmark_rlgames.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,14 @@

# add argparse arguments
parser = argparse.ArgumentParser(description="Train an RL agent with RL-Games.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -205,6 +212,10 @@ def main(

task_startup_time_begin = time.perf_counter_ns()

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)
# wrap for video recording
Expand Down
13 changes: 12 additions & 1 deletion scripts/benchmarks/benchmark_rsl_rl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,14 @@

# add argparse arguments
parser = argparse.ArgumentParser(description="Train an RL agent with RSL-RL.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=4096, help="Number of environments to simulate.")
Expand Down Expand Up @@ -192,6 +199,10 @@ def main(

task_startup_time_begin = time.perf_counter_ns()

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)
# wrap for video recording
Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/rl_games/play.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Play a checkpoint of an RL agent from RL-Games.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during playing. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument(
"--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
Expand Down Expand Up @@ -114,6 +121,10 @@ def main():
obs_groups = agent_cfg["params"]["env"].get("obs_groups")
concate_obs_groups = agent_cfg["params"]["env"].get("concate_obs_groups", True)

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/rl_games/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Train an RL agent with RL-Games.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -158,6 +165,10 @@ def main():
# set the log directory for the environment
env_cfg.log_dir = os.path.join(log_root_path, log_dir)

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
9 changes: 8 additions & 1 deletion scripts/reinforcement_learning/rlinf/play.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,14 @@
parser.add_argument(
"--num_episodes", type=int, default=None, help="Number of evaluation episodes (overrides config if set)."
)
parser.add_argument("--video", action="store_true", default=False, help="Enable video recording.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Enable video recording. MODE is 'perspective' (default) or 'tiled'. Note: mode selection is not yet supported for rlinf; any non-None value enables recording.",
)
cli_args.add_rlinf_args(parser)
args_cli = parser.parse_args()

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/rsl_rl/play.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Train an RL agent with RSL-RL.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during playing. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument(
"--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
Expand Down Expand Up @@ -109,6 +116,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
# set the log directory for the environment
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/rsl_rl/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Train an RL agent with RSL-RL.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -147,6 +154,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
# set the log directory for the environment (works for all environment types)
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/sb3/play.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Play a checkpoint of an RL agent from Stable-Baselines3.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during playing. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument(
"--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
Expand Down Expand Up @@ -107,6 +114,10 @@ def main():
# set the log directory for the environment
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/sb3/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Train an RL agent with Stable-Baselines3.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -137,6 +144,10 @@ def main():
# set the log directory for the environment
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/skrl/play.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Play a checkpoint of an RL agent from skrl.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during playing. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument(
"--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
Expand Down Expand Up @@ -150,6 +157,10 @@ def main():
# set the log directory for the environment
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/reinforcement_learning/skrl/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,14 @@

# -- argparse ----------------------------------------------------------------
parser = argparse.ArgumentParser(description="Train an RL agent with skrl.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during training. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument("--video_interval", type=int, default=2000, help="Interval between video recordings (in steps).")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
Expand Down Expand Up @@ -173,6 +180,10 @@ def main():
# set the log directory for the environment
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
13 changes: 12 additions & 1 deletion scripts/sim2sim_transfer/rsl_rl_transfer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,14 @@

# add argparse arguments
parser = argparse.ArgumentParser(description="Play an RL agent with RSL-RL with policy transfer.")
parser.add_argument("--video", action="store_true", default=False, help="Record videos during training.")
parser.add_argument(
"--video",
nargs="?",
const="perspective",
default=None,
metavar="MODE",
help="Record videos during transfer. MODE is 'perspective' (default, wide-angle isometric view) or 'tiled' (camera-sensor tile-grid).",
)
parser.add_argument("--video_length", type=int, default=200, help="Length of the recorded video (in steps).")
parser.add_argument(
"--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
Expand Down Expand Up @@ -171,6 +178,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
# set the log directory for the environment (works for all environment types)
env_cfg.log_dir = log_dir

# Forward the video mode ("tiled" / "perspective") to the recorder config before env creation.
if args_cli.video and hasattr(env_cfg, "video_recorder") and env_cfg.video_recorder is not None:
env_cfg.video_recorder.video_mode = args_cli.video

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

Expand Down
Loading
Loading