From d4dc70e548f6d6218cbee5e08ae0f4caf96a5224 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Fri, 7 Nov 2025 19:11:23 -0800 Subject: [PATCH 1/9] init --- .../isaaclab/sim/_impl/newton_manager_cfg.py | 15 +- .../isaaclab/isaaclab/sim/simulation_cfg.py | 18 ++ .../isaaclab/sim/visualizers/__init__.py | 30 ++++ .../sim/visualizers/newton_visualizer_cfg.py | 132 +++++++++++++++ .../sim/visualizers/ov_visualizer_cfg.py | 160 ++++++++++++++++++ .../sim/visualizers/rerun_visualizer_cfg.py | 159 +++++++++++++++++ .../sim/visualizers/visualizer_cfg.py | 89 ++++++++++ 7 files changed, 593 insertions(+), 10 deletions(-) create mode 100644 source/isaaclab/isaaclab/sim/visualizers/__init__.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py diff --git a/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py b/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py index d827e28d645..02523192c0b 100644 --- a/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py +++ b/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py @@ -12,7 +12,11 @@ class NewtonCfg: """Configuration for Newton-related parameters. - These parameters are used to configure the Newton simulation. + These parameters are used to configure the Newton physics simulation. + + Note: + Visualizer-related settings have been moved to NewtonVisualizerCfg in + isaaclab.sim.visualizers.newton_visualizer_cfg. """ num_substeps: int = 1 @@ -27,13 +31,4 @@ class NewtonCfg: If set to False, the simulation performance will be severely degraded. """ - newton_viewer_update_frequency: int = 1 - """Frequency of updates to the Newton viewer.""" - - newton_viewer_camera_pos: tuple[float, float, float] = (10.0, 0.0, 3.0) - """Position of the camera in the Newton viewer.""" - - visualizer_train_mode: bool = True - """Whether the visualizer is in training mode (True) or play mode (False).""" - solver_cfg: NewtonSolverCfg = MJWarpSolverCfg() diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index bea86d4150c..f6f2dfd5957 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -15,6 +15,7 @@ from ._impl.newton_manager_cfg import NewtonCfg from .spawners.materials import RigidBodyMaterialCfg +from .visualizers import VisualizerCfg @configclass @@ -203,6 +204,23 @@ class SimulationCfg: renderer is also enabled, both will be called. """ + visualizer: VisualizerCfg | None = None + """Visualizer settings. Default is None (no visualizer). + + To enable a visualizer, set this to one of the visualizer configuration classes: + - NewtonVisualizerCfg: Lightweight OpenGL-based visualizer + - OVVisualizerCfg: Omniverse-based high-fidelity visualizer + - RerunVisualizerCfg: Web-based Rerun visualizer + + Example: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + cfg = SimulationCfg(visualizer=NewtonVisualizerCfg(enabled=True)) + + Note: + This replaces the previous enable_newton_rendering flag and provides a unified + interface for all visualizer backends. + """ + create_stage_in_memory: bool = False """If stage is first created in memory. Default is False. diff --git a/source/isaaclab/isaaclab/sim/visualizers/__init__.py b/source/isaaclab/isaaclab/sim/visualizers/__init__.py new file mode 100644 index 00000000000..64a9dcf7f51 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-package for visualizer configurations. + +This sub-package contains configuration classes for different visualizer backends +that can be used with Isaac Lab. The visualizers are used for debug visualization +and monitoring of the simulation, separate from rendering for sensors. + +Supported visualizers: +- Newton OpenGL Visualizer: Lightweight OpenGL-based visualizer +- Omniverse Visualizer: High-fidelity Omniverse-based visualizer +- Rerun Visualizer: Web-based visualizer using the rerun library +""" + +from .visualizer_cfg import VisualizerCfg +from .newton_visualizer_cfg import NewtonVisualizerCfg +from .ov_visualizer_cfg import OVVisualizerCfg +from .rerun_visualizer_cfg import RerunVisualizerCfg + +__all__ = [ + "VisualizerCfg", + "NewtonVisualizerCfg", + "OVVisualizerCfg", + "RerunVisualizerCfg", +] + + diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py new file mode 100644 index 00000000000..e9304f9eca9 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py @@ -0,0 +1,132 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Newton OpenGL Visualizer.""" + +from typing import Literal + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class NewtonVisualizerCfg(VisualizerCfg): + """Configuration for Newton OpenGL Visualizer. + + The Newton OpenGL Visualizer is a lightweight, fast visualizer built on OpenGL. + It is designed for interactive real-time visualization of simulations with minimal + performance overhead. It requires pyglet (version >= 2.1.6) and imgui_bundle + (version >= 1.92.0) to be installed. + + Features: + - Real-time 3D visualization + - Interactive camera controls + - Debug visualization (contacts, joints, springs, COM) + - Training controls (pause training, pause rendering, update frequency) + - Lightweight and fast + + Note: + The Newton Visualizer currently only supports visualization of collision shapes, + not visual shapes. + """ + + # Override defaults for Newton visualizer + camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) + """Initial position of the camera. Default is (10.0, 0.0, 3.0).""" + + window_width: int = 1920 + """Width of the visualizer window in pixels. Default is 1920.""" + + window_height: int = 1080 + """Height of the visualizer window in pixels. Default is 1080.""" + + # Newton-specific settings + fps: int = 60 + """Target frames per second for the visualizer. Default is 60.""" + + show_joints: bool = True + """Whether to show joint visualizations. Default is True.""" + + show_contacts: bool = False + """Whether to show contact visualizations. Default is False.""" + + show_springs: bool = False + """Whether to show spring visualizations. Default is False.""" + + show_com: bool = False + """Whether to show center of mass visualizations. Default is False.""" + + show_ui: bool = True + """Whether to show the UI sidebar. Default is True. + + The UI can be toggled with the 'H' key during runtime. + """ + + enable_shadows: bool = True + """Whether to enable shadow rendering. Default is True.""" + + enable_sky: bool = True + """Whether to enable sky rendering. Default is True.""" + + enable_wireframe: bool = False + """Whether to enable wireframe rendering mode. Default is False.""" + + up_axis: Literal["X", "Y", "Z"] = "Z" + """The up axis for the visualizer. Default is "Z". + + This should typically match the up axis of your simulation environment. + """ + + fov: float = 60.0 + """Field of view for the camera in degrees. Default is 60.0.""" + + near_plane: float = 0.1 + """Near clipping plane distance for the camera. Default is 0.1.""" + + far_plane: float = 1000.0 + """Far clipping plane distance for the camera. Default is 1000.0.""" + + background_color: tuple[float, float, float] = (0.53, 0.81, 0.92) + """Background color (sky color) as RGB values in range [0, 1]. + Default is (0.53, 0.81, 0.92) (light blue).""" + + ground_color: tuple[float, float, float] = (0.18, 0.20, 0.25) + """Ground color as RGB values in range [0, 1]. + Default is (0.18, 0.20, 0.25) (dark gray).""" + + light_color: tuple[float, float, float] = (1.0, 1.0, 1.0) + """Light color as RGB values in range [0, 1]. Default is (1.0, 1.0, 1.0) (white).""" + + enable_pause_training: bool = True + """Whether to enable the pause training button in the UI. Default is True. + + When enabled, users can pause the simulation/training while keeping the + visualizer running, which is useful for debugging. + """ + + enable_pause_rendering: bool = True + """Whether to enable the pause rendering button in the UI. Default is True. + + When enabled, users can pause rendering while keeping simulation/training + running, which can improve training performance. + """ + + show_training_controls: bool = True + """Whether to show IsaacLab-specific training controls in the UI. Default is True. + + This includes controls for pausing training, pausing rendering, and adjusting + the visualizer update frequency. + """ + + render_mode: Literal["rgb", "depth", "collision"] = "rgb" + """Rendering mode for the visualizer. Default is "rgb". + + - "rgb": Standard RGB rendering + - "depth": Depth visualization + - "collision": Show collision shapes only + """ + + diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py new file mode 100644 index 00000000000..e67cc47b116 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py @@ -0,0 +1,160 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Omniverse Visualizer.""" + +from typing import Literal + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class OVVisualizerCfg(VisualizerCfg): + """Configuration for Omniverse Visualizer. + + The Omniverse Visualizer uses the Omniverse SDK to provide high-fidelity + visualization of the simulation. This is currently implemented through the + Isaac Sim app, but will eventually use the standalone Omniverse SDK module. + + Features: + - High-fidelity rendering with RTX + - Full visual shape support (not just collision shapes) + - USD-based scene representation + - Advanced lighting and materials + - Integration with Omniverse ecosystem + + Note: + The Omniverse Visualizer has higher overhead than the Newton Visualizer + and requires Omniverse/Isaac Sim to be installed. + """ + + # Override defaults for Omniverse visualizer + camera_position: tuple[float, float, float] = (10.0, 10.0, 10.0) + """Initial position of the camera. Default is (10.0, 10.0, 10.0).""" + + window_width: int = 1920 + """Width of the visualizer window in pixels. Default is 1920.""" + + window_height: int = 1080 + """Height of the visualizer window in pixels. Default is 1080.""" + + # Omniverse-specific settings + viewport_name: str = "/OmniverseKit_Persp" + """Name of the viewport to use for visualization. Default is "/OmniverseKit_Persp".""" + + show_origin_axis: bool = True + """Whether to show the origin axis. Default is True.""" + + show_grid: bool = True + """Whether to show the grid. Default is True.""" + + grid_scale: float = 1.0 + """Scale of the grid. Default is 1.0.""" + + enable_scene_lights: bool = True + """Whether to enable scene lights. Default is True.""" + + default_light_intensity: float = 3000.0 + """Default intensity for scene lights. Default is 3000.0.""" + + enable_dome_light: bool = True + """Whether to enable dome (environment) lighting. Default is True.""" + + dome_light_intensity: float = 1000.0 + """Intensity of the dome light. Default is 1000.0.""" + + dome_light_texture: str | None = None + """Path to HDR texture for dome light. Default is None (use default). + + If specified, should be a path to an HDR image file for image-based lighting. + """ + + camera_projection: Literal["perspective", "orthographic"] = "perspective" + """Camera projection type. Default is "perspective".""" + + fov: float = 60.0 + """Field of view for the camera in degrees (for perspective projection). Default is 60.0.""" + + near_plane: float = 0.1 + """Near clipping plane distance. Default is 0.1.""" + + far_plane: float = 10000.0 + """Far clipping plane distance. Default is 10000.0.""" + + enable_ui: bool = True + """Whether to enable the Omniverse UI. Default is True. + + When disabled, runs in a more minimal mode which can improve performance. + """ + + ui_window_layout: str | None = None + """Custom UI window layout file. Default is None (use default layout). + + Can be a path to a .json file specifying the UI layout. + """ + + show_selection_outline: bool = True + """Whether to show selection outline on picked objects. Default is True.""" + + show_physics_debug_viz: bool = False + """Whether to show physics debug visualization (contacts, joints). Default is False.""" + + show_bounding_boxes: bool = False + """Whether to show bounding boxes. Default is False.""" + + display_options: int = 3094 + """Display options bitmask. Default is 3094. + + This controls what is visible in the stage. The default value (3094) hides + extra objects that shouldn't appear in visualization. Another common value + is 3286 for the regular editor experience. + """ + + enable_live_sync: bool = False + """Whether to enable live sync with Omniverse. Default is False. + + When enabled, allows other Omniverse clients to connect and view the simulation. + """ + + antialiasing: Literal["Off", "FXAA", "TAA", "DLSS", "DLAA"] = "TAA" + """Anti-aliasing mode. Default is "TAA". + + - Off: No anti-aliasing + - FXAA: Fast approximate anti-aliasing + - TAA: Temporal anti-aliasing + - DLSS: NVIDIA Deep Learning Super Sampling (requires RTX GPU) + - DLAA: NVIDIA Deep Learning Anti-Aliasing (requires RTX GPU) + """ + + enable_post_processing: bool = True + """Whether to enable post-processing effects. Default is True.""" + + enable_motion_blur: bool = False + """Whether to enable motion blur. Default is False.""" + + enable_depth_of_field: bool = False + """Whether to enable depth of field. Default is False.""" + + background_color: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Background color as RGB values in range [0, 1]. Default is (0.0, 0.0, 0.0) (black).""" + + capture_on_play: bool = False + """Whether to start capturing when play is pressed. Default is False. + + Useful for recording the simulation for later playback. + """ + + capture_path: str | None = None + """Path to save captures. Default is None (don't capture). + + When specified, frames will be captured to this path. + """ + + capture_format: Literal["png", "jpg", "exr"] = "png" + """Format for captured images. Default is "png".""" + + diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py new file mode 100644 index 00000000000..487bc6e69eb --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py @@ -0,0 +1,159 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Rerun Visualizer.""" + +from typing import Literal + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class RerunVisualizerCfg(VisualizerCfg): + """Configuration for Rerun Visualizer. + + The Rerun Visualizer integrates with the rerun visualization library, enabling + real-time or offline visualization with advanced features like time scrubbing + and data inspection through a web-based interface. + + Features: + - Web-based visualization interface + - Time scrubbing and playback controls + - 3D scene navigation + - Data inspection and filtering + - Recording and export capabilities + - Remote viewing support + + Note: + Requires the rerun-sdk package to be installed: pip install rerun-sdk + """ + + # Override defaults for Rerun visualizer + camera_position: tuple[float, float, float] = (10.0, 10.0, 10.0) + """Initial position of the camera. Default is (10.0, 10.0, 10.0).""" + + # Rerun-specific settings + server_mode: bool = True + """Whether to run in server mode. Default is True. + + In server mode, Rerun starts a server that viewers can connect to. + When False, data is logged to a file or sent to an external viewer. + """ + + server_address: str = "127.0.0.1:9876" + """Server address for Rerun. Default is "127.0.0.1:9876". + + Format: "host:port". Only used when server_mode is True. + """ + + launch_viewer: bool = True + """Whether to automatically launch the web viewer. Default is True. + + When True, the Rerun web viewer will be automatically opened in a browser. + """ + + app_id: str = "isaaclab-simulation" + """Application identifier for Rerun. Default is "isaaclab-simulation". + + This is used to identify the application in the Rerun viewer and for + organizing recordings. + """ + + recording_path: str | None = None + """Path to save recordings. Default is None (don't save). + + When specified, the Rerun data will be saved to this path for later replay. + Supported formats: .rrd (Rerun recording format) + """ + + spawn_mode: Literal["connect", "spawn", "save"] = "spawn" + """How to handle the Rerun viewer. Default is "spawn". + + - "connect": Connect to an existing Rerun viewer + - "spawn": Spawn a new Rerun viewer process + - "save": Save to a file without opening a viewer + """ + + max_queue_size: int = 100 + """Maximum number of messages to queue. Default is 100. + + Controls memory usage for buffering visualization data. + """ + + flush_timeout: float = 2.0 + """Timeout for flushing data to Rerun in seconds. Default is 2.0.""" + + log_transforms: bool = True + """Whether to log rigid body transforms. Default is True.""" + + log_meshes: bool = True + """Whether to log mesh data. Default is True. + + When enabled, collision and visual meshes will be logged to Rerun. + """ + + log_cameras: bool = True + """Whether to log camera data. Default is True.""" + + log_point_clouds: bool = False + """Whether to log point cloud data. Default is False.""" + + log_images: bool = False + """Whether to log images from cameras. Default is False. + + Note: Logging images can significantly increase data size and bandwidth. + """ + + log_tensors: bool = False + """Whether to log tensor data (observations, actions, etc.). Default is False. + + When enabled, can log observation buffers, action buffers, and other tensors. + """ + + time_mode: Literal["sim_time", "wall_time", "step_count"] = "sim_time" + """Time mode for logging. Default is "sim_time". + + - "sim_time": Use simulation time as the timeline + - "wall_time": Use wall clock time + - "step_count": Use step count as the timeline + """ + + entity_path_prefix: str = "/world" + """Prefix for entity paths in Rerun. Default is "/world". + + All logged entities will be under this prefix in the Rerun hierarchy. + """ + + log_static_once: bool = True + """Whether to log static scene data only once. Default is True. + + When True, static meshes and other unchanging data are logged only at the + start, reducing data size and bandwidth. + """ + + up_axis: Literal["+x", "-x", "+y", "-y", "+z", "-z"] = "+z" + """The up axis for the 3D space. Default is "+z". + + This should match your simulation's coordinate system. + """ + + mesh_quality: Literal["low", "medium", "high"] = "medium" + """Quality level for mesh data. Default is "medium". + + Higher quality preserves more detail but increases data size. + """ + + enable_compression: bool = True + """Whether to enable data compression. Default is True. + + Compression reduces bandwidth and storage requirements but adds CPU overhead. + """ + + verbose: bool = False + """Whether to enable verbose logging. Default is False.""" + + diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py new file mode 100644 index 00000000000..fce04f0c786 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Base configuration for visualizers.""" + +from typing import Literal + +from isaaclab.utils import configclass + + +@configclass +class VisualizerCfg: + """Base configuration for visualizer backends. + + This is the base class for all visualizer configurations. Visualizers are used + for debug visualization and monitoring during simulation, separate from rendering + for sensors/cameras. + + All visualizer backends should inherit from this class and add their specific + configuration parameters. + """ + + enabled: bool = False + """Whether the visualizer is enabled. Default is False.""" + + update_frequency: int = 1 + """Frequency of updates to the visualizer (in simulation steps). + + Higher values (e.g., 10) mean the visualizer updates less frequently, improving + performance at the cost of less responsive visualization. Lower values (e.g., 1) + provide more responsive visualization but may impact performance. + Default is 1 (update every step). + """ + + env_indices: list[int] | None = None + """List of environment indices to visualize. Default is None. + + If None, all environments will be visualized. If a list is provided, only the + specified environments will be visualized. This is useful for reducing the + visualization overhead when running with many environments. + + Example: [0, 1, 2] will visualize only the first 3 environments. + """ + + enable_markers: bool = True + """Whether to enable visualization markers (debug drawing). Default is True. + + Visualization markers are used for debug drawing of points, lines, frames, etc. + These correspond to the VisualizationMarkers class in isaaclab.markers. + """ + + enable_live_plots: bool = True + """Whether to enable live plotting of data. Default is True. + + Live plots can be used to visualize real-time data such as observations, + rewards, and other metrics during simulation. + """ + + train_mode: bool = True + """Whether the visualizer is in training mode (True) or play/inference mode (False). + + This affects the UI and controls displayed in the visualizer. In training mode, + additional controls may be shown for pausing training, adjusting update frequency, etc. + Default is True. + """ + + camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) + """Initial position of the camera in the visualizer. Default is (10.0, 0.0, 3.0).""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Initial target (look-at point) of the camera. Default is (0.0, 0.0, 0.0).""" + + window_width: int = 1920 + """Width of the visualizer window in pixels. Default is 1920.""" + + window_height: int = 1080 + """Height of the visualizer window in pixels. Default is 1080.""" + + def get_visualizer_type(self) -> str: + """Get the type of visualizer as a string. + + Returns: + String identifier for the visualizer type. + """ + return self.__class__.__name__.replace("VisualizerCfg", "").replace("Cfg", "") + + From d70b0d8998d5c0a718319b392af62e3df88f3995 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Mon, 10 Nov 2025 21:01:22 -0800 Subject: [PATCH 2/9] initial impl --- .../isaaclab/sim/_impl/newton_manager.py | 84 ------- .../isaaclab/isaaclab/sim/simulation_cfg.py | 33 ++- .../isaaclab/sim/simulation_context.py | 130 +++++++++- .../isaaclab/sim/visualizers/__init__.py | 16 +- .../newton_visualizer.py} | 230 ++++++++++++++++-- .../isaaclab/sim/visualizers/ov_visualizer.py | 0 .../sim/visualizers/rerun_visualizer.py | 0 .../isaaclab/sim/visualizers/visualizer.py | 129 ++++++++++ .../isaaclab_tasks/utils/parse_cfg.py | 14 +- 9 files changed, 509 insertions(+), 127 deletions(-) rename source/isaaclab/isaaclab/sim/{_impl/newton_viewer.py => visualizers/newton_visualizer.py} (52%) create mode 100644 source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py create mode 100644 source/isaaclab/isaaclab/sim/visualizers/visualizer.py diff --git a/source/isaaclab/isaaclab/sim/_impl/newton_manager.py b/source/isaaclab/isaaclab/sim/_impl/newton_manager.py index 248d8161901..fb770d6aff3 100644 --- a/source/isaaclab/isaaclab/sim/_impl/newton_manager.py +++ b/source/isaaclab/isaaclab/sim/_impl/newton_manager.py @@ -16,7 +16,6 @@ from newton.solvers import SolverBase, SolverFeatherstone, SolverMuJoCo, SolverXPBD from isaaclab.sim._impl.newton_manager_cfg import NewtonCfg -from isaaclab.sim._impl.newton_viewer import NewtonViewerGL from isaaclab.utils.timer import Timer @@ -66,7 +65,6 @@ class NewtonManager: _report_contacts: bool = False _graph = None _newton_stage_path = None - _renderer = None _sim_time = 0.0 _usdrt_stage = None _newton_index_attr = "newton:index" @@ -76,10 +74,6 @@ class NewtonManager: _gravity_vector: tuple[float, float, float] = (0.0, 0.0, -9.81) _up_axis: str = "Z" _num_envs: int = None - _visualizer_update_counter: int = 0 - _visualizer_update_frequency: int = 1 # Configurable frequency for all rendering updates - _visualizer_train_mode: bool = True # Whether visualizer is in training mode - _visualizer_disabled: bool = False # Whether visualizer has been disabled by user @classmethod def clear(cls): @@ -95,7 +89,6 @@ def clear(cls): NewtonManager._report_contacts = False NewtonManager._graph = None NewtonManager._newton_stage_path = None - NewtonManager._renderer = None NewtonManager._sim_time = 0.0 NewtonManager._on_init_callbacks = [] NewtonManager._on_start_callbacks = [] @@ -103,9 +96,6 @@ def clear(cls): NewtonManager._cfg = NewtonCfg() NewtonManager._up_axis = "Z" NewtonManager._first_call = True - NewtonManager._visualizer_update_counter = 0 - NewtonManager._visualizer_disabled = False - NewtonManager._visualizer_update_frequency = NewtonManager._cfg.newton_viewer_update_frequency @classmethod def set_builder(cls, builder): @@ -302,80 +292,6 @@ def set_simulation_dt(cls, dt: float) -> None: """ NewtonManager._dt = dt - @classmethod - def _render_call(cls, render_func) -> bool: - if NewtonManager._renderer is not None: - try: - if hasattr(NewtonManager._renderer, "renderer") and hasattr(NewtonManager._renderer.renderer, "window"): - if NewtonManager._renderer.renderer.window.has_exit: - NewtonManager._visualizer_disabled = True - NewtonManager._renderer = None - return False - except Exception as e: - print(f"[ERROR] Error in _render_call: {e}") - - try: - render_func() - return True - except (ctypes.ArgumentError, Exception) as e: - if "wrong type" in str(e) or "ArgumentError" in str(e): - NewtonManager._visualizer_disabled = True - if NewtonManager._renderer is not None: - try: - NewtonManager._renderer.close() - except Exception as e: - print(f"[ERROR] Error in _render_call: {e}") - NewtonManager._renderer = None - return False - else: - raise - - @classmethod - def render(cls) -> None: - if NewtonManager._visualizer_disabled: - return - - if NewtonManager._renderer is None: - NewtonManager._visualizer_train_mode = NewtonManager._cfg.visualizer_train_mode - NewtonManager._renderer = NewtonViewerGL( - width=1280, height=720, train_mode=NewtonManager._visualizer_train_mode - ) - NewtonManager._renderer.set_model(NewtonManager._model) - NewtonManager._renderer.camera.pos = wp.vec3(*NewtonManager._cfg.newton_viewer_camera_pos) - NewtonManager._renderer.up_axis = NewtonManager._up_axis - NewtonManager._renderer.scaling = 1.0 - NewtonManager._renderer._paused = False - else: - while NewtonManager._renderer is not None and NewtonManager._renderer.is_training_paused(): - - def render_frame(): - NewtonManager._renderer.begin_frame(NewtonManager._sim_time) - NewtonManager._renderer.log_state(NewtonManager._state_0) - NewtonManager._renderer.end_frame() - - if not NewtonManager._render_call(render_frame): - return - - NewtonManager._visualizer_update_counter += 1 - if ( - NewtonManager._renderer is not None - and NewtonManager._visualizer_update_counter >= NewtonManager._visualizer_update_frequency - ): - if not NewtonManager._renderer.is_paused(): - - def render_frame(): - NewtonManager._renderer.begin_frame(NewtonManager._sim_time) - NewtonManager._renderer.log_state(NewtonManager._state_0) - NewtonManager._renderer.end_frame() - - if not NewtonManager._render_call(render_frame): - return - else: - if not NewtonManager._render_call(lambda: NewtonManager._renderer._update()): - return - - NewtonManager._visualizer_update_counter = 0 - @classmethod def sync_fabric_transforms(cls) -> None: """Syncs the fabric transforms with the Newton state. diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index f6f2dfd5957..4e46f75fcff 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -197,28 +197,35 @@ class SimulationCfg: render: RenderCfg = RenderCfg() """Render settings. Default is RenderCfg().""" - enable_newton_rendering: bool = False - """Enable/disable rendering using Newton. Default is False. - - When enabled, the Newton to renderer will be called every time the simulation is rendered. If Isaac Sim's - renderer is also enabled, both will be called. - """ - - visualizer: VisualizerCfg | None = None + visualizers: list[VisualizerCfg] | VisualizerCfg | None = None """Visualizer settings. Default is None (no visualizer). - To enable a visualizer, set this to one of the visualizer configuration classes: + This field supports multiple visualizer backends for debug visualization and monitoring + during simulation. It accepts: + - A single VisualizerCfg: One visualizer will be created + - A list of VisualizerCfg: Multiple visualizers will be created + - None or empty list: No visualizers will be created + + Supported visualizer backends: - NewtonVisualizerCfg: Lightweight OpenGL-based visualizer - OVVisualizerCfg: Omniverse-based high-fidelity visualizer - RerunVisualizerCfg: Web-based Rerun visualizer - Example: + Examples: + # Single visualizer from isaaclab.sim.visualizers import NewtonVisualizerCfg - cfg = SimulationCfg(visualizer=NewtonVisualizerCfg(enabled=True)) + cfg = SimulationCfg(visualizers=NewtonVisualizerCfg(enabled=True)) + + # Multiple visualizers + from isaaclab.sim.visualizers import NewtonVisualizerCfg, RerunVisualizerCfg + cfg = SimulationCfg(visualizers=[ + NewtonVisualizerCfg(enabled=True), + RerunVisualizerCfg(enabled=True) + ]) Note: - This replaces the previous enable_newton_rendering flag and provides a unified - interface for all visualizer backends. + Visualizers are separate from rendering backends (for cameras/sensors). + They are intended for debug visualization and monitoring only. """ create_stage_in_memory: bool = False diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 33c93873c1c..9664c44da99 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -35,6 +35,7 @@ from .simulation_cfg import SimulationCfg from .spawners import DomeLightCfg, GroundPlaneCfg from .utils import bind_physics_material +from .visualizers import Visualizer, NewtonVisualizer, NewtonVisualizerCfg class SimulationContext(_SimulationContext): @@ -252,6 +253,10 @@ def __init__(self, cfg: SimulationCfg | None = None): self._app_control_on_stop_handle = None self._disable_app_control_on_stop_handle = False + # initialize visualizers + self._visualizers: list[Visualizer] = [] + self._visualizer_step_counter = 0 + # flag for skipping prim deletion callback # when stage in memory is attached self._skip_next_prim_deletion_callback_fn = False @@ -538,6 +543,118 @@ def forward(self) -> None: NewtonManager.forward_kinematics() NewtonManager.sync_fabric_transforms() + def initialize_visualizers(self) -> None: + """Initialize all configured visualizers. + + This method creates and initializes visualizers based on the configuration provided + in SimulationCfg.visualizers. It supports: + - A single VisualizerCfg: Creates one visualizer + - A list of VisualizerCfg: Creates multiple visualizers + - None or empty list: No visualizers are created + """ + # Handle different input formats + visualizer_cfgs = [] + if self.cfg.visualizers is not None: + if isinstance(self.cfg.visualizers, list): + visualizer_cfgs = [cfg for cfg in self.cfg.visualizers if cfg.enabled] + elif self.cfg.visualizers.enabled: + visualizer_cfgs = [self.cfg.visualizers] + + # Create and initialize each visualizer + for viz_cfg in visualizer_cfgs: + try: + # Create visualizer instance based on config type + if isinstance(viz_cfg, NewtonVisualizerCfg): + visualizer = NewtonVisualizer(viz_cfg) + else: + # Skip unsupported visualizer types for now + omni.log.warn( + f"Visualizer type '{type(viz_cfg).__name__}' is not yet implemented. Skipping." + ) + continue + + # Initialize with Newton model if available + if NewtonManager._model is not None: + scene = { + "model": NewtonManager._model, + "state": NewtonManager._state_0, + } + visualizer.initialize(scene) + self._visualizers.append(visualizer) + omni.log.info(f"Initialized visualizer: {type(visualizer).__name__}") + else: + omni.log.warn( + "Newton model not available yet. Visualizer will be initialized later." + ) + + except Exception as e: + omni.log.error(f"Failed to initialize visualizer '{type(viz_cfg).__name__}': {e}") + + def step_visualizers(self, dt: float) -> None: + """Update all active visualizers. + + This method steps all initialized visualizers and updates their state. + It also handles visualizer pause states and removes closed visualizers. + + Args: + dt: Time step in seconds. + """ + if not self._visualizers: + return + + self._visualizer_step_counter += 1 + + # Update visualizers and check if any should be removed + visualizers_to_remove = [] + + for visualizer in self._visualizers: + try: + # Check if visualizer is still running + if not visualizer.is_running(): + visualizers_to_remove.append(visualizer) + continue + + # Handle training pause - block until resumed + while visualizer.is_training_paused() and visualizer.is_running(): + if isinstance(visualizer, NewtonVisualizer): + # Update state before rendering during pause + visualizer.update_state(NewtonManager._state_0) + visualizer.step(0.0) # Step with 0 dt during pause + + # Skip rendering if visualizer has rendering paused + if visualizer.is_rendering_paused(): + continue + + # Normal step: update state and visualizer + if isinstance(visualizer, NewtonVisualizer): + visualizer.update_state(NewtonManager._state_0) + + visualizer.step(dt) + + except Exception as e: + omni.log.error(f"Error stepping visualizer '{type(visualizer).__name__}': {e}") + visualizers_to_remove.append(visualizer) + + # Remove closed visualizers + for visualizer in visualizers_to_remove: + try: + visualizer.close() + self._visualizers.remove(visualizer) + omni.log.info(f"Removed visualizer: {type(visualizer).__name__}") + except Exception as e: + omni.log.error(f"Error closing visualizer: {e}") + + def close_visualizers(self) -> None: + """Close all active visualizers and clean up resources.""" + for visualizer in self._visualizers: + try: + visualizer.close() + except Exception as e: + omni.log.error(f"Error closing visualizer '{type(visualizer).__name__}': {e}") + + self._visualizers.clear() + omni.log.info("All visualizers closed") + def get_initial_stage(self) -> Usd.Stage: """Returns stage handle used during scene creation. @@ -577,6 +694,11 @@ def reset(self, soft: bool = False): if not soft: for _ in range(2): self.render() + + # Initialize visualizers after simulation is set up (only on first reset) + if not soft and not self._visualizers: + self.initialize_visualizers() + self._disable_app_control_on_stop_handle = False def step(self, render: bool = True): @@ -628,9 +750,8 @@ def step(self, render: bool = True): if self.is_playing(): NewtonManager.step() - # Use the NewtonManager to render the scene if enabled - if self.cfg.enable_newton_rendering: - NewtonManager.render() + # Update visualizers + self.step_visualizers(self.cfg.dt) # app.update() may be changing the cuda device in step, so we force it back to our desired device here if "cuda" in self.device: @@ -728,6 +849,9 @@ def clear_instance(cls): if cls._instance._app_control_on_stop_handle is not None: cls._instance._app_control_on_stop_handle.unsubscribe() cls._instance._app_control_on_stop_handle = None + # close all visualizers + if hasattr(cls._instance, '_visualizers'): + cls._instance.close_visualizers() # call parent to clear the instance super().clear_instance() NewtonManager.clear() diff --git a/source/isaaclab/isaaclab/sim/visualizers/__init__.py b/source/isaaclab/isaaclab/sim/visualizers/__init__.py index 64a9dcf7f51..0814b88a412 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/__init__.py +++ b/source/isaaclab/isaaclab/sim/visualizers/__init__.py @@ -3,25 +3,29 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-package for visualizer configurations. +"""Sub-package for visualizer configurations and implementations. -This sub-package contains configuration classes for different visualizer backends -that can be used with Isaac Lab. The visualizers are used for debug visualization -and monitoring of the simulation, separate from rendering for sensors. +This sub-package contains configuration classes and implementations for different +visualizer backends that can be used with Isaac Lab. The visualizers are used for +debug visualization and monitoring of the simulation, separate from rendering for sensors. Supported visualizers: - Newton OpenGL Visualizer: Lightweight OpenGL-based visualizer -- Omniverse Visualizer: High-fidelity Omniverse-based visualizer -- Rerun Visualizer: Web-based visualizer using the rerun library +- Omniverse Visualizer: High-fidelity Omniverse-based visualizer (coming soon) +- Rerun Visualizer: Web-based visualizer using the rerun library (coming soon) """ +from .visualizer import Visualizer from .visualizer_cfg import VisualizerCfg +from .newton_visualizer import NewtonVisualizer from .newton_visualizer_cfg import NewtonVisualizerCfg from .ov_visualizer_cfg import OVVisualizerCfg from .rerun_visualizer_cfg import RerunVisualizerCfg __all__ = [ + "Visualizer", "VisualizerCfg", + "NewtonVisualizer", "NewtonVisualizerCfg", "OVVisualizerCfg", "RerunVisualizerCfg", diff --git a/source/isaaclab/isaaclab/sim/_impl/newton_viewer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py similarity index 52% rename from source/isaaclab/isaaclab/sim/_impl/newton_viewer.py rename to source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index 00f13dc9202..ac9ed2c217c 100644 --- a/source/isaaclab/isaaclab/sim/_impl/newton_viewer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -3,30 +3,34 @@ # # SPDX-License-Identifier: BSD-3-Clause -""" -Training-aware viewer that adds a separate pause for simulation/training while keeping rendering at full rate. - -This class subclasses Newton's ViewerGL and introduces a second pause mode: -- Rendering pause: identical to the base viewer's pause (space key / Pause checkbox) -- Training pause: stops simulation/training steps but keeps rendering running - -The training pause can be toggled from the UI via a button and optionally via the 'T' key. -""" +"""Newton OpenGL Visualizer implementation.""" from __future__ import annotations -import newton as nt import warp as wp from newton.viewer import ViewerGL +from .newton_visualizer_cfg import NewtonVisualizerCfg +from .visualizer import Visualizer + class NewtonViewerGL(ViewerGL): + """Training-aware viewer that adds a separate pause for simulation/training. + + This class subclasses Newton's ViewerGL and introduces a second pause mode: + - Rendering pause: identical to the base viewer's pause (space key / Pause checkbox) + - Training pause: stops simulation/training steps but keeps rendering running + + The training pause can be toggled from the UI via a button and optionally via the 'T' key. + """ + def __init__(self, *args, train_mode: bool = True, **kwargs): super().__init__(*args, **kwargs) self._paused_training: bool = False self._paused_rendering: bool = False self._fallback_draw_controls: bool = False - self._is_train_mode: bool = train_mode # Convert train_mode to play_mode + self._is_train_mode: bool = train_mode + self._visualizer_update_frequency: int = 1 try: self.register_ui_callback(self._render_training_controls, position="side") @@ -34,11 +38,33 @@ def __init__(self, *args, train_mode: bool = True, **kwargs): self._fallback_draw_controls = True def is_training_paused(self) -> bool: + """Check if training is paused.""" return self._paused_training def is_paused(self) -> bool: + """Check if rendering is paused.""" return self._paused_rendering + def is_rendering_paused(self) -> bool: + """Check if rendering is paused (alias for is_paused for consistency).""" + return self._paused_rendering + + def set_visualizer_update_frequency(self, frequency: int) -> None: + """Set the visualizer update frequency. + + Args: + frequency: Number of simulation steps between visualizer updates. + """ + self._visualizer_update_frequency = max(1, frequency) + + def get_visualizer_update_frequency(self) -> int: + """Get the current visualizer update frequency. + + Returns: + Number of simulation steps between visualizer updates. + """ + return self._visualizer_update_frequency + # UI callback rendered inside the "Example Options" panel of the left sidebar def _render_training_controls(self, imgui): imgui.separator() @@ -60,17 +86,14 @@ def _render_training_controls(self, imgui): if imgui.button(rendering_label): self._paused_rendering = not self._paused_rendering - # Import NewtonManager locally to avoid circular imports - from .newton_manager import NewtonManager # noqa: PLC0415 - imgui.text("Visualizer Update Frequency") - current_frequency = NewtonManager._visualizer_update_frequency + current_frequency = self._visualizer_update_frequency changed, new_frequency = imgui.slider_int( "##VisualizerUpdateFreq", current_frequency, 1, 20, f"Every {current_frequency} frames" ) if changed: - NewtonManager._visualizer_update_frequency = new_frequency + self._visualizer_update_frequency = new_frequency if imgui.is_item_hovered(): imgui.set_tooltip( @@ -119,6 +142,8 @@ def _render_ui(self): def _render_left_panel(self): """Override the left panel to remove the base pause checkbox.""" + import newton as nt + imgui = self.ui.imgui # Use theme colors directly @@ -228,3 +253,176 @@ def _render_left_panel(self): imgui.end() return + + +class NewtonVisualizer(Visualizer): + """Newton OpenGL Visualizer for Isaac Lab. + + This visualizer uses Newton's OpenGL-based viewer to provide lightweight, + fast visualization of simulations. It includes IsaacLab-specific features + like training controls, rendering pause, and update frequency adjustment. + + Args: + cfg: Configuration for the Newton visualizer. + """ + + def __init__(self, cfg: NewtonVisualizerCfg): + super().__init__(cfg) + self.cfg: NewtonVisualizerCfg = cfg + self._viewer: NewtonViewerGL | None = None + self._sim_time: float = 0.0 + self._step_counter: int = 0 + self._model = None + self._state = None + + def initialize(self, scene) -> None: + """Initialize the Newton visualizer with the simulation scene. + + Args: + scene: Dictionary containing 'model' and 'state' for Newton simulation, + or the Newton Model object directly. + """ + if self._is_initialized: + return + + # Extract model and state from scene + if isinstance(scene, dict): + self._model = scene.get("model") + self._state = scene.get("state") + else: + # Assume scene is the Newton Model + self._model = scene + self._state = None + + if self._model is None: + raise ValueError("Newton visualizer requires a Newton Model to be provided in the scene") + + # Create the viewer + self._viewer = NewtonViewerGL( + width=self.cfg.window_width, + height=self.cfg.window_height, + train_mode=self.cfg.train_mode, + ) + + # Set the model + self._viewer.set_model(self._model) + + # Configure camera + self._viewer.camera.pos = wp.vec3(*self.cfg.camera_position) + self._viewer.up_axis = ["X", "Y", "Z"].index(self.cfg.up_axis) + self._viewer.scaling = 1.0 + self._viewer._paused = False + + # Configure visualization options + self._viewer.show_joints = self.cfg.show_joints + self._viewer.show_contacts = self.cfg.show_contacts + self._viewer.show_springs = self.cfg.show_springs + self._viewer.show_com = self.cfg.show_com + + # Configure rendering options + self._viewer.renderer.draw_shadows = self.cfg.enable_shadows + self._viewer.renderer.draw_sky = self.cfg.enable_sky + self._viewer.renderer.draw_wireframe = self.cfg.enable_wireframe + + # Configure colors + self._viewer.renderer.sky_upper = self.cfg.background_color + self._viewer.renderer.sky_lower = self.cfg.ground_color + self._viewer.renderer._light_color = self.cfg.light_color + + # Set update frequency + self._viewer.set_visualizer_update_frequency(self.cfg.update_frequency) + + self._is_initialized = True + + def step(self, dt: float) -> None: + """Update the visualizer for one simulation step. + + Args: + dt: Time step in seconds since last visualization update. + + Note: + Pause handling (training and rendering) is managed by SimulationContext. + This method only performs the actual rendering when called. + The visualizer MUST be called every frame to maintain proper ImGui state, + even if we skip rendering some frames based on update_frequency. + """ + if not self._is_initialized or self._is_closed or self._viewer is None: + return + + # Update simulation time + self._sim_time += dt + + # Render the current frame + # Note: We always call begin_frame/end_frame to maintain ImGui state + # The update frequency is handled internally by the viewer + try: + self._viewer.begin_frame(self._sim_time) + try: + if self._state is not None: + self._viewer.log_state(self._state) + finally: + # Always call end_frame if begin_frame succeeded + self._viewer.end_frame() + except Exception as e: + # Handle any rendering errors gracefully + # Frame lifecycle is now properly handled by try-finally + pass # Silently ignore to avoid log spam - the viewer will recover + + def close(self) -> None: + """Close the visualizer and clean up resources.""" + if self._is_closed: + return + + if self._viewer is not None: + try: + # Newton viewer doesn't have an explicit close method, + # but we can clean up our reference + self._viewer = None + except Exception as e: + print(f"[Warning] Error closing Newton visualizer: {e}") + + self._is_closed = True + + def is_running(self) -> bool: + """Check if the visualizer is still running. + + Returns: + True if the visualizer window is open and running. + """ + if not self._is_initialized or self._is_closed or self._viewer is None: + return False + + return self._viewer.is_running() + + def is_training_paused(self) -> bool: + """Check if training is paused by the visualizer. + + Returns: + True if the user has paused training via the visualizer controls. + """ + if not self._is_initialized or self._viewer is None: + return False + + return self._viewer.is_training_paused() + + def is_rendering_paused(self) -> bool: + """Check if rendering is paused by the visualizer. + + Returns: + True if rendering is paused via the visualizer controls. + """ + if not self._is_initialized or self._viewer is None: + return False + + return self._viewer.is_rendering_paused() + + def update_state(self, state) -> None: + """Update the simulation state for visualization. + + This method should be called before step() to provide the latest simulation state. + + Args: + state: The Newton State object to visualize. + """ + self._state = state + diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py new file mode 100644 index 00000000000..e118be751f1 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py @@ -0,0 +1,129 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Base class for visualizers.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .visualizer_cfg import VisualizerCfg + + +class Visualizer(ABC): + """Base class for all visualizer backends. + + Visualizers are used for debug visualization and monitoring during simulation, + separate from rendering for sensors/cameras. Each visualizer backend (Newton OpenGL, + Omniverse, Rerun) should inherit from this class and implement the required methods. + + The visualizer lifecycle follows this pattern: + 1. __init__: Create the visualizer with configuration + 2. initialize: Set up the visualizer with the simulation model/scene + 3. step: Update the visualizer each simulation step + 4. close: Clean up resources when done + + Args: + cfg: Configuration for the visualizer backend. + """ + + def __init__(self, cfg: VisualizerCfg): + """Initialize the visualizer with configuration. + + Args: + cfg: Configuration for the visualizer backend. + """ + self.cfg = cfg + self._is_initialized = False + self._is_closed = False + + @abstractmethod + def initialize(self, scene) -> None: + """Initialize the visualizer with the simulation scene. + + This method is called once after the simulation scene is created and before + the simulation starts. It should set up any necessary resources for visualization. + + Args: + scene: The simulation scene to visualize. This could be a Newton Model, + a USD stage, or other scene representation depending on the backend. + """ + pass + + @abstractmethod + def step(self, dt: float) -> None: + """Update the visualizer for one simulation step. + + This method is called each simulation step to update the visualization. + The frequency of calls is controlled by the update_frequency parameter in the config. + + Args: + dt: Time step in seconds since last visualization update. + """ + pass + + @abstractmethod + def close(self) -> None: + """Close the visualizer and clean up resources. + + This method is called when the simulation is ending or the visualizer + is no longer needed. It should release any resources held by the visualizer. + """ + pass + + @abstractmethod + def is_running(self) -> bool: + """Check if the visualizer is still running. + + Returns: + True if the visualizer is running and should continue receiving updates, + False if it has been closed (e.g., user closed the window). + """ + pass + + def is_training_paused(self) -> bool: + """Check if training/simulation is paused by the visualizer. + + Some visualizers (like Newton OpenGL) provide controls to pause the simulation + while keeping the visualizer running. This is useful for debugging. + + Returns: + True if the user has paused training/simulation, False otherwise. + Default implementation returns False (no pause support). + """ + return False + + def is_rendering_paused(self) -> bool: + """Check if rendering is paused by the visualizer. + + Some visualizers allow pausing rendering while keeping simulation running, + which can improve performance during training. + + Returns: + True if rendering is paused, False otherwise. + Default implementation returns False (no pause support). + """ + return False + + @property + def is_initialized(self) -> bool: + """Check if the visualizer has been initialized. + + Returns: + True if initialize() has been called successfully. + """ + return self._is_initialized + + @property + def is_closed(self) -> bool: + """Check if the visualizer has been closed. + + Returns: + True if close() has been called. + """ + return self._is_closed + diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py index 3842bda1471..755c3ad0acf 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py @@ -119,7 +119,6 @@ def parse_env_cfg( device: str = "cuda:0", num_envs: int | None = None, use_fabric: bool | None = None, - newton_visualizer: bool | None = None, ) -> ManagerBasedRLEnvCfg | DirectRLEnvCfg: """Parse configuration for an environment and override based on inputs. @@ -130,7 +129,6 @@ def parse_env_cfg( use_fabric: Whether to enable/disable fabric interface. If false, all read/write operations go through USD. This slows down the simulation but allows seeing the changes in the USD through the USD stage. Defaults to None, in which case it is left unchanged. - newton_visualizer: Whether to enable/disable Newton rendering. Defaults to None, in which case it is left unchanged. Returns: The parsed configuration object. @@ -138,6 +136,15 @@ def parse_env_cfg( Raises: RuntimeError: If the configuration for the task is not a class. We assume users always use a class for the environment configuration. + + Note: + To enable visualizers, set them directly in the environment configuration using + SimulationCfg.visualizers. For example: + + .. code-block:: python + + from isaaclab.sim.visualizers import NewtonVisualizerCfg + cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) """ # load the default configuration cfg = load_cfg_from_registry(task_name.split(":")[-1], "env_cfg_entry_point") @@ -155,9 +162,6 @@ def parse_env_cfg( # number of environments if num_envs is not None: cfg.scene.num_envs = num_envs - # newton rendering - if newton_visualizer is not None: - cfg.sim.enable_newton_rendering = newton_visualizer return cfg From d5fcc055ae3de5d486265133b12bd03402eb79b5 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Tue, 11 Nov 2025 22:54:44 -0800 Subject: [PATCH 3/9] add scene data provider abstraction layer --- scripts/environments/random_agent.py | 23 ++++- scripts/environments/zero_agent.py | 23 ++++- .../reinforcement_learning/rl_games/play.py | 23 ++++- .../reinforcement_learning/rl_games/train.py | 23 ++++- scripts/reinforcement_learning/rsl_rl/play.py | 37 ++++++-- .../reinforcement_learning/rsl_rl/train.py | 23 ++++- scripts/reinforcement_learning/sb3/play.py | 23 ++++- scripts/reinforcement_learning/sb3/train.py | 23 ++++- scripts/reinforcement_learning/skrl/play.py | 23 ++++- scripts/reinforcement_learning/skrl/train.py | 23 ++++- scripts/sim2sim_transfer/rsl_rl_transfer.py | 23 ++++- source/isaaclab/isaaclab/sim/__init__.py | 1 + .../isaaclab/sim/_impl/newton_manager_cfg.py | 4 - .../sim/scene_data_providers/__init__.py | 23 +++++ .../newton_scene_data_provider.py | 85 +++++++++++++++++++ .../scene_data_provider.py | 84 ++++++++++++++++++ .../isaaclab/sim/simulation_context.py | 62 ++++++++------ .../isaaclab/sim/visualizers/__init__.py | 65 +++++++++++++- .../sim/visualizers/newton_visualizer.py | 45 +++++----- .../sim/visualizers/newton_visualizer_cfg.py | 4 + .../isaaclab/sim/visualizers/ov_visualizer.py | 18 ++++ .../sim/visualizers/rerun_visualizer.py | 18 ++++ .../isaaclab/sim/visualizers/visualizer.py | 21 +++-- .../sim/visualizers/visualizer_cfg.py | 40 ++++++++- .../isaaclab_tasks/utils/parse_cfg.py | 9 -- 25 files changed, 651 insertions(+), 95 deletions(-) create mode 100644 source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py create mode 100644 source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py create mode 100644 source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py diff --git a/scripts/environments/random_agent.py b/scripts/environments/random_agent.py index 46b6cdc1468..653b2be5ccf 100644 --- a/scripts/environments/random_agent.py +++ b/scripts/environments/random_agent.py @@ -18,7 +18,12 @@ ) parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.") parser.add_argument("--task", type=str, default=None, help="Name of the task.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -52,8 +57,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # create environment env = gym.make(args_cli.task, cfg=env_cfg) diff --git a/scripts/environments/zero_agent.py b/scripts/environments/zero_agent.py index ab0301b8e7a..d2a3b64541e 100644 --- a/scripts/environments/zero_agent.py +++ b/scripts/environments/zero_agent.py @@ -18,7 +18,12 @@ ) parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.") parser.add_argument("--task", type=str, default=None, help="Name of the task.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -52,8 +57,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # create environment env = gym.make(args_cli.task, cfg=env_cfg) diff --git a/scripts/reinforcement_learning/rl_games/play.py b/scripts/reinforcement_learning/rl_games/play.py index ca807368406..8cb49c7098c 100644 --- a/scripts/reinforcement_learning/rl_games/play.py +++ b/scripts/reinforcement_learning/rl_games/play.py @@ -32,7 +32,12 @@ help="When no checkpoint provided, use the last saved model. Otherwise use the best saved model.", ) parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -84,8 +89,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True agent_cfg = load_cfg_from_registry(args_cli.task, "rl_games_cfg_entry_point") # specify directory for logging experiments diff --git a/scripts/reinforcement_learning/rl_games/train.py b/scripts/reinforcement_learning/rl_games/train.py index 9e0cfc67739..b12647306da 100644 --- a/scripts/reinforcement_learning/rl_games/train.py +++ b/scripts/reinforcement_learning/rl_games/train.py @@ -38,7 +38,12 @@ const=True, help="if toggled, this experiment will be tracked with Weights and Biases", ) -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -90,7 +95,21 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): # override configurations with non-hydra CLI arguments env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # randomly sample a seed if seed = -1 if args_cli.seed == -1: diff --git a/scripts/reinforcement_learning/rsl_rl/play.py b/scripts/reinforcement_learning/rsl_rl/play.py index 9de5852b233..f703524aec3 100644 --- a/scripts/reinforcement_learning/rsl_rl/play.py +++ b/scripts/reinforcement_learning/rsl_rl/play.py @@ -34,7 +34,12 @@ help="Use the pre-trained checkpoint from Nucleus.", ) parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append RSL-RL cli arguments cli_args.add_rsl_rl_args(parser) # append AppLauncher cli args @@ -96,7 +101,21 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun # note: certain randomizations occur in the environment initialization so we set the seed here env_cfg.seed = agent_cfg.seed env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # specify directory for logging experiments log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name) @@ -117,11 +136,15 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun # set the log directory for the environment (works for all environment types) env_cfg.log_dir = log_dir - # Set play mode for Newton viewer if using Newton visualizer - if args_cli.newton_visualizer: - # Set visualizer to play mode in Newton config - if hasattr(env_cfg.sim, "newton_cfg"): - env_cfg.sim.newton_cfg.visualizer_train_mode = False + # Set play mode for visualizers + if env_cfg.sim.visualizers is not None: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.train_mode = False + elif isinstance(env_cfg.sim.visualizers, NewtonVisualizerCfg): + env_cfg.sim.visualizers.train_mode = False else: # Create newton_cfg if it doesn't exist from isaaclab.sim._impl.newton_manager_cfg import NewtonCfg diff --git a/scripts/reinforcement_learning/rsl_rl/train.py b/scripts/reinforcement_learning/rsl_rl/train.py index 56e712cfe21..63e0dc1293c 100644 --- a/scripts/reinforcement_learning/rsl_rl/train.py +++ b/scripts/reinforcement_learning/rsl_rl/train.py @@ -32,7 +32,12 @@ "--distributed", action="store_true", default=False, help="Run training with multiple GPUs or nodes." ) parser.add_argument("--export_io_descriptors", action="store_true", default=False, help="Export IO descriptors.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append RSL-RL cli arguments cli_args.add_rsl_rl_args(parser) # append AppLauncher cli args @@ -119,7 +124,21 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun # note: certain randomizations occur in the environment initialization so we set the seed here env_cfg.seed = agent_cfg.seed env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # multi-gpu training configuration if args_cli.distributed: diff --git a/scripts/reinforcement_learning/sb3/play.py b/scripts/reinforcement_learning/sb3/play.py index 613ec46d7d8..09df5beba36 100644 --- a/scripts/reinforcement_learning/sb3/play.py +++ b/scripts/reinforcement_learning/sb3/play.py @@ -39,7 +39,12 @@ default=False, help="Use a slower SB3 wrapper but keep all the extra training info.", ) -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -87,8 +92,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True task_name = args_cli.task.split(":")[-1] train_task_name = task_name.replace("-Play", "") diff --git a/scripts/reinforcement_learning/sb3/train.py b/scripts/reinforcement_learning/sb3/train.py index fa34f951ffe..a29a29e66bf 100644 --- a/scripts/reinforcement_learning/sb3/train.py +++ b/scripts/reinforcement_learning/sb3/train.py @@ -32,7 +32,12 @@ default=False, help="Use a slower SB3 wrapper but keep all the extra training info.", ) -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # parse the arguments @@ -113,7 +118,21 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): # note: certain randomizations occur in the environment initialization so we set the seed here env_cfg.seed = agent_cfg["seed"] env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # directory for logging into run_info = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") diff --git a/scripts/reinforcement_learning/skrl/play.py b/scripts/reinforcement_learning/skrl/play.py index 252b3b3bc91..3cf9faba914 100644 --- a/scripts/reinforcement_learning/skrl/play.py +++ b/scripts/reinforcement_learning/skrl/play.py @@ -46,7 +46,12 @@ help="The RL algorithm used for training the skrl agent.", ) parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) @@ -116,8 +121,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True try: experiment_cfg = load_cfg_from_registry(task_name, f"skrl_{algorithm}_cfg_entry_point") except ValueError: diff --git a/scripts/reinforcement_learning/skrl/train.py b/scripts/reinforcement_learning/skrl/train.py index d59b2cc45d7..6ef1a8cb145 100644 --- a/scripts/reinforcement_learning/skrl/train.py +++ b/scripts/reinforcement_learning/skrl/train.py @@ -44,7 +44,12 @@ choices=["AMP", "PPO", "IPPO", "MAPPO"], help="The RL algorithm used for training the skrl agent.", ) -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) @@ -113,7 +118,21 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): # override configurations with non-hydra CLI arguments env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True # multi-gpu training config if args_cli.distributed: diff --git a/scripts/sim2sim_transfer/rsl_rl_transfer.py b/scripts/sim2sim_transfer/rsl_rl_transfer.py index d9b546ffd6f..20295d29eb5 100644 --- a/scripts/sim2sim_transfer/rsl_rl_transfer.py +++ b/scripts/sim2sim_transfer/rsl_rl_transfer.py @@ -29,7 +29,12 @@ help="Use the pre-trained checkpoint from Nucleus.", ) parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.") -parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.") +parser.add_argument( + "--visualize", + action="store_true", + default=False, + help="Launch visualizer(s). Uses visualizers defined in environment config, or defaults to Newton OpenGL if none configured.", +) # Joint ordering arguments parser.add_argument( "--policy_transfer_file", @@ -147,8 +152,22 @@ def main(): device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, - newton_visualizer=args_cli.newton_visualizer, ) + + # handle visualizer launch + if args_cli.visualize: + from isaaclab.sim.visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + else: + env_cfg.sim.visualizers.enabled = True agent_cfg: RslRlOnPolicyRunnerCfg = cli_args.parse_rsl_rl_cfg(task_name, args_cli) # specify directory for logging experiments diff --git a/source/isaaclab/isaaclab/sim/__init__.py b/source/isaaclab/isaaclab/sim/__init__.py index 6819cacf87b..b97f9f41382 100644 --- a/source/isaaclab/isaaclab/sim/__init__.py +++ b/source/isaaclab/isaaclab/sim/__init__.py @@ -27,6 +27,7 @@ """ from .converters import * # noqa: F401, F403 +from .scene_data_providers import NewtonSceneDataProvider, SceneDataProvider # noqa: F401, F403 from .schemas import * # noqa: F401, F403 from .simulation_cfg import RenderCfg, SimulationCfg # noqa: F401, F403 from .simulation_context import SimulationContext, build_simulation_context # noqa: F401, F403 diff --git a/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py b/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py index 02523192c0b..c79a9036250 100644 --- a/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py +++ b/source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py @@ -13,10 +13,6 @@ class NewtonCfg: """Configuration for Newton-related parameters. These parameters are used to configure the Newton physics simulation. - - Note: - Visualizer-related settings have been moved to NewtonVisualizerCfg in - isaaclab.sim.visualizers.newton_visualizer_cfg. """ num_substeps: int = 1 diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py b/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py new file mode 100644 index 00000000000..6cb665547a4 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-package for scene data provider implementations. + +This sub-package contains the abstract base class and concrete implementations +for providing scene data from various physics backends (Newton, PhysX, etc.) +to visualizers and renderers. + +The scene data provider abstraction allows visualizers and renderers to work +with any physics backend without directly coupling to specific backend implementations. +""" + +from .scene_data_provider import SceneDataProvider +from .newton_scene_data_provider import NewtonSceneDataProvider + +__all__ = [ + "SceneDataProvider", + "NewtonSceneDataProvider", +] + diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py new file mode 100644 index 00000000000..f6ee83a6520 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton-specific scene data provider implementation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .scene_data_provider import SceneDataProvider + +if TYPE_CHECKING: + from pxr import Usd + + +class NewtonSceneDataProvider(SceneDataProvider): + """Scene data provider for Newton physics backend. + + This class provides access to Newton physics data without directly exposing + the NewtonManager to downstream consumers. + """ + + def __init__(self, usd_stage: Usd.Stage | None = None): + """Initialize the Newton scene data provider. + + Args: + usd_stage: USD stage reference, if available. + """ + self._usd_stage = usd_stage + + def get_model(self) -> Any | None: + """Get the Newton physics model. + + Returns: + newton.Model instance, or None if not initialized. + """ + from isaaclab.sim._impl.newton_manager import NewtonManager + + return NewtonManager._model + + def get_state(self) -> Any | None: + """Get the current Newton physics state. + + Returns: + newton.State instance, or None if not initialized. + """ + from isaaclab.sim._impl.newton_manager import NewtonManager + + return NewtonManager._state_0 + + def get_usd_stage(self) -> Usd.Stage | None: + """Get the USD stage. + + Returns: + USD stage, or None if not available. + """ + return self._usd_stage + + def get_metadata(self) -> dict[str, Any]: + """Get Newton-specific metadata. + + Returns: + Dictionary containing: + - "physics_backend": "newton" + - "gravity_vector": tuple[float, float, float] + - "clone_physics_only": bool + """ + from isaaclab.sim._impl.newton_manager import NewtonManager + + return { + "physics_backend": "newton", + "gravity_vector": NewtonManager._gravity_vector, + "clone_physics_only": NewtonManager._clone_physics_only, + } + + def update_stage(self, usd_stage: Usd.Stage | None) -> None: + """Update the USD stage reference. + + Args: + usd_stage: New USD stage reference. + """ + self._usd_stage = usd_stage + diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py new file mode 100644 index 00000000000..2ab4c632e70 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Scene data provider abstraction for physics backends.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pxr import Usd + + +class SceneDataProvider(ABC): + """Abstract base class for providing scene data from physics backends to visualizers and renderers. + + This abstraction allows visualizers and renderers to work with any physics backend (Newton, PhysX, etc.) + without directly coupling to specific physics manager implementations. + """ + + @abstractmethod + def get_model(self) -> Any | None: + """Get the physics model. + + Returns: + Physics model object, or None if not available. The type depends on the backend + (e.g., newton.Model for Newton backend). + """ + pass + + @abstractmethod + def get_state(self) -> Any | None: + """Get the current physics state. + + Returns: + Physics state object, or None if not available. The type depends on the backend + (e.g., newton.State for Newton backend). + """ + pass + + @abstractmethod + def get_usd_stage(self) -> Usd.Stage | None: + """Get the USD stage. + + Returns: + USD stage, or None if not available. + """ + pass + + def get_metadata(self) -> dict[str, Any]: + """Get additional metadata about the scene. + + Returns: + Dictionary containing optional metadata such as: + - "physics_backend": str (e.g., "newton", "physx") + - "num_envs": int + - "device": str + - etc. + """ + return {} + + def get_scene_data(self) -> dict[str, Any]: + """Get all available scene data as a dictionary. + + This is a convenience method that collects all scene data into a single dict. + Individual visualizers can extract what they need. + + Returns: + Dictionary containing all available scene data: + - "model": Physics model + - "state": Physics state + - "usd_stage": USD stage + - "metadata": Additional metadata + """ + return { + "model": self.get_model(), + "state": self.get_state(), + "usd_stage": self.get_usd_stage(), + "metadata": self.get_metadata(), + } + diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 9664c44da99..c4b6621d305 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -30,12 +30,13 @@ from pxr import Usd from isaaclab.sim._impl.newton_manager import NewtonManager +from isaaclab.sim.scene_data_providers import NewtonSceneDataProvider, SceneDataProvider from isaaclab.sim.utils import create_new_stage_in_memory, use_stage from .simulation_cfg import SimulationCfg from .spawners import DomeLightCfg, GroundPlaneCfg from .utils import bind_physics_material -from .visualizers import Visualizer, NewtonVisualizer, NewtonVisualizerCfg +from .visualizers import Visualizer class SimulationContext(_SimulationContext): @@ -256,6 +257,9 @@ def __init__(self, cfg: SimulationCfg | None = None): # initialize visualizers self._visualizers: list[Visualizer] = [] self._visualizer_step_counter = 0 + + # Scene data provider for visualizers and renderers (initialized after stage is available) + self._scene_provider: SceneDataProvider | None = None # flag for skipping prim deletion callback # when stage in memory is attached @@ -300,6 +304,9 @@ def __init__(self, cfg: SimulationCfg | None = None): NewtonManager._clone_physics_only = ( self.render_mode == self.RenderMode.NO_GUI_OR_RENDERING or self.render_mode == self.RenderMode.NO_RENDERING ) + + # Initialize scene data provider now that stage is available + self._scene_provider = NewtonSceneDataProvider(self.stage) def _apply_physics_settings(self): """Sets various carb physics settings.""" @@ -563,32 +570,38 @@ def initialize_visualizers(self) -> None: # Create and initialize each visualizer for viz_cfg in visualizer_cfgs: try: - # Create visualizer instance based on config type - if isinstance(viz_cfg, NewtonVisualizerCfg): - visualizer = NewtonVisualizer(viz_cfg) - else: - # Skip unsupported visualizer types for now + # Use factory pattern to create visualizer from config + visualizer = viz_cfg.create_visualizer() + + # Get initial scene data from the scene provider + if self._scene_provider is None: omni.log.warn( - f"Visualizer type '{type(viz_cfg).__name__}' is not yet implemented. Skipping." + f"Scene provider not initialized yet for visualizer '{viz_cfg.visualizer_type}'. " + "Visualizer will be initialized later." ) continue - - # Initialize with Newton model if available - if NewtonManager._model is not None: - scene = { - "model": NewtonManager._model, - "state": NewtonManager._state_0, - } - visualizer.initialize(scene) + + scene_data = self._scene_provider.get_scene_data() + + # Check if we have the minimum required data (physics model) + if scene_data.get("model") is not None: + visualizer.initialize(scene_data) self._visualizers.append(visualizer) - omni.log.info(f"Initialized visualizer: {type(visualizer).__name__}") + omni.log.info( + f"Initialized visualizer: {type(visualizer).__name__} " + f"(type: {viz_cfg.visualizer_type})" + ) else: omni.log.warn( - "Newton model not available yet. Visualizer will be initialized later." + f"Physics model not available yet for visualizer '{viz_cfg.visualizer_type}'. " + "Visualizer will be initialized later." ) except Exception as e: - omni.log.error(f"Failed to initialize visualizer '{type(viz_cfg).__name__}': {e}") + omni.log.error( + f"Failed to initialize visualizer '{viz_cfg.visualizer_type}' " + f"({type(viz_cfg).__name__}): {e}" + ) def step_visualizers(self, dt: float) -> None: """Update all active visualizers. @@ -616,20 +629,15 @@ def step_visualizers(self, dt: float) -> None: # Handle training pause - block until resumed while visualizer.is_training_paused() and visualizer.is_running(): - if isinstance(visualizer, NewtonVisualizer): - # Update state before rendering during pause - visualizer.update_state(NewtonManager._state_0) - visualizer.step(0.0) # Step with 0 dt during pause + # Step with 0 dt during pause, pass scene provider for state updates + visualizer.step(0.0, self._scene_provider) # Skip rendering if visualizer has rendering paused if visualizer.is_rendering_paused(): continue - # Normal step: update state and visualizer - if isinstance(visualizer, NewtonVisualizer): - visualizer.update_state(NewtonManager._state_0) - - visualizer.step(dt) + # Normal step: pass dt and scene provider so visualizer can pull latest state + visualizer.step(dt, self._scene_provider) except Exception as e: omni.log.error(f"Error stepping visualizer '{type(visualizer).__name__}': {e}") diff --git a/source/isaaclab/isaaclab/sim/visualizers/__init__.py b/source/isaaclab/isaaclab/sim/visualizers/__init__.py index 0814b88a412..2a7e8ff9ae8 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/__init__.py +++ b/source/isaaclab/isaaclab/sim/visualizers/__init__.py @@ -13,15 +13,36 @@ - Newton OpenGL Visualizer: Lightweight OpenGL-based visualizer - Omniverse Visualizer: High-fidelity Omniverse-based visualizer (coming soon) - Rerun Visualizer: Web-based visualizer using the rerun library (coming soon) + +Visualizer Registry +------------------- +This module uses a registry pattern to decouple visualizer instantiation from specific types. +Visualizer implementations can register themselves using the `register_visualizer` decorator, +and configs can create visualizers via the `create_visualizer()` factory method. """ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from typing import Type + +# Import base classes first from .visualizer import Visualizer from .visualizer_cfg import VisualizerCfg -from .newton_visualizer import NewtonVisualizer + +# Import config classes (no circular dependency) from .newton_visualizer_cfg import NewtonVisualizerCfg from .ov_visualizer_cfg import OVVisualizerCfg from .rerun_visualizer_cfg import RerunVisualizerCfg +# Import visualizer implementations +from .newton_visualizer import NewtonVisualizer + +# Global registry for visualizer types (defined after Visualizer import) +_VISUALIZER_REGISTRY: dict[str, Any] = {} + __all__ = [ "Visualizer", "VisualizerCfg", @@ -29,6 +50,48 @@ "NewtonVisualizerCfg", "OVVisualizerCfg", "RerunVisualizerCfg", + "register_visualizer", + "get_visualizer_class", ] +def register_visualizer(name: str): + """Decorator to register a visualizer class with the given name. + + This allows visualizer configs to create instances without hard-coded type checking. + + Args: + name: Unique identifier for this visualizer type (e.g., "newton", "rerun", "omniverse"). + + Example: + >>> @register_visualizer("newton") + >>> class NewtonVisualizer(Visualizer): + >>> pass + """ + + def decorator(cls: Type[Visualizer]) -> Type[Visualizer]: + if name in _VISUALIZER_REGISTRY: + raise ValueError(f"Visualizer '{name}' is already registered!") + _VISUALIZER_REGISTRY[name] = cls + return cls + + return decorator + + +def get_visualizer_class(name: str) -> Type[Visualizer] | None: + """Get a registered visualizer class by name. + + Args: + name: Visualizer type name. + + Returns: + Visualizer class, or None if not found. + """ + return _VISUALIZER_REGISTRY.get(name) + + +# Register built-in visualizers +# Note: Registration happens here to avoid circular imports +_VISUALIZER_REGISTRY["newton"] = NewtonVisualizer + + diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index ac9ed2c217c..004b2ccf574 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -7,12 +7,17 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any + import warp as wp from newton.viewer import ViewerGL from .newton_visualizer_cfg import NewtonVisualizerCfg from .visualizer import Visualizer +if TYPE_CHECKING: + from isaaclab.sim.scene_data_providers import SceneDataProvider + class NewtonViewerGL(ViewerGL): """Training-aware viewer that adds a separate pause for simulation/training. @@ -41,12 +46,8 @@ def is_training_paused(self) -> bool: """Check if training is paused.""" return self._paused_training - def is_paused(self) -> bool: - """Check if rendering is paused.""" - return self._paused_rendering - def is_rendering_paused(self) -> bool: - """Check if rendering is paused (alias for is_paused for consistency).""" + """Check if rendering is paused.""" return self._paused_rendering def set_visualizer_update_frequency(self, frequency: int) -> None: @@ -262,6 +263,9 @@ class NewtonVisualizer(Visualizer): fast visualization of simulations. It includes IsaacLab-specific features like training controls, rendering pause, and update frequency adjustment. + This class is registered with the visualizer registry as "newton" and can be + instantiated via NewtonVisualizerCfg.create_visualizer(). + Args: cfg: Configuration for the Newton visualizer. """ @@ -275,27 +279,24 @@ def __init__(self, cfg: NewtonVisualizerCfg): self._model = None self._state = None - def initialize(self, scene) -> None: + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: """Initialize the Newton visualizer with the simulation scene. Args: - scene: Dictionary containing 'model' and 'state' for Newton simulation, - or the Newton Model object directly. + scene_data: Optional dictionary containing initial scene data. Expected keys: + - "model": Newton Model object (required) + - "state": Newton State object (optional) """ if self._is_initialized: return - # Extract model and state from scene - if isinstance(scene, dict): - self._model = scene.get("model") - self._state = scene.get("state") - else: - # Assume scene is the Newton Model - self._model = scene - self._state = None - + # Extract model and state from scene data + if scene_data is not None: + self._model = scene_data.get("model") + self._state = scene_data.get("state") + if self._model is None: - raise ValueError("Newton visualizer requires a Newton Model to be provided in the scene") + raise ValueError("Newton visualizer requires a Newton Model to be provided in scene_data['model']") # Create the viewer self._viewer = NewtonViewerGL( @@ -334,11 +335,13 @@ def initialize(self, scene) -> None: self._is_initialized = True - def step(self, dt: float) -> None: + def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: """Update the visualizer for one simulation step. Args: dt: Time step in seconds since last visualization update. + scene_provider: Provider for accessing current scene data. The visualizer + will pull the latest Newton state from this provider. Note: Pause handling (training and rendering) is managed by SimulationContext. @@ -352,6 +355,10 @@ def step(self, dt: float) -> None: # Update simulation time self._sim_time += dt + # Get the latest state from the scene provider + if scene_provider is not None: + self._state = scene_provider.get_state() + # Render the current frame # Note: We always call begin_frame/end_frame to maintain ImGui state # The update frequency is handled internally by the viewer diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py index e9304f9eca9..dadf5ad88ef 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py @@ -33,6 +33,10 @@ class NewtonVisualizerCfg(VisualizerCfg): not visual shapes. """ + # Visualizer type identifier + visualizer_type: str = "newton" + """Type identifier for Newton visualizer. Used by the factory pattern.""" + # Override defaults for Newton visualizer camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) """Initial position of the camera. Default is (10.0, 0.0, 3.0).""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py index e69de29bb2d..095a8b7de49 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Omniverse Visualizer implementation.""" + +from __future__ import annotations + +from .ov_visualizer_cfg import OVVisualizerCfg +from .visualizer import Visualizer + + +class OmniverseVisualizer(Visualizer): + """Omniverse Visualizer implementation.""" + def __init__(self, cfg: OVVisualizerCfg): + super().__init__(cfg) + # stub for now \ No newline at end of file diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py index e69de29bb2d..7758a938d6e 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Omniverse Visualizer implementation.""" + +from __future__ import annotations + +from .ov_visualizer_cfg import RerunVisualizerCfg +from .visualizer import Visualizer + + +class RerunVisualizer(Visualizer): + """Omniverse Visualizer implementation.""" + def __init__(self, cfg: RerunVisualizerCfg): + super().__init__(cfg) + # stub for now \ No newline at end of file diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py index e118be751f1..271da4da78a 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py @@ -8,9 +8,11 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from isaaclab.sim.scene_data_providers import SceneDataProvider + from .visualizer_cfg import VisualizerCfg @@ -42,27 +44,34 @@ def __init__(self, cfg: VisualizerCfg): self._is_closed = False @abstractmethod - def initialize(self, scene) -> None: + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: """Initialize the visualizer with the simulation scene. This method is called once after the simulation scene is created and before the simulation starts. It should set up any necessary resources for visualization. Args: - scene: The simulation scene to visualize. This could be a Newton Model, - a USD stage, or other scene representation depending on the backend. + scene_data: Optional dictionary containing initial scene data. The contents + depend on what's available at initialization time. May include: + - "model": Physics model object + - "state": Initial physics state + - "usd_stage": USD stage + The visualizer should handle None or missing keys gracefully. """ pass @abstractmethod - def step(self, dt: float) -> None: + def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: """Update the visualizer for one simulation step. This method is called each simulation step to update the visualization. - The frequency of calls is controlled by the update_frequency parameter in the config. + The visualizer should pull any needed data from the scene_provider. Args: dt: Time step in seconds since last visualization update. + scene_provider: Provider for accessing current scene data (physics state, USD stage, etc.). + Visualizers should query this for updated data rather than directly + accessing physics managers. May be None if no scene data is available yet. """ pass diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py index fce04f0c786..76d54673105 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py @@ -5,10 +5,15 @@ """Base configuration for visualizers.""" -from typing import Literal +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal from isaaclab.utils import configclass +if TYPE_CHECKING: + from .visualizer import Visualizer + @configclass class VisualizerCfg: @@ -22,6 +27,13 @@ class VisualizerCfg: configuration parameters. """ + visualizer_type: str = "base" + """Type identifier for this visualizer (e.g., 'newton', 'rerun', 'omniverse'). + + This is used by the factory pattern to instantiate the correct visualizer class. + Subclasses should override this with their specific type identifier. + """ + enabled: bool = False """Whether the visualizer is enabled. Default is False.""" @@ -84,6 +96,30 @@ def get_visualizer_type(self) -> str: Returns: String identifier for the visualizer type. """ - return self.__class__.__name__.replace("VisualizerCfg", "").replace("Cfg", "") + return self.visualizer_type + + def create_visualizer(self) -> Visualizer: + """Factory method to create a visualizer instance from this configuration. + + This method uses the visualizer registry to instantiate the appropriate + visualizer class based on the `visualizer_type` field. + + Returns: + Visualizer instance configured with this config. + + Raises: + ValueError: If the visualizer type is not registered. + """ + # Import here to avoid circular imports + from . import get_visualizer_class + + visualizer_class = get_visualizer_class(self.visualizer_type) + if visualizer_class is None: + raise ValueError( + f"Visualizer type '{self.visualizer_type}' is not registered. " + f"Make sure the visualizer module is imported." + ) + + return visualizer_class(self) diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py index 755c3ad0acf..a07befbd3f9 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py @@ -136,15 +136,6 @@ def parse_env_cfg( Raises: RuntimeError: If the configuration for the task is not a class. We assume users always use a class for the environment configuration. - - Note: - To enable visualizers, set them directly in the environment configuration using - SimulationCfg.visualizers. For example: - - .. code-block:: python - - from isaaclab.sim.visualizers import NewtonVisualizerCfg - cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) """ # load the default configuration cfg = load_cfg_from_registry(task_name.split(":")[-1], "env_cfg_entry_point") From a1e8ec221a855436d68bf3d621f3f988178646e9 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Tue, 11 Nov 2025 23:13:17 -0800 Subject: [PATCH 4/9] bug fix --- .../newton_scene_data_provider.py | 2 ++ .../isaaclab/sim/visualizers/newton_visualizer.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py index f6ee83a6520..1966ba7a33b 100644 --- a/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py @@ -64,6 +64,7 @@ def get_metadata(self) -> dict[str, Any]: Returns: Dictionary containing: - "physics_backend": "newton" + - "num_envs": int - "gravity_vector": tuple[float, float, float] - "clone_physics_only": bool """ @@ -71,6 +72,7 @@ def get_metadata(self) -> dict[str, Any]: return { "physics_backend": "newton", + "num_envs": NewtonManager._num_envs if NewtonManager._num_envs is not None else 0, "gravity_vector": NewtonManager._gravity_vector, "clone_physics_only": NewtonManager._clone_physics_only, } diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index 004b2ccf574..52368afa4fd 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -29,13 +29,14 @@ class NewtonViewerGL(ViewerGL): The training pause can be toggled from the UI via a button and optionally via the 'T' key. """ - def __init__(self, *args, train_mode: bool = True, **kwargs): + def __init__(self, *args, train_mode: bool = True, metadata: dict | None = None, **kwargs): super().__init__(*args, **kwargs) self._paused_training: bool = False self._paused_rendering: bool = False self._fallback_draw_controls: bool = False self._is_train_mode: bool = train_mode self._visualizer_update_frequency: int = 1 + self._metadata = metadata or {} try: self.register_ui_callback(self._render_training_controls, position="side") @@ -174,7 +175,8 @@ def _render_left_panel(self): imgui.set_next_item_open(True, imgui.Cond_.appearing) if imgui.collapsing_header("Model Information", flags=header_flags): imgui.separator() - imgui.text(f"Environments: {self.model.num_envs}") + num_envs = self._metadata.get("num_envs", 0) + imgui.text(f"Environments: {num_envs}") axis_names = ["X", "Y", "Z"] imgui.text(f"Up Axis: {axis_names[self.model.up_axis]}") gravity = wp.to_torch(self.model.gravity)[0] @@ -290,19 +292,22 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: if self._is_initialized: return - # Extract model and state from scene data + # Extract model, state, and metadata from scene data + metadata = {} if scene_data is not None: self._model = scene_data.get("model") self._state = scene_data.get("state") + metadata = scene_data.get("metadata", {}) if self._model is None: raise ValueError("Newton visualizer requires a Newton Model to be provided in scene_data['model']") - # Create the viewer + # Create the viewer with metadata self._viewer = NewtonViewerGL( width=self.cfg.window_width, height=self.cfg.window_height, train_mode=self.cfg.train_mode, + metadata=metadata, ) # Set the model From c927485795e89a9332d9dc40a248452b5227b8a4 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Tue, 11 Nov 2025 23:33:05 -0800 Subject: [PATCH 5/9] clean --- scripts/environments/random_agent.py | 16 ++------- scripts/environments/zero_agent.py | 16 ++------- .../reinforcement_learning/rl_games/play.py | 22 +----------- .../reinforcement_learning/rl_games/train.py | 16 ++------- scripts/reinforcement_learning/rsl_rl/play.py | 33 ++--------------- .../reinforcement_learning/rsl_rl/train.py | 16 ++------- scripts/reinforcement_learning/sb3/play.py | 22 +----------- scripts/reinforcement_learning/sb3/train.py | 16 ++------- scripts/reinforcement_learning/skrl/play.py | 22 +----------- scripts/reinforcement_learning/skrl/train.py | 16 ++------- scripts/sim2sim_transfer/rsl_rl_transfer.py | 16 ++------- source/isaaclab/isaaclab/sim/__init__.py | 2 +- .../isaaclab/sim/simulation_context.py | 36 +++++++++++++++++++ .../isaaclab_tasks/utils/parse_cfg.py | 10 ++++++ 14 files changed, 74 insertions(+), 185 deletions(-) diff --git a/scripts/environments/random_agent.py b/scripts/environments/random_agent.py index 653b2be5ccf..dfb1302d3d5 100644 --- a/scripts/environments/random_agent.py +++ b/scripts/environments/random_agent.py @@ -59,20 +59,10 @@ def main(): use_fabric=not args_cli.disable_fabric, ) - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # create environment env = gym.make(args_cli.task, cfg=env_cfg) diff --git a/scripts/environments/zero_agent.py b/scripts/environments/zero_agent.py index d2a3b64541e..5b690c7caaf 100644 --- a/scripts/environments/zero_agent.py +++ b/scripts/environments/zero_agent.py @@ -59,20 +59,10 @@ def main(): use_fabric=not args_cli.disable_fabric, ) - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # create environment env = gym.make(args_cli.task, cfg=env_cfg) diff --git a/scripts/reinforcement_learning/rl_games/play.py b/scripts/reinforcement_learning/rl_games/play.py index 8cb49c7098c..189f89349e7 100644 --- a/scripts/reinforcement_learning/rl_games/play.py +++ b/scripts/reinforcement_learning/rl_games/play.py @@ -84,27 +84,7 @@ def main(): """Play with RL-Games agent.""" task_name = args_cli.task.split(":")[-1] # parse env configuration - env_cfg = parse_env_cfg( - args_cli.task, - device=args_cli.device, - num_envs=args_cli.num_envs, - use_fabric=not args_cli.disable_fabric, - ) - - # handle visualizer launch - if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + env_cfg = parse_env_cfg(args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, visualize=args_cli.visualize, train_mode=False) agent_cfg = load_cfg_from_registry(args_cli.task, "rl_games_cfg_entry_point") # specify directory for logging experiments diff --git a/scripts/reinforcement_learning/rl_games/train.py b/scripts/reinforcement_learning/rl_games/train.py index b12647306da..80a077eefa8 100644 --- a/scripts/reinforcement_learning/rl_games/train.py +++ b/scripts/reinforcement_learning/rl_games/train.py @@ -96,20 +96,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # randomly sample a seed if seed = -1 if args_cli.seed == -1: diff --git a/scripts/reinforcement_learning/rsl_rl/play.py b/scripts/reinforcement_learning/rsl_rl/play.py index f703524aec3..cf02ba60ea6 100644 --- a/scripts/reinforcement_learning/rsl_rl/play.py +++ b/scripts/reinforcement_learning/rsl_rl/play.py @@ -102,20 +102,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun env_cfg.seed = agent_cfg.seed env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg, train_mode=False) # specify directory for logging experiments log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name) @@ -136,23 +126,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun # set the log directory for the environment (works for all environment types) env_cfg.log_dir = log_dir - # Set play mode for visualizers - if env_cfg.sim.visualizers is not None: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.train_mode = False - elif isinstance(env_cfg.sim.visualizers, NewtonVisualizerCfg): - env_cfg.sim.visualizers.train_mode = False - else: - # Create newton_cfg if it doesn't exist - from isaaclab.sim._impl.newton_manager_cfg import NewtonCfg - - newton_cfg = NewtonCfg() - newton_cfg.visualizer_train_mode = False - env_cfg.sim.newton_cfg = newton_cfg - # create isaac environment env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None) diff --git a/scripts/reinforcement_learning/rsl_rl/train.py b/scripts/reinforcement_learning/rsl_rl/train.py index 63e0dc1293c..380c1ae0b91 100644 --- a/scripts/reinforcement_learning/rsl_rl/train.py +++ b/scripts/reinforcement_learning/rsl_rl/train.py @@ -125,20 +125,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun env_cfg.seed = agent_cfg.seed env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # multi-gpu training configuration if args_cli.distributed: diff --git a/scripts/reinforcement_learning/sb3/play.py b/scripts/reinforcement_learning/sb3/play.py index 09df5beba36..cb3473a29e3 100644 --- a/scripts/reinforcement_learning/sb3/play.py +++ b/scripts/reinforcement_learning/sb3/play.py @@ -87,27 +87,7 @@ def main(): """Play with stable-baselines agent.""" # parse configuration - env_cfg = parse_env_cfg( - args_cli.task, - device=args_cli.device, - num_envs=args_cli.num_envs, - use_fabric=not args_cli.disable_fabric, - ) - - # handle visualizer launch - if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + env_cfg = parse_env_cfg(args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, visualize=args_cli.visualize, train_mode=False) task_name = args_cli.task.split(":")[-1] train_task_name = task_name.replace("-Play", "") diff --git a/scripts/reinforcement_learning/sb3/train.py b/scripts/reinforcement_learning/sb3/train.py index a29a29e66bf..03ae5246269 100644 --- a/scripts/reinforcement_learning/sb3/train.py +++ b/scripts/reinforcement_learning/sb3/train.py @@ -119,20 +119,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): env_cfg.seed = agent_cfg["seed"] env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # directory for logging into run_info = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") diff --git a/scripts/reinforcement_learning/skrl/play.py b/scripts/reinforcement_learning/skrl/play.py index 3cf9faba914..54b93ec97f4 100644 --- a/scripts/reinforcement_learning/skrl/play.py +++ b/scripts/reinforcement_learning/skrl/play.py @@ -116,27 +116,7 @@ def main(): task_name = args_cli.task.split(":")[-1] # parse configuration - env_cfg = parse_env_cfg( - args_cli.task, - device=args_cli.device, - num_envs=args_cli.num_envs, - use_fabric=not args_cli.disable_fabric, - ) - - # handle visualizer launch - if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + env_cfg = parse_env_cfg(args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric, visualize=args_cli.visualize, train_mode=False) try: experiment_cfg = load_cfg_from_registry(task_name, f"skrl_{algorithm}_cfg_entry_point") except ValueError: diff --git a/scripts/reinforcement_learning/skrl/train.py b/scripts/reinforcement_learning/skrl/train.py index 6ef1a8cb145..add4686603f 100644 --- a/scripts/reinforcement_learning/skrl/train.py +++ b/scripts/reinforcement_learning/skrl/train.py @@ -119,20 +119,10 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict): env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) # multi-gpu training config if args_cli.distributed: diff --git a/scripts/sim2sim_transfer/rsl_rl_transfer.py b/scripts/sim2sim_transfer/rsl_rl_transfer.py index 20295d29eb5..63f92c2e3e7 100644 --- a/scripts/sim2sim_transfer/rsl_rl_transfer.py +++ b/scripts/sim2sim_transfer/rsl_rl_transfer.py @@ -154,20 +154,10 @@ def main(): use_fabric=not args_cli.disable_fabric, ) - # handle visualizer launch + # enable visualizers if requested if args_cli.visualize: - from isaaclab.sim.visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True) - else: - # Enable configured visualizer(s) - if isinstance(env_cfg.sim.visualizers, list): - for viz_cfg in env_cfg.sim.visualizers: - viz_cfg.enabled = True - else: - env_cfg.sim.visualizers.enabled = True + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(env_cfg) agent_cfg: RslRlOnPolicyRunnerCfg = cli_args.parse_rsl_rl_cfg(task_name, args_cli) # specify directory for logging experiments diff --git a/source/isaaclab/isaaclab/sim/__init__.py b/source/isaaclab/isaaclab/sim/__init__.py index b97f9f41382..7186730171c 100644 --- a/source/isaaclab/isaaclab/sim/__init__.py +++ b/source/isaaclab/isaaclab/sim/__init__.py @@ -30,6 +30,6 @@ from .scene_data_providers import NewtonSceneDataProvider, SceneDataProvider # noqa: F401, F403 from .schemas import * # noqa: F401, F403 from .simulation_cfg import RenderCfg, SimulationCfg # noqa: F401, F403 -from .simulation_context import SimulationContext, build_simulation_context # noqa: F401, F403 +from .simulation_context import SimulationContext, build_simulation_context, enable_visualizers # noqa: F401, F403 from .spawners import * # noqa: F401, F403 from .utils import * # noqa: F401, F403 diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index c4b6621d305..3904906911b 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -1033,3 +1033,39 @@ def build_simulation_context( exception_to_raise = builtins.ISAACLAB_CALLBACK_EXCEPTION builtins.ISAACLAB_CALLBACK_EXCEPTION = None raise exception_to_raise + + +def enable_visualizers(env_cfg, train_mode: bool = True) -> None: + """Enable visualizers for an environment configuration. + + If no visualizers are configured, defaults to Newton OpenGL visualizer. + If visualizers are already configured, enables them. + + This is a utility function for use in scripts that want to enable visualization + based on command-line arguments. + + Args: + env_cfg: Environment configuration (DirectRLEnvCfg or ManagerBasedRLEnvCfg) to modify. + train_mode: Whether to run visualizers in training mode (True) or play/inference mode (False). + Default is True. + + Example: + >>> import isaaclab.sim as sim_utils + >>> if args_cli.visualize: + ... sim_utils.enable_visualizers(env_cfg) # For training + ... sim_utils.enable_visualizers(env_cfg, train_mode=False) # For play/inference + """ + from .visualizers import NewtonVisualizerCfg + + if env_cfg.sim.visualizers is None: + # No visualizers in config - use default Newton visualizer + env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True, train_mode=train_mode) + else: + # Enable configured visualizer(s) + if isinstance(env_cfg.sim.visualizers, list): + for viz_cfg in env_cfg.sim.visualizers: + viz_cfg.enabled = True + viz_cfg.train_mode = train_mode + else: + env_cfg.sim.visualizers.enabled = True + env_cfg.sim.visualizers.train_mode = train_mode diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py index a07befbd3f9..b8f3357e7ea 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py @@ -119,6 +119,8 @@ def parse_env_cfg( device: str = "cuda:0", num_envs: int | None = None, use_fabric: bool | None = None, + visualize: bool = False, + train_mode: bool = True, ) -> ManagerBasedRLEnvCfg | DirectRLEnvCfg: """Parse configuration for an environment and override based on inputs. @@ -129,6 +131,10 @@ def parse_env_cfg( use_fabric: Whether to enable/disable fabric interface. If false, all read/write operations go through USD. This slows down the simulation but allows seeing the changes in the USD through the USD stage. Defaults to None, in which case it is left unchanged. + visualize: Whether to launch visualizer(s). Uses visualizers defined in environment config, or defaults + to Newton OpenGL if none configured. Defaults to False. + train_mode: Whether to run visualizers in training mode (True) or play/inference mode (False). + Only applies if visualize is True. Defaults to True. Returns: The parsed configuration object. @@ -153,6 +159,10 @@ def parse_env_cfg( # number of environments if num_envs is not None: cfg.scene.num_envs = num_envs + # visualizer configuration + if visualize: + import isaaclab.sim as sim_utils + sim_utils.enable_visualizers(cfg, train_mode=train_mode) return cfg From 1b24ba5682dc757245af150bb8958bc39ece6390 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Thu, 13 Nov 2025 14:49:51 -0800 Subject: [PATCH 6/9] wip --- .../isaaclab/isaaclab/sim/simulation_cfg.py | 7 +- .../isaaclab/sim/simulation_context.py | 28 +- .../isaaclab/sim/visualizers/__init__.py | 6 +- .../sim/visualizers/newton_visualizer.py | 35 +- .../isaaclab/sim/visualizers/ov_visualizer.py | 302 +++++++++++++++++- .../sim/visualizers/ov_visualizer_cfg.py | 166 ++-------- .../isaaclab/sim/visualizers/visualizer.py | 25 ++ 7 files changed, 402 insertions(+), 167 deletions(-) diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index 4e46f75fcff..a3ff7250f4f 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -15,7 +15,7 @@ from ._impl.newton_manager_cfg import NewtonCfg from .spawners.materials import RigidBodyMaterialCfg -from .visualizers import VisualizerCfg +from .visualizers import VisualizerCfg, NewtonVisualizerCfg, OVVisualizerCfg, RerunVisualizerCfg @configclass @@ -197,8 +197,9 @@ class SimulationCfg: render: RenderCfg = RenderCfg() """Render settings. Default is RenderCfg().""" - visualizers: list[VisualizerCfg] | VisualizerCfg | None = None - """Visualizer settings. Default is None (no visualizer). + # visualizers: list[VisualizerCfg] | VisualizerCfg | None = NewtonVisualizerCfg(enabled=True) + visualizers: list[VisualizerCfg] | VisualizerCfg | None = OVVisualizerCfg(enabled=True) + """Visualizer settings. Default is Newton (no visualizer). This field supports multiple visualizer backends for debug visualization and monitoring during simulation. It accepts: diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 3904906911b..5eb886ba60c 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -583,19 +583,14 @@ def initialize_visualizers(self) -> None: scene_data = self._scene_provider.get_scene_data() - # Check if we have the minimum required data (physics model) - if scene_data.get("model") is not None: - visualizer.initialize(scene_data) - self._visualizers.append(visualizer) - omni.log.info( - f"Initialized visualizer: {type(visualizer).__name__} " - f"(type: {viz_cfg.visualizer_type})" - ) - else: - omni.log.warn( - f"Physics model not available yet for visualizer '{viz_cfg.visualizer_type}'. " - "Visualizer will be initialized later." - ) + # Let each visualizer validate its own requirements + # (e.g., NewtonVisualizer needs Newton model, OVVisualizer needs USD stage) + visualizer.initialize(scene_data) + self._visualizers.append(visualizer) + omni.log.info( + f"Initialized visualizer: {type(visualizer).__name__} " + f"(type: {viz_cfg.visualizer_type})" + ) except Exception as e: omni.log.error( @@ -1055,12 +1050,7 @@ def enable_visualizers(env_cfg, train_mode: bool = True) -> None: ... sim_utils.enable_visualizers(env_cfg) # For training ... sim_utils.enable_visualizers(env_cfg, train_mode=False) # For play/inference """ - from .visualizers import NewtonVisualizerCfg - - if env_cfg.sim.visualizers is None: - # No visualizers in config - use default Newton visualizer - env_cfg.sim.visualizers = NewtonVisualizerCfg(enabled=True, train_mode=train_mode) - else: + if env_cfg.sim.visualizers : # Enable configured visualizer(s) if isinstance(env_cfg.sim.visualizers, list): for viz_cfg in env_cfg.sim.visualizers: diff --git a/source/isaaclab/isaaclab/sim/visualizers/__init__.py b/source/isaaclab/isaaclab/sim/visualizers/__init__.py index 2a7e8ff9ae8..cee4eac6df9 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/__init__.py +++ b/source/isaaclab/isaaclab/sim/visualizers/__init__.py @@ -11,7 +11,7 @@ Supported visualizers: - Newton OpenGL Visualizer: Lightweight OpenGL-based visualizer -- Omniverse Visualizer: High-fidelity Omniverse-based visualizer (coming soon) +- Omniverse Visualizer: High-fidelity Omniverse-based visualizer using Isaac Sim viewport - Rerun Visualizer: Web-based visualizer using the rerun library (coming soon) Visualizer Registry @@ -39,6 +39,7 @@ # Import visualizer implementations from .newton_visualizer import NewtonVisualizer +from .ov_visualizer import OVVisualizer # Global registry for visualizer types (defined after Visualizer import) _VISUALIZER_REGISTRY: dict[str, Any] = {} @@ -48,6 +49,7 @@ "VisualizerCfg", "NewtonVisualizer", "NewtonVisualizerCfg", + "OVVisualizer", "OVVisualizerCfg", "RerunVisualizerCfg", "register_visualizer", @@ -93,5 +95,7 @@ def get_visualizer_class(name: str) -> Type[Visualizer] | None: # Register built-in visualizers # Note: Registration happens here to avoid circular imports _VISUALIZER_REGISTRY["newton"] = NewtonVisualizer +_VISUALIZER_REGISTRY["omniverse"] = OVVisualizer +_VISUALIZER_REGISTRY["ov"] = OVVisualizer # Alias for convenience diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index 52368afa4fd..fa643dab1db 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -288,6 +288,10 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: scene_data: Optional dictionary containing initial scene data. Expected keys: - "model": Newton Model object (required) - "state": Newton State object (optional) + - "metadata": Scene metadata (contains physics_backend) + + Raises: + RuntimeError: If Newton model is not available or if physics backend is incompatible. """ if self._is_initialized: return @@ -299,8 +303,20 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: self._state = scene_data.get("state") metadata = scene_data.get("metadata", {}) + # Validate physics backend compatibility + physics_backend = metadata.get("physics_backend", "unknown") + if physics_backend != "newton" and physics_backend != "unknown": + raise RuntimeError( + f"Newton visualizer requires Newton physics backend, but '{physics_backend}' is running. " + f"Please use a compatible visualizer for {physics_backend} physics (e.g., OVVisualizer)." + ) + + # Validate required data if self._model is None: - raise ValueError("Newton visualizer requires a Newton Model to be provided in scene_data['model']") + raise RuntimeError( + "Newton visualizer requires a Newton Model in scene_data['model']. " + "Make sure Newton physics is initialized before creating the visualizer." + ) # Create the viewer with metadata self._viewer = NewtonViewerGL( @@ -405,6 +421,23 @@ def is_running(self) -> bool: return False return self._viewer.is_running() + + def supports_markers(self) -> bool: + """Check if Newton visualizer supports visualization markers. + + Returns: + False - Newton visualizer currently does not support VisualizationMarkers + (they are USD-based and Newton uses its own rendering). + """ + return False + + def supports_live_plots(self) -> bool: + """Check if Newton visualizer supports live plots. + + Returns: + True - Newton visualizer supports live plots via ImGui integration. + """ + return True def is_training_paused(self) -> bool: """Check if training is paused by the visualizer. diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py index 095a8b7de49..ceee98e1cd7 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py @@ -1,18 +1,310 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -"""Omniverse Visualizer implementation.""" +"""Omniverse-based visualizer using Isaac Sim viewport.""" from __future__ import annotations +import omni.log +from typing import Any + +from isaaclab.sim.scene_data_providers import SceneDataProvider + from .ov_visualizer_cfg import OVVisualizerCfg from .visualizer import Visualizer -class OmniverseVisualizer(Visualizer): - """Omniverse Visualizer implementation.""" +class OVVisualizer(Visualizer): + """Omniverse-based visualizer using Isaac Sim viewport. + + This visualizer leverages the existing Isaac Sim application and viewport for visualization. + It provides: + - Automatic rendering of the USD stage in the viewport + - Support for VisualizationMarkers (via USD prims, automatically visible) + - Support for LivePlots (via Isaac Lab UI, automatically displayed) + - Configurable viewport camera positioning + + The visualizer can operate in two modes: + 1. **Attached mode**: Uses an existing Isaac Sim app instance (typical case) + 2. **Standalone mode**: Launches a new Isaac Sim app if none exists (fallback) + + Note: + VisualizationMarkers and LivePlots are managed by the scene and environment, + not directly by this visualizer. This class primarily ensures the viewport + is configured correctly to display them. + """ + def __init__(self, cfg: OVVisualizerCfg): + """Initialize OV visualizer. + + Args: + cfg: Configuration for OV visualizer. + """ super().__init__(cfg) - # stub for now \ No newline at end of file + self.cfg: OVVisualizerCfg = cfg + + # Simulation app instance + self._simulation_app = None + self._app_launched_by_visualizer = False + + # Viewport references + self._viewport_window = None + self._viewport_api = None + + # Internal state + self._is_initialized = False + self._sim_time = 0.0 + self._step_counter = 0 + + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + """Initialize OV visualizer with scene data. + + This method: + 1. Validates required data (USD stage) + 2. Checks if Isaac Sim app is running, launches if needed + 3. Configures the viewport camera + 4. Prepares for visualization of markers and plots + + Args: + scene_data: Scene data from SceneDataProvider. Contains: + - "usd_stage": The USD stage (required) + - "metadata": Scene metadata (physics backend, num_envs, etc.) + + Raises: + RuntimeError: If USD stage is not available. + + Note: + OV visualizer works with any physics backend (Newton, PhysX, etc.) + as long as a USD stage is available. + """ + if self._is_initialized: + omni.log.warn("[OVVisualizer] Already initialized. Skipping re-initialization.") + return + + # Extract scene data + metadata = {} + usd_stage = None + if scene_data is not None: + usd_stage = scene_data.get("usd_stage") + metadata = scene_data.get("metadata", {}) + + # Validate required data + if usd_stage is None: + raise RuntimeError( + "OV visualizer requires a USD stage in scene_data['usd_stage']. " + "Make sure the simulation context is initialized before creating the visualizer." + ) + + # Check if Isaac Sim app is running + self._ensure_simulation_app() + + # Setup viewport + self._setup_viewport(usd_stage, metadata) + + # Log initialization + physics_backend = metadata.get("physics_backend", "unknown") + num_envs = metadata.get("num_envs", 0) + omni.log.info( + f"[OVVisualizer] Initialized with {num_envs} environments " + f"(physics: {physics_backend})" + ) + + self._is_initialized = True + + def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: + """Update visualizer each step. + + For the OV visualizer, most rendering is handled automatically by Isaac Sim. + This method primarily updates internal timing. + + Args: + dt: Time step in seconds. + scene_provider: Optional scene data provider (not used in minimal implementation). + """ + if not self._is_initialized: + omni.log.warn("[OVVisualizer] Not initialized. Call initialize() first.") + return + + # Update internal state + self._sim_time += dt + self._step_counter += 1 + + # Note: Viewport rendering is handled automatically by Isaac Sim's render loop + # VisualizationMarkers are updated by their respective owners + # LivePlots are updated by ManagerLiveVisualizer + + def close(self) -> None: + """Clean up visualizer resources.""" + if not self._is_initialized: + return + + # Close app if we launched it + if self._app_launched_by_visualizer and self._simulation_app is not None: + omni.log.info("[OVVisualizer] Closing Isaac Sim app launched by visualizer.") + self._simulation_app.close() + self._simulation_app = None + + self._viewport_window = None + self._viewport_api = None + self._is_initialized = False + + def is_running(self) -> bool: + """Check if visualizer is running.""" + if self._simulation_app is None: + return False + return self._simulation_app.is_running() + + def is_training_paused(self) -> bool: + """Check if training is paused. + + Note: OV visualizer does not have a built-in pause mechanism. + Returns False (never pauses training). + """ + return False + + def supports_markers(self) -> bool: + """Check if this visualizer supports visualization markers. + + Returns: + True - OV visualizer supports markers via USD prims. + """ + return True + + def supports_live_plots(self) -> bool: + """Check if this visualizer supports live plots. + + Returns: + True - OV visualizer supports live plots via Isaac Lab UI. + """ + return True + + # ------------------------------------------------------------------ + # Private methods + # ------------------------------------------------------------------ + + def _ensure_simulation_app(self) -> None: + """Ensure Isaac Sim app is running, launch if needed.""" + # Try to get existing SimulationApp instance + try: + from isaacsim import SimulationApp + + # Check if there's an existing app instance + # SimulationApp uses a singleton pattern + if hasattr(SimulationApp, '_instance') and SimulationApp._instance is not None: + self._simulation_app = SimulationApp._instance + omni.log.info("[OVVisualizer] Using existing Isaac Sim app instance.") + return + except ImportError: + omni.log.warn("[OVVisualizer] Could not import SimulationApp. May not be available.") + + # If we get here, no app is running + if not self.cfg.launch_app_if_missing: + omni.log.warn( + "[OVVisualizer] No Isaac Sim app is running and launch_app_if_missing=False. " + "Visualizer may not function correctly." + ) + return + + # Launch a new app + omni.log.info("[OVVisualizer] No Isaac Sim app found. Launching new instance...") + try: + from isaacsim import SimulationApp + + # Launch app with minimal config + launch_config = { + "headless": False, + "experience": self.cfg.app_experience, + } + + self._simulation_app = SimulationApp(launch_config) + self._app_launched_by_visualizer = True + + omni.log.info(f"[OVVisualizer] Launched Isaac Sim app with experience: {self.cfg.app_experience}") + + except Exception as e: + omni.log.error(f"[OVVisualizer] Failed to launch Isaac Sim app: {e}") + self._simulation_app = None + + def _setup_viewport(self, usd_stage, metadata: dict) -> None: + """Setup viewport with camera positioning. + + Args: + usd_stage: USD stage to display. + metadata: Scene metadata. + """ + try: + import omni.kit.viewport.utility as vp_utils + from omni.kit.viewport.utility import get_active_viewport + + # Get the active viewport + if self.cfg.viewport_name: + # Try to get specific viewport by name + self._viewport_window = get_active_viewport() # For now, use active + else: + self._viewport_window = get_active_viewport() + + if self._viewport_window is None: + omni.log.warn("[OVVisualizer] Could not get viewport window.") + return + + # Get viewport API for camera control + self._viewport_api = self._viewport_window.viewport_api + + # Set camera position if specified + if self.cfg.camera_position is not None and self.cfg.camera_target is not None: + self._set_viewport_camera( + self.cfg.camera_position, + self.cfg.camera_target + ) + + omni.log.info("[OVVisualizer] Viewport configured successfully.") + + except ImportError as e: + omni.log.warn(f"[OVVisualizer] Viewport utilities not available: {e}") + except Exception as e: + omni.log.error(f"[OVVisualizer] Error setting up viewport: {e}") + + def _set_viewport_camera( + self, + position: tuple[float, float, float], + target: tuple[float, float, float] + ) -> None: + """Set viewport camera position and target. + + Args: + position: Camera position (x, y, z). + target: Camera target/look-at point (x, y, z). + """ + if self._viewport_api is None: + return + + try: + from pxr import Gf + + # Create camera transformation + eye = Gf.Vec3d(*position) + target_pos = Gf.Vec3d(*target) + up = Gf.Vec3d(0, 0, 1) # Z-up + + # Set camera transform + # Note: The exact API might vary depending on Isaac Sim version + # This is a common pattern, but may need adjustment + transform = Gf.Matrix4d() + transform.SetLookAt(eye, target_pos, up) + + # Try to apply to viewport + # The API for this can vary, so we'll try a few approaches + if hasattr(self._viewport_api, 'set_view'): + self._viewport_api.set_view(eye, target_pos, up) + elif hasattr(self._viewport_window, 'set_camera_position'): + self._viewport_window.set_camera_position(*position, True) + self._viewport_window.set_camera_target(*target, True) + + omni.log.info( + f"[OVVisualizer] Set camera: pos={position}, target={target}" + ) + + except Exception as e: + omni.log.warn(f"[OVVisualizer] Could not set camera transform: {e}") diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py index e67cc47b116..47252a87a46 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py @@ -1,11 +1,9 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -"""Configuration for Omniverse Visualizer.""" - -from typing import Literal +"""Configuration for Omniverse-based visualizer.""" from isaaclab.utils import configclass @@ -14,147 +12,39 @@ @configclass class OVVisualizerCfg(VisualizerCfg): - """Configuration for Omniverse Visualizer. - - The Omniverse Visualizer uses the Omniverse SDK to provide high-fidelity - visualization of the simulation. This is currently implemented through the - Isaac Sim app, but will eventually use the standalone Omniverse SDK module. - - Features: - - High-fidelity rendering with RTX - - Full visual shape support (not just collision shapes) - - USD-based scene representation - - Advanced lighting and materials - - Integration with Omniverse ecosystem + """Configuration for Omniverse-based visualizer. - Note: - The Omniverse Visualizer has higher overhead than the Newton Visualizer - and requires Omniverse/Isaac Sim to be installed. - """ - - # Override defaults for Omniverse visualizer - camera_position: tuple[float, float, float] = (10.0, 10.0, 10.0) - """Initial position of the camera. Default is (10.0, 10.0, 10.0).""" - - window_width: int = 1920 - """Width of the visualizer window in pixels. Default is 1920.""" - - window_height: int = 1080 - """Height of the visualizer window in pixels. Default is 1080.""" - - # Omniverse-specific settings - viewport_name: str = "/OmniverseKit_Persp" - """Name of the viewport to use for visualization. Default is "/OmniverseKit_Persp".""" - - show_origin_axis: bool = True - """Whether to show the origin axis. Default is True.""" - - show_grid: bool = True - """Whether to show the grid. Default is True.""" - - grid_scale: float = 1.0 - """Scale of the grid. Default is 1.0.""" - - enable_scene_lights: bool = True - """Whether to enable scene lights. Default is True.""" - - default_light_intensity: float = 3000.0 - """Default intensity for scene lights. Default is 3000.0.""" - - enable_dome_light: bool = True - """Whether to enable dome (environment) lighting. Default is True.""" - - dome_light_intensity: float = 1000.0 - """Intensity of the dome light. Default is 1000.0.""" - - dome_light_texture: str | None = None - """Path to HDR texture for dome light. Default is None (use default). + This visualizer uses the Isaac Sim application viewport for visualization. + It automatically displays: + - The USD stage (all environment prims) + - VisualizationMarkers (via USD prims) + - LivePlots (via Isaac Lab UI widgets) - If specified, should be a path to an HDR image file for image-based lighting. + The visualizer can operate in two modes: + 1. Attached mode: Uses an existing Isaac Sim app instance + 2. Standalone mode: Launches a new Isaac Sim app if none exists """ - - camera_projection: Literal["perspective", "orthographic"] = "perspective" - """Camera projection type. Default is "perspective".""" - - fov: float = 60.0 - """Field of view for the camera in degrees (for perspective projection). Default is 60.0.""" - - near_plane: float = 0.1 - """Near clipping plane distance. Default is 0.1.""" - - far_plane: float = 10000.0 - """Far clipping plane distance. Default is 10000.0.""" - - enable_ui: bool = True - """Whether to enable the Omniverse UI. Default is True. - When disabled, runs in a more minimal mode which can improve performance. - """ - - ui_window_layout: str | None = None - """Custom UI window layout file. Default is None (use default layout). + visualizer_type: str = "omniverse" - Can be a path to a .json file specifying the UI layout. - """ - - show_selection_outline: bool = True - """Whether to show selection outline on picked objects. Default is True.""" - - show_physics_debug_viz: bool = False - """Whether to show physics debug visualization (contacts, joints). Default is False.""" - - show_bounding_boxes: bool = False - """Whether to show bounding boxes. Default is False.""" - - display_options: int = 3094 - """Display options bitmask. Default is 3094. + # Viewport settings + viewport_name: str = "/OmniverseKit/Viewport" + """Name of the viewport to use. If None, uses the default active viewport.""" - This controls what is visible in the stage. The default value (3094) hides - extra objects that shouldn't appear in visualization. Another common value - is 3286 for the regular editor experience. - """ - - enable_live_sync: bool = False - """Whether to enable live sync with Omniverse. Default is False. + camera_position: tuple[float, float, float] | None = (10.0, 10.0, 10.0) + """Initial camera position for viewport (x, y, z). If None, keeps current camera pose.""" - When enabled, allows other Omniverse clients to connect and view the simulation. - """ - - antialiasing: Literal["Off", "FXAA", "TAA", "DLSS", "DLAA"] = "TAA" - """Anti-aliasing mode. Default is "TAA". + camera_target: tuple[float, float, float] | None = (0.0, 0.0, 0.0) + """Initial camera target/look-at point (x, y, z). If None, keeps current target.""" - - Off: No anti-aliasing - - FXAA: Fast approximate anti-aliasing - - TAA: Temporal anti-aliasing - - DLSS: NVIDIA Deep Learning Super Sampling (requires RTX GPU) - - DLAA: NVIDIA Deep Learning Anti-Aliasing (requires RTX GPU) - """ - - enable_post_processing: bool = True - """Whether to enable post-processing effects. Default is True.""" - - enable_motion_blur: bool = False - """Whether to enable motion blur. Default is False.""" - - enable_depth_of_field: bool = False - """Whether to enable depth of field. Default is False.""" - - background_color: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Background color as RGB values in range [0, 1]. Default is (0.0, 0.0, 0.0) (black).""" - - capture_on_play: bool = False - """Whether to start capturing when play is pressed. Default is False. + # App launch settings (for standalone mode) + launch_app_if_missing: bool = True + """If True and no app is running, launch a new Isaac Sim app instance.""" - Useful for recording the simulation for later playback. - """ - - capture_path: str | None = None - """Path to save captures. Default is None (don't capture). + app_experience: str = "isaac-sim.python.kit" + """Isaac Sim app experience file to use when launching standalone app.""" - When specified, frames will be captured to this path. - """ - - capture_format: Literal["png", "jpg", "exr"] = "png" - """Format for captured images. Default is "png".""" - - + # Environment visibility (for partial visualization - future use) + # NOTE: Partial visualization (REQ-11) is not implemented in this minimal version + # visualize_all_envs: bool = True + # env_indices_to_visualize: list[int] | None = None diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py index 271da4da78a..bbb684df68c 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py @@ -135,4 +135,29 @@ def is_closed(self) -> bool: True if close() has been called. """ return self._is_closed + + def supports_markers(self) -> bool: + """Check if this visualizer supports visualization markers. + + Visualization markers are geometric shapes (spheres, arrows, frames, etc.) + used for debug visualization. They are typically managed by the scene/environment + but rendered by the visualizer. + + Returns: + True if the visualizer can display VisualizationMarkers, False otherwise. + Default implementation returns False. + """ + return False + + def supports_live_plots(self) -> bool: + """Check if this visualizer supports live plots. + + Live plots display time-series data (observations, rewards, etc.) in real-time + via UI widgets. They are typically managed by manager-based environments. + + Returns: + True if the visualizer can display live plots, False otherwise. + Default implementation returns False. + """ + return False From b67e00e1fb954912746e7477f9bdbdb6e39241ab Mon Sep 17 00:00:00 2001 From: mtrepte Date: Fri, 14 Nov 2025 16:57:08 -0800 Subject: [PATCH 7/9] add rerun viz --- .../isaaclab/isaaclab/sim/simulation_cfg.py | 3 +- .../isaaclab/sim/visualizers/__init__.py | 4 +- .../sim/visualizers/rerun_visualizer.py | 571 +++++++++++++++++- .../sim/visualizers/rerun_visualizer_cfg.py | 202 ++++--- 4 files changed, 677 insertions(+), 103 deletions(-) diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index a3ff7250f4f..c314d946bd3 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -198,7 +198,8 @@ class SimulationCfg: """Render settings. Default is RenderCfg().""" # visualizers: list[VisualizerCfg] | VisualizerCfg | None = NewtonVisualizerCfg(enabled=True) - visualizers: list[VisualizerCfg] | VisualizerCfg | None = OVVisualizerCfg(enabled=True) + # visualizers: list[VisualizerCfg] | VisualizerCfg | None = OVVisualizerCfg(enabled=True) + visualizers: list[VisualizerCfg] | VisualizerCfg | None = RerunVisualizerCfg(enabled=True) """Visualizer settings. Default is Newton (no visualizer). This field supports multiple visualizer backends for debug visualization and monitoring diff --git a/source/isaaclab/isaaclab/sim/visualizers/__init__.py b/source/isaaclab/isaaclab/sim/visualizers/__init__.py index cee4eac6df9..271897c1030 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/__init__.py +++ b/source/isaaclab/isaaclab/sim/visualizers/__init__.py @@ -40,6 +40,7 @@ # Import visualizer implementations from .newton_visualizer import NewtonVisualizer from .ov_visualizer import OVVisualizer +from .rerun_visualizer import RerunVisualizer # Global registry for visualizer types (defined after Visualizer import) _VISUALIZER_REGISTRY: dict[str, Any] = {} @@ -51,6 +52,7 @@ "NewtonVisualizerCfg", "OVVisualizer", "OVVisualizerCfg", + "RerunVisualizer", "RerunVisualizerCfg", "register_visualizer", "get_visualizer_class", @@ -97,5 +99,5 @@ def get_visualizer_class(name: str) -> Type[Visualizer] | None: _VISUALIZER_REGISTRY["newton"] = NewtonVisualizer _VISUALIZER_REGISTRY["omniverse"] = OVVisualizer _VISUALIZER_REGISTRY["ov"] = OVVisualizer # Alias for convenience - +_VISUALIZER_REGISTRY["rerun"] = RerunVisualizer diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py index 7758a938d6e..6c356c4dd8c 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py @@ -1,18 +1,579 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -"""Omniverse Visualizer implementation.""" +"""Rerun-based visualizer using rerun-sdk.""" from __future__ import annotations -from .ov_visualizer_cfg import RerunVisualizerCfg +import numpy as np +import omni.log +import torch +from typing import Any + +from isaaclab.sim.scene_data_providers import SceneDataProvider + +from .rerun_visualizer_cfg import RerunVisualizerCfg from .visualizer import Visualizer +# Try to import rerun and Newton's ViewerRerun +try: + import rerun as rr + import rerun.blueprint as rrb + from newton.viewer import ViewerRerun + + _RERUN_AVAILABLE = True +except ImportError: + rr = None + rrb = None + ViewerRerun = None + _RERUN_AVAILABLE = False + + +class NewtonViewerRerun(ViewerRerun if _RERUN_AVAILABLE else object): + """Isaac Lab wrapper around Newton's ViewerRerun. + + This wrapper adds Isaac Lab-specific features on top of Newton's base ViewerRerun: + + - Integration with Isaac Lab's VisualizationMarkers system + - Integration with Isaac Lab's LivePlots system + - Environment metadata display + - Custom Rerun blueprint for Isaac Lab workflow + - Partial environment visualization support + + The wrapper uses composition to extend Newton's ViewerRerun without modifying + the base Newton library, similar to how NewtonViewerGL wraps ViewerGL. + + Note: + This class requires Newton physics backend and rerun-sdk to be installed. + """ + + def __init__( + self, + # Newton ViewerRerun parameters + server: bool = True, + address: str = "127.0.0.1:9876", + launch_viewer: bool = True, + app_id: str | None = None, + keep_historical_data: bool = False, + keep_scalar_history: bool = True, + record_to_rrd: str | None = None, + # Isaac Lab-specific parameters + train_mode: bool = True, + metadata: dict | None = None, + visualize_markers: bool = True, + visualize_plots: bool = True, + env_indices: list[int] | None = None, + ): + """Initialize Isaac Lab Rerun viewer wrapper. + + Args: + server: If True, start rerun in server mode (gRPC). + address: Address and port for rerun server mode. + launch_viewer: If True, launch a local rerun viewer client. + app_id: Application ID for rerun (defaults to 'newton-viewer'). + keep_historical_data: If True, keep historical data in the timeline. + keep_scalar_history: If True, keep historical scalar data. + record_to_rrd: Path to record viewer to .rrd file. + train_mode: Whether in training mode (affects behavior/UI). + metadata: Scene metadata (num_envs, physics backend, etc.). + visualize_markers: Whether to actively log VisualizationMarkers. + visualize_plots: Whether to actively log LivePlot data. + env_indices: Which environments to visualize (None = all). + """ + if not _RERUN_AVAILABLE: + raise ImportError( + "Rerun visualizer requires rerun-sdk and Newton to be installed. " + "Install with: pip install rerun-sdk" + ) + + # Call parent constructor with only Newton parameters + super().__init__( + server=server, + address=address, + launch_viewer=launch_viewer, + app_id=app_id, + # keep_historical_data=keep_historical_data, + # keep_scalar_history=keep_scalar_history, + # record_to_rrd=record_to_rrd, + ) + + # Isaac Lab specific state + self._metadata = metadata or {} + self._train_mode = train_mode + self._visualize_markers = visualize_markers + self._visualize_plots = visualize_plots + self._env_indices = env_indices + + # Storage for registered markers and plots + self._registered_markers = [] + self._registered_plots = {} + + # Log metadata on initialization + self._log_metadata() + + def _log_metadata(self) -> None: + """Log scene metadata to Rerun as text.""" + metadata_text = "# Isaac Lab Scene Metadata\n\n" + + # Physics info + physics_backend = self._metadata.get("physics_backend", "unknown") + metadata_text += f"**Physics Backend:** {physics_backend}\n" + + # Environment info + num_envs = self._metadata.get("num_envs", 0) + metadata_text += f"**Total Environments:** {num_envs}\n" + + if self._env_indices is not None: + metadata_text += f"**Visualized Environments:** {len(self._env_indices)} (indices: {self._env_indices[:5]}...)\n" + else: + metadata_text += f"**Visualized Environments:** All ({num_envs})\n" + + # Mode info + mode = "Training" if self._train_mode else "Inference/Play" + metadata_text += f"**Mode:** {mode}\n" + + # Visualization features + metadata_text += f"**Markers Enabled:** {self._visualize_markers}\n" + metadata_text += f"**Plots Enabled:** {self._visualize_plots}\n" + + # Additional metadata + for key, value in self._metadata.items(): + if key not in ["physics_backend", "num_envs"]: + metadata_text += f"**{key}:** {value}\n" + + # Log to Rerun + rr.log("metadata", rr.TextDocument(metadata_text, media_type=rr.MediaType.MARKDOWN)) + + def register_markers(self, markers: Any) -> None: + """Register VisualizationMarkers for active logging. + + Args: + markers: VisualizationMarkers instance to log each frame. + """ + if self._visualize_markers: + self._registered_markers.append(markers) + omni.log.info(f"[RerunVisualizer] Registered markers: {markers}") + + def register_plots(self, plots: dict[str, Any]) -> None: + """Register LivePlot instances for active logging. + + Args: + plots: Dictionary mapping plot names to LivePlot instances. + """ + if self._visualize_plots: + self._registered_plots.update(plots) + omni.log.info(f"[RerunVisualizer] Registered {len(plots)} plot(s)") + + def log_markers(self) -> None: + """Actively log all registered VisualizationMarkers to Rerun. + + This method converts Isaac Lab's USD-based markers to Rerun entities. + + Supported marker types: + - Arrows: Logged as line segments via rr.LineStrips3D + - Frames: Logged as XYZ axes via rr.LineStrips3D (3 lines per frame) + - Spheres: Logged as points via rr.Points3D with radii + + Implementation Strategy: + We convert USD-based visualization markers to Rerun primitives. + Since markers are scene-managed and updated by their owners, + we need to extract their current state each frame and log it. + + TODO: Future enhancements + - Support more marker types (cylinders, cones, boxes) + - Optimize batch logging for large marker counts + - Add color/material support for better visual distinction + """ + if not self._visualize_markers or len(self._registered_markers) == 0: + return + + try: + for marker_idx, markers in enumerate(self._registered_markers): + # Extract marker data + # Note: This is a simplified implementation that assumes markers + # expose their data through specific methods/properties. + # Actual implementation depends on VisualizationMarkers API. + + # For now, we'll use Newton's built-in logging methods + # which VisualizationMarkers should be compatible with + + # TODO: Implement proper marker extraction and conversion + # marker_data = markers.get_marker_data() + # self._log_marker_data(marker_data, f"markers/{marker_idx}") + + pass # Stub for now + + except Exception as e: + omni.log.warn(f"[RerunVisualizer] Failed to log markers: {e}") + + def log_plot_data(self) -> None: + """Actively log all registered LivePlot data to Rerun as time series. + + This method extracts scalar data from LivePlot objects and logs them + as Rerun Scalars, enabling visualization alongside the 3D scene. + + Implementation Strategy: + LivePlots in Isaac Lab are typically omni.ui-based widgets that + display time-series data. For Rerun, we need to extract the raw + scalar values and log them using rr.Scalar(). + + TODO: Full implementation + - Extract data from LiveLinePlot objects + - Handle multiple series per plot + - Maintain proper timeline synchronization + - Support different plot types (line, bar, etc.) + """ + if not self._visualize_plots or len(self._registered_plots) == 0: + return + + try: + for plot_name, plot_obj in self._registered_plots.items(): + # TODO: Extract data from plot object + # For now, this is a stub + # data = plot_obj.get_latest_data() + # rr.log(f"plots/{plot_name}", rr.Scalar(data)) + + pass # Stub for now + + except Exception as e: + omni.log.warn(f"[RerunVisualizer] Failed to log plot data: {e}") + + def _log_marker_data(self, marker_data: dict, entity_path: str) -> None: + """Helper to log specific marker data to Rerun. + + Args: + marker_data: Dictionary containing marker positions, types, colors, etc. + entity_path: Rerun entity path for logging. + """ + marker_type = marker_data.get("type", "unknown") + + if marker_type == "arrow": + # Log arrows as line segments + starts = marker_data.get("positions") # Start points + directions = marker_data.get("directions") # Direction vectors + + if starts is not None and directions is not None: + ends = starts + directions + self.log_lines( + entity_path, + starts=starts, + ends=ends, + colors=marker_data.get("colors"), + width=marker_data.get("width", 0.01), + ) + + elif marker_type == "frame": + # Log coordinate frames as 3 lines (XYZ axes) + positions = marker_data.get("positions") + orientations = marker_data.get("orientations") + scale = marker_data.get("scale", 0.1) + + if positions is not None and orientations is not None: + # TODO: Convert quaternions to XYZ axis lines + # For each frame, create 3 lines (red=X, green=Y, blue=Z) + pass + + elif marker_type == "sphere": + # Log spheres as points with radii + positions = marker_data.get("positions") + radii = marker_data.get("radii", 0.05) + colors = marker_data.get("colors") + + if positions is not None: + self.log_points( + entity_path, + points=positions, + radii=radii, + colors=colors, + ) + class RerunVisualizer(Visualizer): - """Omniverse Visualizer implementation.""" + """Rerun-based visualizer for Isaac Lab using rerun-sdk. + + This visualizer provides web-based visualization with advanced features: + + - **Time Scrubbing**: Rewind and replay simulation timeline + - **Data Inspection**: Inspect transforms, meshes, and data in detail + - **Recording**: Save to .rrd files for offline analysis + - **Active Logging**: Explicitly logs markers and plots (unlike passive OV visualizer) + + The Rerun visualizer wraps Newton's ViewerRerun and requires the Newton physics + backend. It uses active logging, meaning all visualization data (markers, plots) + must be explicitly logged each frame, unlike the OV visualizer where USD-based + markers automatically appear. + + Requirements: + - Newton physics backend (scene_data must contain Newton Model and State) + - rerun-sdk package: ``pip install rerun-sdk`` + + Future Support: + - PhysX backend support (planned for future release) + - Additional marker types (cylinders, cones, boxes) + - Full LivePlot integration (currently stub) + + Example: + + .. code-block:: python + + from isaaclab.sim import SimulationCfg + from isaaclab.sim.visualizers import RerunVisualizerCfg + + sim_cfg = SimulationCfg( + dt=0.01, + visualizers=RerunVisualizerCfg( + enabled=True, + launch_viewer=True, + keep_historical_data=False, # Constant memory + ), + ) + """ + def __init__(self, cfg: RerunVisualizerCfg): + """Initialize Rerun visualizer. + + Args: + cfg: Configuration for Rerun visualizer. + + Raises: + ImportError: If rerun-sdk or Newton is not installed. + """ super().__init__(cfg) - # stub for now \ No newline at end of file + self.cfg: RerunVisualizerCfg = cfg + + # Check dependencies + if not _RERUN_AVAILABLE: + raise ImportError( + "Rerun visualizer requires rerun-sdk and Newton to be installed.\n" + "Install with: pip install rerun-sdk\n" + "Make sure Newton physics is also available in your environment." + ) + + # Will be initialized in initialize() + self._viewer: NewtonViewerRerun | None = None + self._model = None + self._state = None + self._is_initialized = False + self._sim_time = 0.0 + + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + """Initialize Rerun visualizer with scene data. + + This method: + 1. Validates required data (Newton Model and State) + 2. Checks physics backend compatibility + 3. Creates NewtonViewerRerun instance with Isaac Lab extensions + 4. Sets up the model for visualization + + Args: + scene_data: Scene data from SceneDataProvider. Must contain: + - "model": Newton Model (required) + - "state": Newton State (required) + - "metadata": Scene metadata (physics backend, num_envs, etc.) + + Raises: + RuntimeError: If Newton Model/State is not available or physics backend is incompatible. + """ + if self._is_initialized: + omni.log.warn("[RerunVisualizer] Already initialized. Skipping re-initialization.") + return + + # Extract scene data + metadata = {} + if scene_data is not None: + self._model = scene_data.get("model") + self._state = scene_data.get("state") + metadata = scene_data.get("metadata", {}) + + # Validate physics backend + physics_backend = metadata.get("physics_backend", "unknown") + if physics_backend != "newton" and physics_backend != "unknown": + raise RuntimeError( + f"Rerun visualizer currently requires Newton physics backend, " + f"but '{physics_backend}' is running. " + f"Please use a compatible visualizer for {physics_backend} physics " + f"(e.g., OVVisualizer).\n\n" + f"Future versions will support multiple physics backends." + ) + + # Validate required Newton data + if self._model is None: + raise RuntimeError( + "Rerun visualizer requires a Newton Model in scene_data['model']. " + "Make sure Newton physics is initialized before creating the visualizer." + ) + + if self._state is None: + omni.log.warn( + "[RerunVisualizer] No Newton State provided in scene_data['state']. " + "Visualization may not work correctly." + ) + + # Create Newton ViewerRerun wrapper + try: + self._viewer = NewtonViewerRerun( + server=self.cfg.server_mode, + address=self.cfg.server_address, + launch_viewer=self.cfg.launch_viewer, + app_id=self.cfg.app_id, + keep_historical_data=self.cfg.keep_historical_data, + keep_scalar_history=self.cfg.keep_scalar_history, + record_to_rrd=self.cfg.record_to_rrd, + train_mode=self.cfg.train_mode, + metadata=metadata, + visualize_markers=self.cfg.visualize_markers, + visualize_plots=self.cfg.visualize_plots, + env_indices=self.cfg.env_indices, + ) + + # Set the model + self._viewer.set_model(self._model) + + # Log initialization + num_envs = metadata.get("num_envs", 0) + viz_envs = len(self.cfg.env_indices) if self.cfg.env_indices else num_envs + omni.log.info( + f"[RerunVisualizer] Initialized with {viz_envs}/{num_envs} environments " + f"(physics: {physics_backend})" + ) + + self._is_initialized = True + + except Exception as e: + omni.log.error(f"[RerunVisualizer] Failed to initialize viewer: {e}") + raise + + def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: + """Update visualizer each step. + + This method: + 1. Updates state from scene provider (if available) + 2. Logs current state to Rerun (transforms, meshes) + 3. Actively logs markers (if enabled) + 4. Actively logs plot data (if enabled) + + Implementation Note: + Partial visualization (env_indices) is handled internally by filtering + which instance transforms are logged. We log all meshes once (they're + shared assets), but only log transforms for selected environments. + + Alternative implementations: + - Option A: Filter at the Newton state level (more invasive) + - Option B: Filter during logging (current - most flexible) + - Option C: Filter at the scene provider level (future consideration) + + Args: + dt: Time step in seconds. + scene_provider: Optional scene data provider for updated state. + """ + if not self._is_initialized or self._viewer is None: + omni.log.warn("[RerunVisualizer] Not initialized. Call initialize() first.") + return + + # Update state from scene provider if available + if scene_provider is not None: + self._state = scene_provider.get_state() + + # Begin frame + self._viewer.begin_frame(self._sim_time) + + # Log state (transforms) - Newton's ViewerRerun handles this + if self._state is not None: + # Handle partial visualization if env_indices is set + if self.cfg.env_indices is not None: + # TODO: Filter state to only visualized environments + # For now, log all state (Newton's ViewerRerun will handle it) + self._viewer.log_state(self._state) + else: + self._viewer.log_state(self._state) + + # Actively log markers (if enabled) + if self.cfg.visualize_markers: + self._viewer.log_markers() + + # Actively log plot data (if enabled) + if self.cfg.visualize_plots: + self._viewer.log_plot_data() + + # End frame + self._viewer.end_frame() + + # Update internal time + self._sim_time += dt + + def close(self) -> None: + """Clean up Rerun visualizer resources. + + This closes the Rerun viewer, disconnects from the server, and + finalizes any recording files. + """ + if not self._is_initialized or self._viewer is None: + return + + try: + self._viewer.close() + omni.log.info("[RerunVisualizer] Closed successfully.") + except Exception as e: + omni.log.warn(f"[RerunVisualizer] Error during close: {e}") + + self._viewer = None + self._is_initialized = False + + def is_running(self) -> bool: + """Check if visualizer is running. + + Returns: + True if viewer is initialized and running, False otherwise. + """ + if self._viewer is None: + return False + return self._viewer.is_running() + + def is_training_paused(self) -> bool: + """Check if training is paused. + + Note: + Rerun visualizer uses Rerun's built-in timeline controls for playback. + It does not provide a training pause mechanism like NewtonVisualizer. + + Returns: + False - Rerun does not support training pause. + """ + return False + + def supports_markers(self) -> bool: + """Check if this visualizer supports visualization markers. + + Returns: + True - Rerun supports markers via active logging. + """ + return True + + def supports_live_plots(self) -> bool: + """Check if this visualizer supports live plots. + + Returns: + True - Rerun supports plots via active logging (currently stub). + """ + return True + + def register_markers(self, markers: Any) -> None: + """Register VisualizationMarkers for active logging. + + Args: + markers: VisualizationMarkers instance to visualize. + """ + if self._viewer: + self._viewer.register_markers(markers) + + def register_plots(self, plots: dict[str, Any]) -> None: + """Register LivePlot instances for active logging. + + Args: + plots: Dictionary mapping plot names to LivePlot instances. + """ + if self._viewer: + self._viewer.register_plots(plots) + diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py index 487bc6e69eb..d366f1a5bfb 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py @@ -1,11 +1,11 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause -"""Configuration for Rerun Visualizer.""" +"""Configuration for the Rerun visualizer.""" -from typing import Literal +from __future__ import annotations from isaaclab.utils import configclass @@ -14,146 +14,156 @@ @configclass class RerunVisualizerCfg(VisualizerCfg): - """Configuration for Rerun Visualizer. + """Configuration for the Rerun visualizer using rerun-sdk. - The Rerun Visualizer integrates with the rerun visualization library, enabling - real-time or offline visualization with advanced features like time scrubbing - and data inspection through a web-based interface. + The Rerun visualizer provides web-based visualization with advanced features: - Features: - - Web-based visualization interface - Time scrubbing and playback controls - - 3D scene navigation - - Data inspection and filtering - - Recording and export capabilities - - Remote viewing support + - 3D scene navigation and inspection + - Data filtering and analysis + - Recording to .rrd files for offline replay + - Built-in timeline and data inspection tools + + This visualizer requires the Newton physics backend and the rerun-sdk package: + + .. code-block:: bash + + pip install rerun-sdk Note: - Requires the rerun-sdk package to be installed: pip install rerun-sdk - """ - - # Override defaults for Rerun visualizer - camera_position: tuple[float, float, float] = (10.0, 10.0, 10.0) - """Initial position of the camera. Default is (10.0, 10.0, 10.0).""" - - # Rerun-specific settings + The Rerun visualizer wraps Newton's ViewerRerun, which requires a Newton Model + and State. It will not work with other physics backends (e.g., PhysX) until + future support is added. + + Example: + + .. code-block:: python + + from isaaclab.sim.visualizers import RerunVisualizerCfg + + visualizer_cfg = RerunVisualizerCfg( + enabled=True, + server_mode=True, + launch_viewer=True, + keep_historical_data=False, # Constant memory for training + record_to_rrd="recording.rrd", # Save to file + ) + """ + + visualizer_type: str = "rerun" + """Type identifier for the Rerun visualizer. Defaults to "rerun".""" + + # Connection settings server_mode: bool = True - """Whether to run in server mode. Default is True. + """Whether to run Rerun in server mode (gRPC). Defaults to True. - In server mode, Rerun starts a server that viewers can connect to. - When False, data is logged to a file or sent to an external viewer. + If True, Rerun will start a gRPC server that the web viewer connects to. + If False, data is logged directly (useful for recording to file only). """ server_address: str = "127.0.0.1:9876" - """Server address for Rerun. Default is "127.0.0.1:9876". + """Server address and port for gRPC mode. Defaults to "127.0.0.1:9876". - Format: "host:port". Only used when server_mode is True. + Only used if server_mode=True. The web viewer will connect to this address. """ launch_viewer: bool = True - """Whether to automatically launch the web viewer. Default is True. + """Whether to auto-launch the web viewer. Defaults to True. - When True, the Rerun web viewer will be automatically opened in a browser. + If True, a web browser will open showing the Rerun viewer interface. + The viewer provides 3D visualization, timeline controls, and data inspection. """ app_id: str = "isaaclab-simulation" - """Application identifier for Rerun. Default is "isaaclab-simulation". + """Application identifier for Rerun. Defaults to "isaaclab-simulation". - This is used to identify the application in the Rerun viewer and for - organizing recordings. + This is displayed in the Rerun viewer title and used to distinguish + multiple Rerun sessions. """ - recording_path: str | None = None - """Path to save recordings. Default is None (don't save). + # Data management + keep_historical_data: bool = False + """Whether to keep historical transform data in viewer timeline. Defaults to False. - When specified, the Rerun data will be saved to this path for later replay. - Supported formats: .rrd (Rerun recording format) - """ - - spawn_mode: Literal["connect", "spawn", "save"] = "spawn" - """How to handle the Rerun viewer. Default is "spawn". + - If False: Only current frame is kept, memory usage is constant (good for training) + - If True: Full timeline is kept, enables time scrubbing (good for debugging) - - "connect": Connect to an existing Rerun viewer - - "spawn": Spawn a new Rerun viewer process - - "save": Save to a file without opening a viewer + For long training runs with many environments, False is recommended to avoid + memory issues. For analysis and debugging, True allows rewinding the simulation. """ - max_queue_size: int = 100 - """Maximum number of messages to queue. Default is 100. + keep_scalar_history: bool = True + """Whether to keep historical scalar/plot data in viewer. Defaults to True. - Controls memory usage for buffering visualization data. + Scalar data (plots, metrics) is typically small, so keeping history is + reasonable even for long runs. This enables viewing plot trends over time. """ - flush_timeout: float = 2.0 - """Timeout for flushing data to Rerun in seconds. Default is 2.0.""" - - log_transforms: bool = True - """Whether to log rigid body transforms. Default is True.""" - - log_meshes: bool = True - """Whether to log mesh data. Default is True. + # Recording + record_to_rrd: str | None = None + """Path to save recording as .rrd file. Defaults to None (no recording). - When enabled, collision and visual meshes will be logged to Rerun. - """ - - log_cameras: bool = True - """Whether to log camera data. Default is True.""" - - log_point_clouds: bool = False - """Whether to log point cloud data. Default is False.""" - - log_images: bool = False - """Whether to log images from cameras. Default is False. + If specified (e.g., "my_recording.rrd"), all logged data will be saved to + this file for offline replay and analysis. The file can be opened later with: - Note: Logging images can significantly increase data size and bandwidth. - """ - - log_tensors: bool = False - """Whether to log tensor data (observations, actions, etc.). Default is False. + .. code-block:: bash - When enabled, can log observation buffers, action buffers, and other tensors. - """ - - time_mode: Literal["sim_time", "wall_time", "step_count"] = "sim_time" - """Time mode for logging. Default is "sim_time". + rerun my_recording.rrd - - "sim_time": Use simulation time as the timeline - - "wall_time": Use wall clock time - - "step_count": Use step count as the timeline + Example paths: + - "recording.rrd" - saves to current directory + - "/tmp/recordings/run_{timestamp}.rrd" - absolute path with timestamp + - None - no recording saved """ - entity_path_prefix: str = "/world" - """Prefix for entity paths in Rerun. Default is "/world". + # Visualization options + log_transforms: bool = True + """Whether to log rigid body transforms. Defaults to True. - All logged entities will be under this prefix in the Rerun hierarchy. + Transform logging shows the position and orientation of all rigid bodies + in the scene. This is the core visualization data. """ - log_static_once: bool = True - """Whether to log static scene data only once. Default is True. + log_meshes: bool = True + """Whether to log mesh geometry. Defaults to True. - When True, static meshes and other unchanging data are logged only at the - start, reducing data size and bandwidth. + Mesh logging shows the 3D shapes of objects. If False, only transforms + (positions/orientations) are shown as coordinate frames. """ - up_axis: Literal["+x", "-x", "+y", "-y", "+z", "-z"] = "+z" - """The up axis for the 3D space. Default is "+z". + visualize_markers: bool = True + """Whether to actively log VisualizationMarkers to Rerun. Defaults to True. - This should match your simulation's coordinate system. + If True, markers created via VisualizationMarkers (arrows, frames, spheres, etc.) + will be converted to Rerun entities and logged each frame. This requires active + logging (unlike OV visualizer where markers auto-appear in the viewport). + + Supported marker types: + - Arrows (via log_lines) + - Coordinate frames (via log_lines for XYZ axes) + - Spheres (via log_points) """ - mesh_quality: Literal["low", "medium", "high"] = "medium" - """Quality level for mesh data. Default is "medium". + visualize_plots: bool = True + """Whether to actively log LivePlot data to Rerun. Defaults to True. + + If True, scalar data from LivePlots will be logged as time series in Rerun. + This allows viewing training metrics, rewards, and other scalars alongside + the 3D visualization. - Higher quality preserves more detail but increases data size. + Note: Currently a stub - full implementation coming soon. """ - enable_compression: bool = True - """Whether to enable data compression. Default is True. + # Performance and filtering + max_instances_per_env: int | None = None + """Maximum number of instances to visualize per environment. Defaults to None (all). - Compression reduces bandwidth and storage requirements but adds CPU overhead. + For scenes with many instances per environment, this limits how many are + visualized to improve performance. None means visualize all instances. """ - verbose: bool = False - """Whether to enable verbose logging. Default is False.""" - + # Future: PhysX backend support + # When PhysX support is added, these fields will be used: + # physics_backend: Literal["newton", "physx"] | None = None + # """Physics backend to use. Auto-detected if None.""" From 30116c02333e97759ee63c1a9b16e99fc25fa420 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Fri, 14 Nov 2025 17:40:28 -0800 Subject: [PATCH 8/9] clean --- .../isaaclab/isaaclab/sim/simulation_cfg.py | 3 +- .../sim/visualizers/newton_visualizer.py | 24 +-- .../sim/visualizers/newton_visualizer_cfg.py | 95 +++------ .../isaaclab/sim/visualizers/ov_visualizer.py | 184 +++++------------- .../sim/visualizers/ov_visualizer_cfg.py | 40 ++-- .../sim/visualizers/rerun_visualizer.py | 151 +++----------- .../sim/visualizers/rerun_visualizer_cfg.py | 147 ++------------ .../isaaclab/sim/visualizers/visualizer.py | 111 ++--------- .../sim/visualizers/visualizer_cfg.py | 89 ++------- source/isaaclab/setup.py | 4 +- 10 files changed, 161 insertions(+), 687 deletions(-) diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index c314d946bd3..dc445985978 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -199,7 +199,8 @@ class SimulationCfg: # visualizers: list[VisualizerCfg] | VisualizerCfg | None = NewtonVisualizerCfg(enabled=True) # visualizers: list[VisualizerCfg] | VisualizerCfg | None = OVVisualizerCfg(enabled=True) - visualizers: list[VisualizerCfg] | VisualizerCfg | None = RerunVisualizerCfg(enabled=True) + # visualizers: list[VisualizerCfg] | VisualizerCfg | None = RerunVisualizerCfg(enabled=True) + visualizers: list[VisualizerCfg] | VisualizerCfg | None = [NewtonVisualizerCfg(enabled=True), RerunVisualizerCfg(enabled=True)] """Visualizer settings. Default is Newton (no visualizer). This field supports multiple visualizer backends for debug visualization and monitoring diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index fa643dab1db..6314e6744f3 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -29,12 +29,11 @@ class NewtonViewerGL(ViewerGL): The training pause can be toggled from the UI via a button and optionally via the 'T' key. """ - def __init__(self, *args, train_mode: bool = True, metadata: dict | None = None, **kwargs): + def __init__(self, *args, metadata: dict | None = None, **kwargs): super().__init__(*args, **kwargs) self._paused_training: bool = False self._paused_rendering: bool = False self._fallback_draw_controls: bool = False - self._is_train_mode: bool = train_mode self._visualizer_update_frequency: int = 1 self._metadata = metadata or {} @@ -70,23 +69,15 @@ def get_visualizer_update_frequency(self) -> int: # UI callback rendered inside the "Example Options" panel of the left sidebar def _render_training_controls(self, imgui): imgui.separator() - - # Use simple flag to adjust labels - if self._is_train_mode: - imgui.text("IsaacLab Training Controls") - pause_label = "Resume Training" if self._paused_training else "Pause Training" - else: - imgui.text("IsaacLab Playback Controls") - pause_label = "Resume Playing" if self._paused_training else "Pause Playing" - + imgui.text("Isaac Lab Training Controls") + + pause_label = "Resume Training" if self._paused_training else "Pause Training" if imgui.button(pause_label): self._paused_training = not self._paused_training - # Only show rendering controls when in training mode - if self._is_train_mode: - rendering_label = "Resume Rendering" if self._paused_rendering else "Pause Rendering" - if imgui.button(rendering_label): - self._paused_rendering = not self._paused_rendering + rendering_label = "Resume Rendering" if self._paused_rendering else "Pause Rendering" + if imgui.button(rendering_label): + self._paused_rendering = not self._paused_rendering imgui.text("Visualizer Update Frequency") @@ -322,7 +313,6 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: self._viewer = NewtonViewerGL( width=self.cfg.window_width, height=self.cfg.window_height, - train_mode=self.cfg.train_mode, metadata=metadata, ) diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py index dadf5ad88ef..ccbd183ac80 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py @@ -14,123 +14,82 @@ @configclass class NewtonVisualizerCfg(VisualizerCfg): - """Configuration for Newton OpenGL Visualizer. + """Configuration for Newton OpenGL visualizer. - The Newton OpenGL Visualizer is a lightweight, fast visualizer built on OpenGL. - It is designed for interactive real-time visualization of simulations with minimal - performance overhead. It requires pyglet (version >= 2.1.6) and imgui_bundle - (version >= 1.92.0) to be installed. + Lightweight OpenGL-based visualizer with real-time 3D rendering, interactive + camera controls, and debug visualization (contacts, joints, springs, COM). - Features: - - Real-time 3D visualization - - Interactive camera controls - - Debug visualization (contacts, joints, springs, COM) - - Training controls (pause training, pause rendering, update frequency) - - Lightweight and fast - - Note: - The Newton Visualizer currently only supports visualization of collision shapes, - not visual shapes. + Requires: pyglet >= 2.1.6, imgui_bundle >= 1.92.0 """ - # Visualizer type identifier visualizer_type: str = "newton" - """Type identifier for Newton visualizer. Used by the factory pattern.""" - - # Override defaults for Newton visualizer - camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) - """Initial position of the camera. Default is (10.0, 0.0, 3.0).""" + """Type identifier for Newton visualizer.""" window_width: int = 1920 - """Width of the visualizer window in pixels. Default is 1920.""" + """Window width in pixels.""" window_height: int = 1080 - """Height of the visualizer window in pixels. Default is 1080.""" + """Window height in pixels.""" # Newton-specific settings fps: int = 60 - """Target frames per second for the visualizer. Default is 60.""" + """Target FPS.""" show_joints: bool = True - """Whether to show joint visualizations. Default is True.""" + """Show joint visualizations.""" show_contacts: bool = False - """Whether to show contact visualizations. Default is False.""" + """Show contact visualizations.""" show_springs: bool = False - """Whether to show spring visualizations. Default is False.""" + """Show spring visualizations.""" show_com: bool = False - """Whether to show center of mass visualizations. Default is False.""" + """Show center of mass visualizations.""" show_ui: bool = True - """Whether to show the UI sidebar. Default is True. - - The UI can be toggled with the 'H' key during runtime. - """ + """Show UI sidebar (toggle with 'H' key).""" enable_shadows: bool = True - """Whether to enable shadow rendering. Default is True.""" + """Enable shadow rendering.""" enable_sky: bool = True - """Whether to enable sky rendering. Default is True.""" + """Enable sky rendering.""" enable_wireframe: bool = False - """Whether to enable wireframe rendering mode. Default is False.""" + """Enable wireframe rendering mode.""" up_axis: Literal["X", "Y", "Z"] = "Z" - """The up axis for the visualizer. Default is "Z". - - This should typically match the up axis of your simulation environment. - """ + """Up axis for visualizer (should match simulation).""" fov: float = 60.0 - """Field of view for the camera in degrees. Default is 60.0.""" + """Camera field of view in degrees.""" near_plane: float = 0.1 - """Near clipping plane distance for the camera. Default is 0.1.""" + """Camera near clipping plane distance.""" far_plane: float = 1000.0 - """Far clipping plane distance for the camera. Default is 1000.0.""" + """Camera far clipping plane distance.""" background_color: tuple[float, float, float] = (0.53, 0.81, 0.92) - """Background color (sky color) as RGB values in range [0, 1]. - Default is (0.53, 0.81, 0.92) (light blue).""" + """Background/sky color RGB [0,1] (light blue).""" ground_color: tuple[float, float, float] = (0.18, 0.20, 0.25) - """Ground color as RGB values in range [0, 1]. - Default is (0.18, 0.20, 0.25) (dark gray).""" + """Ground color RGB [0,1] (dark gray).""" light_color: tuple[float, float, float] = (1.0, 1.0, 1.0) - """Light color as RGB values in range [0, 1]. Default is (1.0, 1.0, 1.0) (white).""" + """Light color RGB [0,1] (white).""" enable_pause_training: bool = True - """Whether to enable the pause training button in the UI. Default is True. - - When enabled, users can pause the simulation/training while keeping the - visualizer running, which is useful for debugging. - """ + """Enable pause training button in UI.""" enable_pause_rendering: bool = True - """Whether to enable the pause rendering button in the UI. Default is True. - - When enabled, users can pause rendering while keeping simulation/training - running, which can improve training performance. - """ + """Enable pause rendering button in UI.""" show_training_controls: bool = True - """Whether to show IsaacLab-specific training controls in the UI. Default is True. - - This includes controls for pausing training, pausing rendering, and adjusting - the visualizer update frequency. - """ + """Show Isaac Lab training controls in UI.""" render_mode: Literal["rgb", "depth", "collision"] = "rgb" - """Rendering mode for the visualizer. Default is "rgb". - - - "rgb": Standard RGB rendering - - "depth": Depth visualization - - "collision": Show collision shapes only - """ + """Rendering mode: rgb (standard), depth, or collision.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py index ceee98e1cd7..5826b944c0a 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py @@ -17,132 +17,62 @@ class OVVisualizer(Visualizer): - """Omniverse-based visualizer using Isaac Sim viewport. + """Omniverse visualizer using Isaac Sim viewport. - This visualizer leverages the existing Isaac Sim application and viewport for visualization. - It provides: - - Automatic rendering of the USD stage in the viewport - - Support for VisualizationMarkers (via USD prims, automatically visible) - - Support for LivePlots (via Isaac Lab UI, automatically displayed) - - Configurable viewport camera positioning - - The visualizer can operate in two modes: - 1. **Attached mode**: Uses an existing Isaac Sim app instance (typical case) - 2. **Standalone mode**: Launches a new Isaac Sim app if none exists (fallback) - - Note: - VisualizationMarkers and LivePlots are managed by the scene and environment, - not directly by this visualizer. This class primarily ensures the viewport - is configured correctly to display them. + Renders USD stage with VisualizationMarkers and LivePlots. + Can attach to existing app or launch standalone. """ def __init__(self, cfg: OVVisualizerCfg): - """Initialize OV visualizer. - - Args: - cfg: Configuration for OV visualizer. - """ super().__init__(cfg) self.cfg: OVVisualizerCfg = cfg - # Simulation app instance self._simulation_app = None self._app_launched_by_visualizer = False - - # Viewport references self._viewport_window = None self._viewport_api = None - - # Internal state self._is_initialized = False self._sim_time = 0.0 self._step_counter = 0 def initialize(self, scene_data: dict[str, Any] | None = None) -> None: - """Initialize OV visualizer with scene data. - - This method: - 1. Validates required data (USD stage) - 2. Checks if Isaac Sim app is running, launches if needed - 3. Configures the viewport camera - 4. Prepares for visualization of markers and plots - - Args: - scene_data: Scene data from SceneDataProvider. Contains: - - "usd_stage": The USD stage (required) - - "metadata": Scene metadata (physics backend, num_envs, etc.) - - Raises: - RuntimeError: If USD stage is not available. - - Note: - OV visualizer works with any physics backend (Newton, PhysX, etc.) - as long as a USD stage is available. - """ + """Initialize OV visualizer.""" if self._is_initialized: - omni.log.warn("[OVVisualizer] Already initialized. Skipping re-initialization.") + omni.log.warn("[OVVisualizer] Already initialized.") return - # Extract scene data metadata = {} usd_stage = None if scene_data is not None: usd_stage = scene_data.get("usd_stage") metadata = scene_data.get("metadata", {}) - # Validate required data if usd_stage is None: - raise RuntimeError( - "OV visualizer requires a USD stage in scene_data['usd_stage']. " - "Make sure the simulation context is initialized before creating the visualizer." - ) + raise RuntimeError("OV visualizer requires a USD stage.") - # Check if Isaac Sim app is running self._ensure_simulation_app() - - # Setup viewport self._setup_viewport(usd_stage, metadata) - # Log initialization physics_backend = metadata.get("physics_backend", "unknown") num_envs = metadata.get("num_envs", 0) - omni.log.info( - f"[OVVisualizer] Initialized with {num_envs} environments " - f"(physics: {physics_backend})" - ) + omni.log.info(f"[OVVisualizer] Initialized ({num_envs} envs, {physics_backend} physics)") self._is_initialized = True def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: - """Update visualizer each step. - - For the OV visualizer, most rendering is handled automatically by Isaac Sim. - This method primarily updates internal timing. - - Args: - dt: Time step in seconds. - scene_provider: Optional scene data provider (not used in minimal implementation). - """ + """Update visualizer (rendering handled automatically by Isaac Sim).""" if not self._is_initialized: - omni.log.warn("[OVVisualizer] Not initialized. Call initialize() first.") return - - # Update internal state self._sim_time += dt self._step_counter += 1 - - # Note: Viewport rendering is handled automatically by Isaac Sim's render loop - # VisualizationMarkers are updated by their respective owners - # LivePlots are updated by ManagerLiveVisualizer def close(self) -> None: """Clean up visualizer resources.""" if not self._is_initialized: return - # Close app if we launched it if self._app_launched_by_visualizer and self._simulation_app is not None: - omni.log.info("[OVVisualizer] Closing Isaac Sim app launched by visualizer.") + omni.log.info("[OVVisualizer] Closing Isaac Sim app.") self._simulation_app.close() self._simulation_app = None @@ -157,27 +87,15 @@ def is_running(self) -> bool: return self._simulation_app.is_running() def is_training_paused(self) -> bool: - """Check if training is paused. - - Note: OV visualizer does not have a built-in pause mechanism. - Returns False (never pauses training). - """ + """Check if training is paused (always False for OV).""" return False def supports_markers(self) -> bool: - """Check if this visualizer supports visualization markers. - - Returns: - True - OV visualizer supports markers via USD prims. - """ + """Supports markers via USD prims.""" return True def supports_live_plots(self) -> bool: - """Check if this visualizer supports live plots. - - Returns: - True - OV visualizer supports live plots via Isaac Lab UI. - """ + """Supports live plots via Isaac Lab UI.""" return True # ------------------------------------------------------------------ @@ -228,41 +146,48 @@ def _ensure_simulation_app(self) -> None: self._simulation_app = None def _setup_viewport(self, usd_stage, metadata: dict) -> None: - """Setup viewport with camera positioning. - - Args: - usd_stage: USD stage to display. - metadata: Scene metadata. - """ + """Setup viewport with camera and window size.""" try: - import omni.kit.viewport.utility as vp_utils - from omni.kit.viewport.utility import get_active_viewport - - # Get the active viewport - if self.cfg.viewport_name: - # Try to get specific viewport by name - self._viewport_window = get_active_viewport() # For now, use active + from omni.kit.viewport.utility import get_active_viewport, create_viewport_window + import omni.ui as ui + + # Create new viewport or use existing + if self.cfg.create_viewport and self.cfg.viewport_name: + # Create new viewport + self._viewport_window = create_viewport_window( + title=self.cfg.viewport_name, + width=self.cfg.window_width, + height=self.cfg.window_height, + ) + # Make viewport visible in UI + if self._viewport_window: + self._viewport_window.visible = True + self._viewport_window.docked = False + omni.log.info(f"[OVVisualizer] Created viewport '{self.cfg.viewport_name}'") else: + # Use existing viewport self._viewport_window = get_active_viewport() + # Try to resize if window API is available + if self._viewport_window and hasattr(self._viewport_window, 'width'): + try: + self._viewport_window.width = self.cfg.window_width + self._viewport_window.height = self.cfg.window_height + except: + pass if self._viewport_window is None: - omni.log.warn("[OVVisualizer] Could not get viewport window.") + omni.log.warn("[OVVisualizer] Could not get/create viewport.") return - # Get viewport API for camera control self._viewport_api = self._viewport_window.viewport_api - # Set camera position if specified - if self.cfg.camera_position is not None and self.cfg.camera_target is not None: - self._set_viewport_camera( - self.cfg.camera_position, - self.cfg.camera_target - ) + # Set camera + self._set_viewport_camera(self.cfg.camera_position, self.cfg.camera_target) - omni.log.info("[OVVisualizer] Viewport configured successfully.") + omni.log.info("[OVVisualizer] Viewport configured.") except ImportError as e: - omni.log.warn(f"[OVVisualizer] Viewport utilities not available: {e}") + omni.log.warn(f"[OVVisualizer] Viewport utilities unavailable: {e}") except Exception as e: omni.log.error(f"[OVVisualizer] Error setting up viewport: {e}") @@ -271,40 +196,25 @@ def _set_viewport_camera( position: tuple[float, float, float], target: tuple[float, float, float] ) -> None: - """Set viewport camera position and target. - - Args: - position: Camera position (x, y, z). - target: Camera target/look-at point (x, y, z). - """ + """Set viewport camera position and target.""" if self._viewport_api is None: return try: from pxr import Gf - # Create camera transformation eye = Gf.Vec3d(*position) target_pos = Gf.Vec3d(*target) - up = Gf.Vec3d(0, 0, 1) # Z-up - - # Set camera transform - # Note: The exact API might vary depending on Isaac Sim version - # This is a common pattern, but may need adjustment - transform = Gf.Matrix4d() - transform.SetLookAt(eye, target_pos, up) + up = Gf.Vec3d(0, 0, 1) - # Try to apply to viewport - # The API for this can vary, so we'll try a few approaches + # Try viewport API methods if hasattr(self._viewport_api, 'set_view'): self._viewport_api.set_view(eye, target_pos, up) elif hasattr(self._viewport_window, 'set_camera_position'): self._viewport_window.set_camera_position(*position, True) self._viewport_window.set_camera_target(*target, True) - omni.log.info( - f"[OVVisualizer] Set camera: pos={position}, target={target}" - ) + omni.log.info(f"[OVVisualizer] Camera: pos={position}, target={target}") except Exception as e: - omni.log.warn(f"[OVVisualizer] Could not set camera transform: {e}") + omni.log.warn(f"[OVVisualizer] Could not set camera: {e}") diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py index 47252a87a46..b107f1cd0d4 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py @@ -12,39 +12,29 @@ @configclass class OVVisualizerCfg(VisualizerCfg): - """Configuration for Omniverse-based visualizer. + """Configuration for Omniverse visualizer using Isaac Sim viewport. - This visualizer uses the Isaac Sim application viewport for visualization. - It automatically displays: - - The USD stage (all environment prims) - - VisualizationMarkers (via USD prims) - - LivePlots (via Isaac Lab UI widgets) - - The visualizer can operate in two modes: - 1. Attached mode: Uses an existing Isaac Sim app instance - 2. Standalone mode: Launches a new Isaac Sim app if none exists + Displays USD stage, VisualizationMarkers, and LivePlots. + Can attach to existing app or launch standalone. """ visualizer_type: str = "omniverse" + """Type identifier for Omniverse visualizer.""" + + viewport_name: str | None = "/OmniverseKit/Viewport" + """Viewport name to use. If None, uses active viewport.""" - # Viewport settings - viewport_name: str = "/OmniverseKit/Viewport" - """Name of the viewport to use. If None, uses the default active viewport.""" + create_viewport: bool = False + """Create new viewport with specified name and camera pose.""" - camera_position: tuple[float, float, float] | None = (10.0, 10.0, 10.0) - """Initial camera position for viewport (x, y, z). If None, keeps current camera pose.""" + window_width: int = 1920 + """Viewport width in pixels.""" - camera_target: tuple[float, float, float] | None = (0.0, 0.0, 0.0) - """Initial camera target/look-at point (x, y, z). If None, keeps current target.""" + window_height: int = 1080 + """Viewport height in pixels.""" - # App launch settings (for standalone mode) launch_app_if_missing: bool = True - """If True and no app is running, launch a new Isaac Sim app instance.""" + """Launch Isaac Sim if not already running.""" app_experience: str = "isaac-sim.python.kit" - """Isaac Sim app experience file to use when launching standalone app.""" - - # Environment visibility (for partial visualization - future use) - # NOTE: Partial visualization (REQ-11) is not implemented in this minimal version - # visualize_all_envs: bool = True - # env_indices_to_visualize: list[int] | None = None + """Isaac Sim experience file for standalone launch.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py index 6c356c4dd8c..eaa23fea369 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py @@ -32,26 +32,12 @@ class NewtonViewerRerun(ViewerRerun if _RERUN_AVAILABLE else object): - """Isaac Lab wrapper around Newton's ViewerRerun. + """Isaac Lab wrapper for Newton's ViewerRerun. - This wrapper adds Isaac Lab-specific features on top of Newton's base ViewerRerun: - - - Integration with Isaac Lab's VisualizationMarkers system - - Integration with Isaac Lab's LivePlots system - - Environment metadata display - - Custom Rerun blueprint for Isaac Lab workflow - - Partial environment visualization support - - The wrapper uses composition to extend Newton's ViewerRerun without modifying - the base Newton library, similar to how NewtonViewerGL wraps ViewerGL. - - Note: - This class requires Newton physics backend and rerun-sdk to be installed. - """ + Adds VisualizationMarkers, LivePlots, metadata display, and partial visualization.""" def __init__( self, - # Newton ViewerRerun parameters server: bool = True, address: str = "127.0.0.1:9876", launch_viewer: bool = True, @@ -59,51 +45,30 @@ def __init__( keep_historical_data: bool = False, keep_scalar_history: bool = True, record_to_rrd: str | None = None, - # Isaac Lab-specific parameters - train_mode: bool = True, metadata: dict | None = None, - visualize_markers: bool = True, - visualize_plots: bool = True, + enable_markers: bool = True, + enable_live_plots: bool = True, env_indices: list[int] | None = None, ): - """Initialize Isaac Lab Rerun viewer wrapper. - - Args: - server: If True, start rerun in server mode (gRPC). - address: Address and port for rerun server mode. - launch_viewer: If True, launch a local rerun viewer client. - app_id: Application ID for rerun (defaults to 'newton-viewer'). - keep_historical_data: If True, keep historical data in the timeline. - keep_scalar_history: If True, keep historical scalar data. - record_to_rrd: Path to record viewer to .rrd file. - train_mode: Whether in training mode (affects behavior/UI). - metadata: Scene metadata (num_envs, physics backend, etc.). - visualize_markers: Whether to actively log VisualizationMarkers. - visualize_plots: Whether to actively log LivePlot data. - env_indices: Which environments to visualize (None = all). - """ + """Initialize Newton ViewerRerun wrapper.""" if not _RERUN_AVAILABLE: - raise ImportError( - "Rerun visualizer requires rerun-sdk and Newton to be installed. " - "Install with: pip install rerun-sdk" - ) + raise ImportError("Rerun visualizer requires rerun-sdk and Newton. Install: pip install rerun-sdk") - # Call parent constructor with only Newton parameters + # Call parent with only Newton parameters super().__init__( server=server, address=address, launch_viewer=launch_viewer, app_id=app_id, - # keep_historical_data=keep_historical_data, - # keep_scalar_history=keep_scalar_history, - # record_to_rrd=record_to_rrd, + keep_historical_data=keep_historical_data, + keep_scalar_history=keep_scalar_history, + record_to_rrd=record_to_rrd, ) - # Isaac Lab specific state + # Isaac Lab state self._metadata = metadata or {} - self._train_mode = train_mode - self._visualize_markers = visualize_markers - self._visualize_plots = visualize_plots + self._enable_markers = enable_markers + self._enable_live_plots = enable_live_plots self._env_indices = env_indices # Storage for registered markers and plots @@ -130,13 +95,13 @@ def _log_metadata(self) -> None: else: metadata_text += f"**Visualized Environments:** All ({num_envs})\n" - # Mode info - mode = "Training" if self._train_mode else "Inference/Play" - metadata_text += f"**Mode:** {mode}\n" + # Physics backend info + physics_backend = self._metadata.get("physics_backend", "unknown") + metadata_text += f"**Physics:** {physics_backend}\n" # Visualization features - metadata_text += f"**Markers Enabled:** {self._visualize_markers}\n" - metadata_text += f"**Plots Enabled:** {self._visualize_plots}\n" + metadata_text += f"**Markers Enabled:** {self._enable_markers}\n" + metadata_text += f"**Plots Enabled:** {self._enable_live_plots}\n" # Additional metadata for key, value in self._metadata.items(): @@ -291,67 +256,18 @@ def _log_marker_data(self, marker_data: dict, entity_path: str) -> None: class RerunVisualizer(Visualizer): - """Rerun-based visualizer for Isaac Lab using rerun-sdk. - - This visualizer provides web-based visualization with advanced features: - - - **Time Scrubbing**: Rewind and replay simulation timeline - - **Data Inspection**: Inspect transforms, meshes, and data in detail - - **Recording**: Save to .rrd files for offline analysis - - **Active Logging**: Explicitly logs markers and plots (unlike passive OV visualizer) - - The Rerun visualizer wraps Newton's ViewerRerun and requires the Newton physics - backend. It uses active logging, meaning all visualization data (markers, plots) - must be explicitly logged each frame, unlike the OV visualizer where USD-based - markers automatically appear. + """Rerun web-based visualizer with time scrubbing, recording, and data inspection. - Requirements: - - Newton physics backend (scene_data must contain Newton Model and State) - - rerun-sdk package: ``pip install rerun-sdk`` - - Future Support: - - PhysX backend support (planned for future release) - - Additional marker types (cylinders, cones, boxes) - - Full LivePlot integration (currently stub) - - Example: - - .. code-block:: python - - from isaaclab.sim import SimulationCfg - from isaaclab.sim.visualizers import RerunVisualizerCfg - - sim_cfg = SimulationCfg( - dt=0.01, - visualizers=RerunVisualizerCfg( - enabled=True, - launch_viewer=True, - keep_historical_data=False, # Constant memory - ), - ) - """ + Requires Newton physics backend and rerun-sdk (pip install rerun-sdk).""" def __init__(self, cfg: RerunVisualizerCfg): - """Initialize Rerun visualizer. - - Args: - cfg: Configuration for Rerun visualizer. - - Raises: - ImportError: If rerun-sdk or Newton is not installed. - """ + """Initialize Rerun visualizer.""" super().__init__(cfg) self.cfg: RerunVisualizerCfg = cfg - # Check dependencies if not _RERUN_AVAILABLE: - raise ImportError( - "Rerun visualizer requires rerun-sdk and Newton to be installed.\n" - "Install with: pip install rerun-sdk\n" - "Make sure Newton physics is also available in your environment." - ) + raise ImportError("Rerun visualizer requires rerun-sdk and Newton. Install: pip install rerun-sdk") - # Will be initialized in initialize() self._viewer: NewtonViewerRerun | None = None self._model = None self._state = None @@ -359,23 +275,7 @@ def __init__(self, cfg: RerunVisualizerCfg): self._sim_time = 0.0 def initialize(self, scene_data: dict[str, Any] | None = None) -> None: - """Initialize Rerun visualizer with scene data. - - This method: - 1. Validates required data (Newton Model and State) - 2. Checks physics backend compatibility - 3. Creates NewtonViewerRerun instance with Isaac Lab extensions - 4. Sets up the model for visualization - - Args: - scene_data: Scene data from SceneDataProvider. Must contain: - - "model": Newton Model (required) - - "state": Newton State (required) - - "metadata": Scene metadata (physics backend, num_envs, etc.) - - Raises: - RuntimeError: If Newton Model/State is not available or physics backend is incompatible. - """ + """Initialize visualizer with Newton Model and State.""" if self._is_initialized: omni.log.warn("[RerunVisualizer] Already initialized. Skipping re-initialization.") return @@ -421,10 +321,9 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: keep_historical_data=self.cfg.keep_historical_data, keep_scalar_history=self.cfg.keep_scalar_history, record_to_rrd=self.cfg.record_to_rrd, - train_mode=self.cfg.train_mode, metadata=metadata, - visualize_markers=self.cfg.visualize_markers, - visualize_plots=self.cfg.visualize_plots, + enable_markers=self.cfg.enable_markers, + enable_live_plots=self.cfg.enable_live_plots, env_indices=self.cfg.env_indices, ) diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py index d366f1a5bfb..77b3716d036 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py @@ -14,156 +14,33 @@ @configclass class RerunVisualizerCfg(VisualizerCfg): - """Configuration for the Rerun visualizer using rerun-sdk. + """Configuration for Rerun visualizer (web-based visualization). - The Rerun visualizer provides web-based visualization with advanced features: - - - Time scrubbing and playback controls - - 3D scene navigation and inspection - - Data filtering and analysis - - Recording to .rrd files for offline replay - - Built-in timeline and data inspection tools - - This visualizer requires the Newton physics backend and the rerun-sdk package: - - .. code-block:: bash - - pip install rerun-sdk - - Note: - The Rerun visualizer wraps Newton's ViewerRerun, which requires a Newton Model - and State. It will not work with other physics backends (e.g., PhysX) until - future support is added. - - Example: - - .. code-block:: python - - from isaaclab.sim.visualizers import RerunVisualizerCfg - - visualizer_cfg = RerunVisualizerCfg( - enabled=True, - server_mode=True, - launch_viewer=True, - keep_historical_data=False, # Constant memory for training - record_to_rrd="recording.rrd", # Save to file - ) + Provides time scrubbing, 3D navigation, data filtering, and .rrd recording. + Requires Newton physics backend and rerun-sdk: `pip install rerun-sdk` """ visualizer_type: str = "rerun" - """Type identifier for the Rerun visualizer. Defaults to "rerun".""" + """Type identifier for Rerun visualizer.""" - # Connection settings server_mode: bool = True - """Whether to run Rerun in server mode (gRPC). Defaults to True. - - If True, Rerun will start a gRPC server that the web viewer connects to. - If False, data is logged directly (useful for recording to file only). - """ + """Run Rerun in server mode (gRPC for web viewer).""" server_address: str = "127.0.0.1:9876" - """Server address and port for gRPC mode. Defaults to "127.0.0.1:9876". - - Only used if server_mode=True. The web viewer will connect to this address. - """ + """Server address and port for gRPC mode.""" launch_viewer: bool = True - """Whether to auto-launch the web viewer. Defaults to True. - - If True, a web browser will open showing the Rerun viewer interface. - The viewer provides 3D visualization, timeline controls, and data inspection. - """ + """Auto-launch web viewer in browser.""" app_id: str = "isaaclab-simulation" - """Application identifier for Rerun. Defaults to "isaaclab-simulation". - - This is displayed in the Rerun viewer title and used to distinguish - multiple Rerun sessions. - """ + """Application identifier shown in viewer title.""" - # Data management keep_historical_data: bool = False - """Whether to keep historical transform data in viewer timeline. Defaults to False. - - - If False: Only current frame is kept, memory usage is constant (good for training) - - If True: Full timeline is kept, enables time scrubbing (good for debugging) - - For long training runs with many environments, False is recommended to avoid - memory issues. For analysis and debugging, True allows rewinding the simulation. - """ + """Keep transform history for time scrubbing (False = constant memory for training).""" keep_scalar_history: bool = True - """Whether to keep historical scalar/plot data in viewer. Defaults to True. - - Scalar data (plots, metrics) is typically small, so keeping history is - reasonable even for long runs. This enables viewing plot trends over time. - """ - - # Recording - record_to_rrd: str | None = None - """Path to save recording as .rrd file. Defaults to None (no recording). - - If specified (e.g., "my_recording.rrd"), all logged data will be saved to - this file for offline replay and analysis. The file can be opened later with: - - .. code-block:: bash - - rerun my_recording.rrd - - Example paths: - - "recording.rrd" - saves to current directory - - "/tmp/recordings/run_{timestamp}.rrd" - absolute path with timestamp - - None - no recording saved - """ - - # Visualization options - log_transforms: bool = True - """Whether to log rigid body transforms. Defaults to True. - - Transform logging shows the position and orientation of all rigid bodies - in the scene. This is the core visualization data. - """ - - log_meshes: bool = True - """Whether to log mesh geometry. Defaults to True. - - Mesh logging shows the 3D shapes of objects. If False, only transforms - (positions/orientations) are shown as coordinate frames. - """ - - visualize_markers: bool = True - """Whether to actively log VisualizationMarkers to Rerun. Defaults to True. - - If True, markers created via VisualizationMarkers (arrows, frames, spheres, etc.) - will be converted to Rerun entities and logged each frame. This requires active - logging (unlike OV visualizer where markers auto-appear in the viewport). - - Supported marker types: - - Arrows (via log_lines) - - Coordinate frames (via log_lines for XYZ axes) - - Spheres (via log_points) - """ - - visualize_plots: bool = True - """Whether to actively log LivePlot data to Rerun. Defaults to True. - - If True, scalar data from LivePlots will be logged as time series in Rerun. - This allows viewing training metrics, rewards, and other scalars alongside - the 3D visualization. - - Note: Currently a stub - full implementation coming soon. - """ - - # Performance and filtering - max_instances_per_env: int | None = None - """Maximum number of instances to visualize per environment. Defaults to None (all). - - For scenes with many instances per environment, this limits how many are - visualized to improve performance. None means visualize all instances. - """ + """Keep scalar/plot history in timeline.""" - # Future: PhysX backend support - # When PhysX support is added, these fields will be used: - # physics_backend: Literal["newton", "physx"] | None = None - # """Physics backend to use. Auto-detected if None.""" + record_to_rrd: str | None = test.rrd + """Path to save .rrd recording file. None = no recording.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py index bbb684df68c..930cde03bcd 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer.py @@ -19,145 +19,58 @@ class Visualizer(ABC): """Base class for all visualizer backends. - Visualizers are used for debug visualization and monitoring during simulation, - separate from rendering for sensors/cameras. Each visualizer backend (Newton OpenGL, - Omniverse, Rerun) should inherit from this class and implement the required methods. - - The visualizer lifecycle follows this pattern: - 1. __init__: Create the visualizer with configuration - 2. initialize: Set up the visualizer with the simulation model/scene - 3. step: Update the visualizer each simulation step - 4. close: Clean up resources when done - - Args: - cfg: Configuration for the visualizer backend. + Lifecycle: __init__() -> initialize() -> step() (repeated) -> close() """ def __init__(self, cfg: VisualizerCfg): - """Initialize the visualizer with configuration. - - Args: - cfg: Configuration for the visualizer backend. - """ + """Initialize visualizer with config.""" self.cfg = cfg self._is_initialized = False self._is_closed = False @abstractmethod def initialize(self, scene_data: dict[str, Any] | None = None) -> None: - """Initialize the visualizer with the simulation scene. - - This method is called once after the simulation scene is created and before - the simulation starts. It should set up any necessary resources for visualization. - - Args: - scene_data: Optional dictionary containing initial scene data. The contents - depend on what's available at initialization time. May include: - - "model": Physics model object - - "state": Initial physics state - - "usd_stage": USD stage - The visualizer should handle None or missing keys gracefully. - """ + """Initialize visualizer with scene data (model, state, usd_stage, etc.).""" pass @abstractmethod def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: - """Update the visualizer for one simulation step. - - This method is called each simulation step to update the visualization. - The visualizer should pull any needed data from the scene_provider. - - Args: - dt: Time step in seconds since last visualization update. - scene_provider: Provider for accessing current scene data (physics state, USD stage, etc.). - Visualizers should query this for updated data rather than directly - accessing physics managers. May be None if no scene data is available yet. - """ + """Update visualization for one step.""" pass @abstractmethod def close(self) -> None: - """Close the visualizer and clean up resources. - - This method is called when the simulation is ending or the visualizer - is no longer needed. It should release any resources held by the visualizer. - """ + """Clean up resources.""" pass @abstractmethod def is_running(self) -> bool: - """Check if the visualizer is still running. - - Returns: - True if the visualizer is running and should continue receiving updates, - False if it has been closed (e.g., user closed the window). - """ + """Check if visualizer is still running (e.g., window not closed).""" pass def is_training_paused(self) -> bool: - """Check if training/simulation is paused by the visualizer. - - Some visualizers (like Newton OpenGL) provide controls to pause the simulation - while keeping the visualizer running. This is useful for debugging. - - Returns: - True if the user has paused training/simulation, False otherwise. - Default implementation returns False (no pause support). - """ + """Check if training is paused by visualizer controls.""" return False def is_rendering_paused(self) -> bool: - """Check if rendering is paused by the visualizer. - - Some visualizers allow pausing rendering while keeping simulation running, - which can improve performance during training. - - Returns: - True if rendering is paused, False otherwise. - Default implementation returns False (no pause support). - """ + """Check if rendering is paused by visualizer controls.""" return False @property def is_initialized(self) -> bool: - """Check if the visualizer has been initialized. - - Returns: - True if initialize() has been called successfully. - """ + """Check if initialize() has been called.""" return self._is_initialized @property def is_closed(self) -> bool: - """Check if the visualizer has been closed. - - Returns: - True if close() has been called. - """ + """Check if close() has been called.""" return self._is_closed def supports_markers(self) -> bool: - """Check if this visualizer supports visualization markers. - - Visualization markers are geometric shapes (spheres, arrows, frames, etc.) - used for debug visualization. They are typically managed by the scene/environment - but rendered by the visualizer. - - Returns: - True if the visualizer can display VisualizationMarkers, False otherwise. - Default implementation returns False. - """ + """Check if visualizer supports VisualizationMarkers.""" return False def supports_live_plots(self) -> bool: - """Check if this visualizer supports live plots. - - Live plots display time-series data (observations, rewards, etc.) in real-time - via UI widgets. They are typically managed by manager-based environments. - - Returns: - True if the visualizer can display live plots, False otherwise. - Default implementation returns False. - """ + """Check if visualizer supports LivePlots.""" return False diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py index 76d54673105..5d494f1839b 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py @@ -17,108 +17,43 @@ @configclass class VisualizerCfg: - """Base configuration for visualizer backends. - - This is the base class for all visualizer configurations. Visualizers are used - for debug visualization and monitoring during simulation, separate from rendering - for sensors/cameras. - - All visualizer backends should inherit from this class and add their specific - configuration parameters. - """ + """Base configuration for all visualizer backends.""" visualizer_type: str = "base" - """Type identifier for this visualizer (e.g., 'newton', 'rerun', 'omniverse'). - - This is used by the factory pattern to instantiate the correct visualizer class. - Subclasses should override this with their specific type identifier. - """ + """Type identifier (e.g., 'newton', 'rerun', 'omniverse').""" enabled: bool = False - """Whether the visualizer is enabled. Default is False.""" + """Whether the visualizer is enabled.""" update_frequency: int = 1 - """Frequency of updates to the visualizer (in simulation steps). - - Higher values (e.g., 10) mean the visualizer updates less frequently, improving - performance at the cost of less responsive visualization. Lower values (e.g., 1) - provide more responsive visualization but may impact performance. - Default is 1 (update every step). - """ + """Update frequency in simulation steps (1 = every step).""" env_indices: list[int] | None = None - """List of environment indices to visualize. Default is None. - - If None, all environments will be visualized. If a list is provided, only the - specified environments will be visualized. This is useful for reducing the - visualization overhead when running with many environments. - - Example: [0, 1, 2] will visualize only the first 3 environments. - """ + """Environment indices to visualize. None = all environments.""" enable_markers: bool = True - """Whether to enable visualization markers (debug drawing). Default is True. - - Visualization markers are used for debug drawing of points, lines, frames, etc. - These correspond to the VisualizationMarkers class in isaaclab.markers. - """ + """Enable visualization markers (debug drawing).""" enable_live_plots: bool = True - """Whether to enable live plotting of data. Default is True. - - Live plots can be used to visualize real-time data such as observations, - rewards, and other metrics during simulation. - """ - - train_mode: bool = True - """Whether the visualizer is in training mode (True) or play/inference mode (False). - - This affects the UI and controls displayed in the visualizer. In training mode, - additional controls may be shown for pausing training, adjusting update frequency, etc. - Default is True. - """ + """Enable live plotting of data.""" camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) - """Initial position of the camera in the visualizer. Default is (10.0, 0.0, 3.0).""" + """Initial camera position (x, y, z).""" camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Initial target (look-at point) of the camera. Default is (0.0, 0.0, 0.0).""" - - window_width: int = 1920 - """Width of the visualizer window in pixels. Default is 1920.""" - - window_height: int = 1080 - """Height of the visualizer window in pixels. Default is 1080.""" + """Initial camera target/look-at point (x, y, z).""" def get_visualizer_type(self) -> str: - """Get the type of visualizer as a string. - - Returns: - String identifier for the visualizer type. - """ + """Get the visualizer type identifier.""" return self.visualizer_type def create_visualizer(self) -> Visualizer: - """Factory method to create a visualizer instance from this configuration. - - This method uses the visualizer registry to instantiate the appropriate - visualizer class based on the `visualizer_type` field. - - Returns: - Visualizer instance configured with this config. - - Raises: - ValueError: If the visualizer type is not registered. - """ - # Import here to avoid circular imports + """Create visualizer instance from this config using factory pattern.""" from . import get_visualizer_class visualizer_class = get_visualizer_class(self.visualizer_type) if visualizer_class is None: - raise ValueError( - f"Visualizer type '{self.visualizer_type}' is not registered. " - f"Make sure the visualizer module is imported." - ) + raise ValueError(f"Visualizer type '{self.visualizer_type}' is not registered.") return visualizer_class(self) diff --git a/source/isaaclab/setup.py b/source/isaaclab/setup.py index f487b388bde..c332f1468c0 100644 --- a/source/isaaclab/setup.py +++ b/source/isaaclab/setup.py @@ -50,12 +50,12 @@ "usd-core==25.05.0", "mujoco>=3.3.8.dev821851540", "mujoco-warp @ git+https://github.com/google-deepmind/mujoco_warp.git@bbd757cace561de47512b560517ee728c8416de5", - "newton @ git+https://github.com/newton-physics/newton.git@15b9955bafa61f8fcb40c17dc00f0b552d3c65ca", + # "newton @ git+https://github.com/newton-physics/newton.git@15b9955bafa61f8fcb40c17dc00f0b552d3c65ca", + "newton @ git+https://github.com/newton-physics/newton.git@c4baa06c3e8ea0a3090037b2b197e9aa453265f1", "imgui-bundle==1.92.0", "PyOpenGL-accelerate==3.1.10", ] - # Additional dependencies that are only available on Linux platforms if platform.system() == "Linux": INSTALL_REQUIRES += [ From 95d9429ce23feaa9625acb8b0a0b0e25927ca682 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Fri, 14 Nov 2025 18:29:45 -0800 Subject: [PATCH 9/9] clean --- .../sim/visualizers/newton_visualizer.py | 51 +--------------- .../sim/visualizers/newton_visualizer_cfg.py | 6 ++ .../isaaclab/sim/visualizers/ov_visualizer.py | 59 +++++++++---------- .../sim/visualizers/ov_visualizer_cfg.py | 14 +++-- .../sim/visualizers/rerun_visualizer.py | 36 ++++++----- .../sim/visualizers/rerun_visualizer_cfg.py | 4 +- .../sim/visualizers/visualizer_cfg.py | 9 --- 7 files changed, 70 insertions(+), 109 deletions(-) diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py index 6314e6744f3..590b21b13dd 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer.py @@ -34,7 +34,6 @@ def __init__(self, *args, metadata: dict | None = None, **kwargs): self._paused_training: bool = False self._paused_rendering: bool = False self._fallback_draw_controls: bool = False - self._visualizer_update_frequency: int = 1 self._metadata = metadata or {} try: @@ -50,22 +49,6 @@ def is_rendering_paused(self) -> bool: """Check if rendering is paused.""" return self._paused_rendering - def set_visualizer_update_frequency(self, frequency: int) -> None: - """Set the visualizer update frequency. - - Args: - frequency: Number of simulation steps between visualizer updates. - """ - self._visualizer_update_frequency = max(1, frequency) - - def get_visualizer_update_frequency(self) -> int: - """Get the current visualizer update frequency. - - Returns: - Number of simulation steps between visualizer updates. - """ - return self._visualizer_update_frequency - # UI callback rendered inside the "Example Options" panel of the left sidebar def _render_training_controls(self, imgui): imgui.separator() @@ -79,21 +62,6 @@ def _render_training_controls(self, imgui): if imgui.button(rendering_label): self._paused_rendering = not self._paused_rendering - imgui.text("Visualizer Update Frequency") - - current_frequency = self._visualizer_update_frequency - changed, new_frequency = imgui.slider_int( - "##VisualizerUpdateFreq", current_frequency, 1, 20, f"Every {current_frequency} frames" - ) - if changed: - self._visualizer_update_frequency = new_frequency - - if imgui.is_item_hovered(): - imgui.set_tooltip( - "Controls visualizer update frequency\nlower values-> more responsive visualizer but slower" - " training\nhigher values-> less responsive visualizer but faster training" - ) - # Override only SPACE key to use rendering pause, preserve all other shortcuts def on_key_press(self, symbol, modifiers): if self.ui.is_capturing(): @@ -341,24 +309,13 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: self._viewer.renderer.sky_lower = self.cfg.ground_color self._viewer.renderer._light_color = self.cfg.light_color - # Set update frequency - self._viewer.set_visualizer_update_frequency(self.cfg.update_frequency) - self._is_initialized = True def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> None: """Update the visualizer for one simulation step. - Args: - dt: Time step in seconds since last visualization update. - scene_provider: Provider for accessing current scene data. The visualizer - will pull the latest Newton state from this provider. - - Note: - Pause handling (training and rendering) is managed by SimulationContext. - This method only performs the actual rendering when called. - The visualizer MUST be called every frame to maintain proper ImGui state, - even if we skip rendering some frames based on update_frequency. + Note: The visualizer MUST be called every frame to maintain proper ImGui state. + Pause handling (training and rendering) is managed by SimulationContext. """ if not self._is_initialized or self._is_closed or self._viewer is None: return @@ -370,9 +327,7 @@ def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> No if scene_provider is not None: self._state = scene_provider.get_state() - # Render the current frame - # Note: We always call begin_frame/end_frame to maintain ImGui state - # The update frequency is handled internally by the viewer + # Render the current frame (always call to maintain ImGui state) try: self._viewer.begin_frame(self._sim_time) try: diff --git a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py index ccbd183ac80..eeca2a53736 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/newton_visualizer_cfg.py @@ -31,6 +31,12 @@ class NewtonVisualizerCfg(VisualizerCfg): window_height: int = 1080 """Window height in pixels.""" + camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) + """Initial camera position (x, y, z).""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Initial camera target/look-at point (x, y, z).""" + # Newton-specific settings fps: int = 60 """Target FPS.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py index 5826b944c0a..ae83ae90be8 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer.py @@ -148,43 +148,35 @@ def _ensure_simulation_app(self) -> None: def _setup_viewport(self, usd_stage, metadata: dict) -> None: """Setup viewport with camera and window size.""" try: - from omni.kit.viewport.utility import get_active_viewport, create_viewport_window - import omni.ui as ui + import omni.kit.viewport.utility as vp_utils # Create new viewport or use existing if self.cfg.create_viewport and self.cfg.viewport_name: - # Create new viewport - self._viewport_window = create_viewport_window( - title=self.cfg.viewport_name, + # Create new viewport with proper API + self._viewport_window = vp_utils.create_viewport_window( + name=self.cfg.viewport_name, width=self.cfg.window_width, height=self.cfg.window_height, + position_x=50, # Window position on screen + position_y=50, ) - # Make viewport visible in UI - if self._viewport_window: - self._viewport_window.visible = True - self._viewport_window.docked = False omni.log.info(f"[OVVisualizer] Created viewport '{self.cfg.viewport_name}'") else: - # Use existing viewport - self._viewport_window = get_active_viewport() - # Try to resize if window API is available - if self._viewport_window and hasattr(self._viewport_window, 'width'): - try: - self._viewport_window.width = self.cfg.window_width - self._viewport_window.height = self.cfg.window_height - except: - pass + # Use existing active viewport + self._viewport_window = vp_utils.get_active_viewport_window() + omni.log.info("[OVVisualizer] Using existing active viewport") if self._viewport_window is None: omni.log.warn("[OVVisualizer] Could not get/create viewport.") return + # Get viewport API for camera control self._viewport_api = self._viewport_window.viewport_api - # Set camera + # Set camera pose using Isaac Sim utility self._set_viewport_camera(self.cfg.camera_position, self.cfg.camera_target) - omni.log.info("[OVVisualizer] Viewport configured.") + omni.log.info(f"[OVVisualizer] Viewport configured (size: {self.cfg.window_width}x{self.cfg.window_height})") except ImportError as e: omni.log.warn(f"[OVVisualizer] Viewport utilities unavailable: {e}") @@ -196,25 +188,28 @@ def _set_viewport_camera( position: tuple[float, float, float], target: tuple[float, float, float] ) -> None: - """Set viewport camera position and target.""" + """Set viewport camera position and target using Isaac Sim utilities.""" if self._viewport_api is None: return try: - from pxr import Gf + # Import Isaac Sim viewport utilities + import isaacsim.core.utils.viewports as vp_utils - eye = Gf.Vec3d(*position) - target_pos = Gf.Vec3d(*target) - up = Gf.Vec3d(0, 0, 1) + # Get the camera prim path for this viewport + camera_path = self._viewport_api.get_active_camera() + if not camera_path: + camera_path = "/OmniverseKit_Persp" # Default camera - # Try viewport API methods - if hasattr(self._viewport_api, 'set_view'): - self._viewport_api.set_view(eye, target_pos, up) - elif hasattr(self._viewport_window, 'set_camera_position'): - self._viewport_window.set_camera_position(*position, True) - self._viewport_window.set_camera_target(*target, True) + # Use Isaac Sim utility to set camera view + vp_utils.set_camera_view( + eye=list(position), + target=list(target), + camera_prim_path=camera_path, + viewport_api=self._viewport_api + ) - omni.log.info(f"[OVVisualizer] Camera: pos={position}, target={target}") + omni.log.info(f"[OVVisualizer] Camera set: pos={position}, target={target}, camera={camera_path}") except Exception as e: omni.log.warn(f"[OVVisualizer] Could not set camera: {e}") diff --git a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py index b107f1cd0d4..c75328028fc 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/ov_visualizer_cfg.py @@ -21,18 +21,24 @@ class OVVisualizerCfg(VisualizerCfg): visualizer_type: str = "omniverse" """Type identifier for Omniverse visualizer.""" - viewport_name: str | None = "/OmniverseKit/Viewport" + viewport_name: str | None = "Goldenstate" # "/OmniverseKit/Viewport" """Viewport name to use. If None, uses active viewport.""" - create_viewport: bool = False + create_viewport: bool = True #False """Create new viewport with specified name and camera pose.""" - window_width: int = 1920 + window_width: int = 777 # 1920 """Viewport width in pixels.""" - window_height: int = 1080 + window_height: int = 777 # 1080 """Viewport height in pixels.""" + camera_position: tuple[float, float, float] = (10.0, 10.0, 10.0) + """Initial camera position (x, y, z).""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Initial camera target/look-at point (x, y, z).""" + launch_app_if_missing: bool = True """Launch Isaac Sim if not already running.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py index eaa23fea369..a6438e76a38 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer.py @@ -60,9 +60,9 @@ def __init__( address=address, launch_viewer=launch_viewer, app_id=app_id, - keep_historical_data=keep_historical_data, - keep_scalar_history=keep_scalar_history, - record_to_rrd=record_to_rrd, + # keep_historical_data=keep_historical_data, + # keep_scalar_history=keep_scalar_history, + # record_to_rrd=record_to_rrd, ) # Isaac Lab state @@ -117,7 +117,7 @@ def register_markers(self, markers: Any) -> None: Args: markers: VisualizationMarkers instance to log each frame. """ - if self._visualize_markers: + if self._enable_markers: self._registered_markers.append(markers) omni.log.info(f"[RerunVisualizer] Registered markers: {markers}") @@ -127,7 +127,7 @@ def register_plots(self, plots: dict[str, Any]) -> None: Args: plots: Dictionary mapping plot names to LivePlot instances. """ - if self._visualize_plots: + if self._enable_live_plots: self._registered_plots.update(plots) omni.log.info(f"[RerunVisualizer] Registered {len(plots)} plot(s)") @@ -151,7 +151,7 @@ def log_markers(self) -> None: - Optimize batch logging for large marker counts - Add color/material support for better visual distinction """ - if not self._visualize_markers or len(self._registered_markers) == 0: + if not self._enable_markers or len(self._registered_markers) == 0: return try: @@ -190,7 +190,7 @@ def log_plot_data(self) -> None: - Maintain proper timeline synchronization - Support different plot types (line, bar, etc.) """ - if not self._visualize_plots or len(self._registered_plots) == 0: + if not self._enable_live_plots or len(self._registered_plots) == 0: return try: @@ -313,6 +313,9 @@ def initialize(self, scene_data: dict[str, Any] | None = None) -> None: # Create Newton ViewerRerun wrapper try: + if self.cfg.record_to_rrd: + omni.log.info(f"[RerunVisualizer] Recording enabled to: {self.cfg.record_to_rrd}") + self._viewer = NewtonViewerRerun( server=self.cfg.server_mode, address=self.cfg.server_address, @@ -389,11 +392,11 @@ def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> No self._viewer.log_state(self._state) # Actively log markers (if enabled) - if self.cfg.visualize_markers: + if self.cfg.enable_markers: self._viewer.log_markers() # Actively log plot data (if enabled) - if self.cfg.visualize_plots: + if self.cfg.enable_live_plots: self._viewer.log_plot_data() # End frame @@ -403,17 +406,22 @@ def step(self, dt: float, scene_provider: SceneDataProvider | None = None) -> No self._sim_time += dt def close(self) -> None: - """Clean up Rerun visualizer resources. - - This closes the Rerun viewer, disconnects from the server, and - finalizes any recording files. - """ + """Clean up Rerun visualizer resources and finalize recordings.""" if not self._is_initialized or self._viewer is None: return try: + if self.cfg.record_to_rrd: + omni.log.info(f"[RerunVisualizer] Finalizing recording to: {self.cfg.record_to_rrd}") self._viewer.close() omni.log.info("[RerunVisualizer] Closed successfully.") + if self.cfg.record_to_rrd: + import os + if os.path.exists(self.cfg.record_to_rrd): + size = os.path.getsize(self.cfg.record_to_rrd) + omni.log.info(f"[RerunVisualizer] Recording saved: {self.cfg.record_to_rrd} ({size} bytes)") + else: + omni.log.warn(f"[RerunVisualizer] Recording file not found: {self.cfg.record_to_rrd}") except Exception as e: omni.log.warn(f"[RerunVisualizer] Error during close: {e}") diff --git a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py index 77b3716d036..f722ee4a746 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/rerun_visualizer_cfg.py @@ -35,12 +35,12 @@ class RerunVisualizerCfg(VisualizerCfg): app_id: str = "isaaclab-simulation" """Application identifier shown in viewer title.""" - keep_historical_data: bool = False + keep_historical_data: bool = True """Keep transform history for time scrubbing (False = constant memory for training).""" keep_scalar_history: bool = True """Keep scalar/plot history in timeline.""" - record_to_rrd: str | None = test.rrd + record_to_rrd: str | None = "/tmp/test.rrd" """Path to save .rrd recording file. None = no recording.""" diff --git a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py index 5d494f1839b..afc618780c7 100644 --- a/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/sim/visualizers/visualizer_cfg.py @@ -25,9 +25,6 @@ class VisualizerCfg: enabled: bool = False """Whether the visualizer is enabled.""" - update_frequency: int = 1 - """Update frequency in simulation steps (1 = every step).""" - env_indices: list[int] | None = None """Environment indices to visualize. None = all environments.""" @@ -37,12 +34,6 @@ class VisualizerCfg: enable_live_plots: bool = True """Enable live plotting of data.""" - camera_position: tuple[float, float, float] = (10.0, 0.0, 3.0) - """Initial camera position (x, y, z).""" - - camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Initial camera target/look-at point (x, y, z).""" - def get_visualizer_type(self) -> str: """Get the visualizer type identifier.""" return self.visualizer_type