[wip] Add initial design for Visualizers

source/isaaclab/isaaclab/sim/_impl/newton_manager.py

@@ -16,7 +16,6 @@
 from newton.solvers import SolverBase, SolverFeatherstone, SolverMuJoCo, SolverXPBD
 
 from isaaclab.sim._impl.newton_manager_cfg import NewtonCfg
-from isaaclab.sim._impl.newton_viewer import NewtonViewerGL
 from isaaclab.utils.timer import Timer
 
 
@@ -66,7 +65,6 @@ class NewtonManager:
     _report_contacts: bool = False
     _graph = None
     _newton_stage_path = None
-    _renderer = None
     _sim_time = 0.0
     _usdrt_stage = None
     _newton_index_attr = "newton:index"
@@ -76,10 +74,6 @@ class NewtonManager:
     _gravity_vector: tuple[float, float, float] = (0.0, 0.0, -9.81)
     _up_axis: str = "Z"
     _num_envs: int = None
-    _visualizer_update_counter: int = 0
-    _visualizer_update_frequency: int = 1  # Configurable frequency for all rendering updates
-    _visualizer_train_mode: bool = True  # Whether visualizer is in training mode
-    _visualizer_disabled: bool = False  # Whether visualizer has been disabled by user
 
     @classmethod
     def clear(cls):
@@ -95,17 +89,13 @@ def clear(cls):
         NewtonManager._report_contacts = False
         NewtonManager._graph = None
         NewtonManager._newton_stage_path = None
-        NewtonManager._renderer = None
         NewtonManager._sim_time = 0.0
         NewtonManager._on_init_callbacks = []
         NewtonManager._on_start_callbacks = []
         NewtonManager._usdrt_stage = None
         NewtonManager._cfg = NewtonCfg()
         NewtonManager._up_axis = "Z"
         NewtonManager._first_call = True
-        NewtonManager._visualizer_update_counter = 0
-        NewtonManager._visualizer_disabled = False
-        NewtonManager._visualizer_update_frequency = NewtonManager._cfg.newton_viewer_update_frequency
 
     @classmethod
     def set_builder(cls, builder):
@@ -302,80 +292,6 @@ def set_simulation_dt(cls, dt: float) -> None:
         """
         NewtonManager._dt = dt
 
-    @classmethod
-    def _render_call(cls, render_func) -> bool:
-        if NewtonManager._renderer is not None:
-            try:
-                if hasattr(NewtonManager._renderer, "renderer") and hasattr(NewtonManager._renderer.renderer, "window"):
-                    if NewtonManager._renderer.renderer.window.has_exit:
-                        NewtonManager._visualizer_disabled = True
-                        NewtonManager._renderer = None
-                        return False
-            except Exception as e:
-                print(f"[ERROR] Error in _render_call: {e}")
-
-        try:
-            render_func()
-            return True
-        except (ctypes.ArgumentError, Exception) as e:
-            if "wrong type" in str(e) or "ArgumentError" in str(e):
-                NewtonManager._visualizer_disabled = True
-                if NewtonManager._renderer is not None:
-                    try:
-                        NewtonManager._renderer.close()
-                    except Exception as e:
-                        print(f"[ERROR] Error in _render_call: {e}")
-                    NewtonManager._renderer = None
-                return False
-            else:
-                raise
-
-    @classmethod
-    def render(cls) -> None:
-        if NewtonManager._visualizer_disabled:
-            return
-
-        if NewtonManager._renderer is None:
-            NewtonManager._visualizer_train_mode = NewtonManager._cfg.visualizer_train_mode
-            NewtonManager._renderer = NewtonViewerGL(
-                width=1280, height=720, train_mode=NewtonManager._visualizer_train_mode
-            )
-            NewtonManager._renderer.set_model(NewtonManager._model)
-            NewtonManager._renderer.camera.pos = wp.vec3(*NewtonManager._cfg.newton_viewer_camera_pos)
-            NewtonManager._renderer.up_axis = NewtonManager._up_axis
-            NewtonManager._renderer.scaling = 1.0
-            NewtonManager._renderer._paused = False
-        else:
-            while NewtonManager._renderer is not None and NewtonManager._renderer.is_training_paused():
-
-                def render_frame():
-                    NewtonManager._renderer.begin_frame(NewtonManager._sim_time)
-                    NewtonManager._renderer.log_state(NewtonManager._state_0)
-                    NewtonManager._renderer.end_frame()
-
-                if not NewtonManager._render_call(render_frame):
-                    return
-
-            NewtonManager._visualizer_update_counter += 1
-            if (
-                NewtonManager._renderer is not None
-                and NewtonManager._visualizer_update_counter >= NewtonManager._visualizer_update_frequency
-            ):
-                if not NewtonManager._renderer.is_paused():
-
-                    def render_frame():
-                        NewtonManager._renderer.begin_frame(NewtonManager._sim_time)
-                        NewtonManager._renderer.log_state(NewtonManager._state_0)
-                        NewtonManager._renderer.end_frame()
-
-                    if not NewtonManager._render_call(render_frame):
-                        return
-                else:
-                    if not NewtonManager._render_call(lambda: NewtonManager._renderer._update()):
-                        return
-
-                NewtonManager._visualizer_update_counter = 0
-
     @classmethod
     def sync_fabric_transforms(cls) -> None:
         """Syncs the fabric transforms with the Newton state.

source/isaaclab/isaaclab/sim/_impl/newton_manager_cfg.py

@@ -12,7 +12,11 @@
 class NewtonCfg:
     """Configuration for Newton-related parameters.
 
-    These parameters are used to configure the Newton simulation.
+    These parameters are used to configure the Newton physics simulation.
+    
+    Note:
+        Visualizer-related settings have been moved to NewtonVisualizerCfg in
+        isaaclab.sim.visualizers.newton_visualizer_cfg.
     """
 
     num_substeps: int = 1
@@ -27,13 +31,4 @@ class NewtonCfg:
     If set to False, the simulation performance will be severely degraded.
     """
 
-    newton_viewer_update_frequency: int = 1
-    """Frequency of updates to the Newton viewer."""
-
-    newton_viewer_camera_pos: tuple[float, float, float] = (10.0, 0.0, 3.0)
-    """Position of the camera in the Newton viewer."""
-
-    visualizer_train_mode: bool = True
-    """Whether the visualizer is in training mode (True) or play mode (False)."""
-
     solver_cfg: NewtonSolverCfg = MJWarpSolverCfg()

source/isaaclab/isaaclab/sim/simulation_cfg.py

@@ -15,6 +15,7 @@
 
 from ._impl.newton_manager_cfg import NewtonCfg
 from .spawners.materials import RigidBodyMaterialCfg
+from .visualizers import VisualizerCfg
 
 
 @configclass
@@ -196,11 +197,35 @@ class SimulationCfg:
     render: RenderCfg = RenderCfg()
     """Render settings. Default is RenderCfg()."""
 
-    enable_newton_rendering: bool = False
-    """Enable/disable rendering using Newton. Default is False.
-
-    When enabled, the Newton to renderer will be called every time the simulation is rendered. If Isaac Sim's
-    renderer is also enabled, both will be called.
+    visualizers: list[VisualizerCfg] | VisualizerCfg | None = None
+    """Visualizer settings. Default is None (no visualizer).
+
+    This field supports multiple visualizer backends for debug visualization and monitoring
+    during simulation. It accepts:
+    - A single VisualizerCfg: One visualizer will be created
+    - A list of VisualizerCfg: Multiple visualizers will be created
+    - None or empty list: No visualizers will be created
+
+    Supported visualizer backends:
+    - NewtonVisualizerCfg: Lightweight OpenGL-based visualizer
+    - OVVisualizerCfg: Omniverse-based high-fidelity visualizer  
+    - RerunVisualizerCfg: Web-based Rerun visualizer
+
+    Examples:
+        # Single visualizer
+        from isaaclab.sim.visualizers import NewtonVisualizerCfg
+        cfg = SimulationCfg(visualizers=NewtonVisualizerCfg(enabled=True))
+
+        # Multiple visualizers
+        from isaaclab.sim.visualizers import NewtonVisualizerCfg, RerunVisualizerCfg
+        cfg = SimulationCfg(visualizers=[
+            NewtonVisualizerCfg(enabled=True),
+            RerunVisualizerCfg(enabled=True)
+        ])
+
+    Note:
+        Visualizers are separate from rendering backends (for cameras/sensors).
+        They are intended for debug visualization and monitoring only.
     """
 
     create_stage_in_memory: bool = False

source/isaaclab/isaaclab/sim/simulation_context.py

@@ -35,6 +35,7 @@
 from .simulation_cfg import SimulationCfg
 from .spawners import DomeLightCfg, GroundPlaneCfg
 from .utils import bind_physics_material
+from .visualizers import Visualizer, NewtonVisualizer, NewtonVisualizerCfg
 
 
 class SimulationContext(_SimulationContext):
@@ -252,6 +253,10 @@ def __init__(self, cfg: SimulationCfg | None = None):
             self._app_control_on_stop_handle = None
         self._disable_app_control_on_stop_handle = False
 
+        # initialize visualizers
+        self._visualizers: list[Visualizer] = []
+        self._visualizer_step_counter = 0
+
         # flag for skipping prim deletion callback
         # when stage in memory is attached
         self._skip_next_prim_deletion_callback_fn = False
@@ -538,6 +543,118 @@ def forward(self) -> None:
         NewtonManager.forward_kinematics()
         NewtonManager.sync_fabric_transforms()
 
+    def initialize_visualizers(self) -> None:
+        """Initialize all configured visualizers.
+
+        This method creates and initializes visualizers based on the configuration provided
+        in SimulationCfg.visualizers. It supports:
+        - A single VisualizerCfg: Creates one visualizer
+        - A list of VisualizerCfg: Creates multiple visualizers
+        - None or empty list: No visualizers are created
+        """
+        # Handle different input formats
+        visualizer_cfgs = []
+        if self.cfg.visualizers is not None:
+            if isinstance(self.cfg.visualizers, list):
+                visualizer_cfgs = [cfg for cfg in self.cfg.visualizers if cfg.enabled]
+            elif self.cfg.visualizers.enabled:
+                visualizer_cfgs = [self.cfg.visualizers]
+
+        # Create and initialize each visualizer
+        for viz_cfg in visualizer_cfgs:
+            try:
+                # Create visualizer instance based on config type
+                if isinstance(viz_cfg, NewtonVisualizerCfg):
+                    visualizer = NewtonVisualizer(viz_cfg)
+                else:
+                    # Skip unsupported visualizer types for now
+                    omni.log.warn(
+                        f"Visualizer type '{type(viz_cfg).__name__}' is not yet implemented. Skipping."
+                    )
+                    continue
+
+                # Initialize with Newton model if available
+                if NewtonManager._model is not None:
+                    scene = {
+                        "model": NewtonManager._model,
+                        "state": NewtonManager._state_0,
+                    }
+                    visualizer.initialize(scene)
+                    self._visualizers.append(visualizer)
+                    omni.log.info(f"Initialized visualizer: {type(visualizer).__name__}")
+                else:
+                    omni.log.warn(
+                        "Newton model not available yet. Visualizer will be initialized later."
+                    )
+
+            except Exception as e:
+                omni.log.error(f"Failed to initialize visualizer '{type(viz_cfg).__name__}': {e}")
+
+    def step_visualizers(self, dt: float) -> None:
+        """Update all active visualizers.
+
+        This method steps all initialized visualizers and updates their state.
+        It also handles visualizer pause states and removes closed visualizers.
+
+        Args:
+            dt: Time step in seconds.
+        """
+        if not self._visualizers:
+            return
+
+        self._visualizer_step_counter += 1
+
+        # Update visualizers and check if any should be removed
+        visualizers_to_remove = []
+
+        for visualizer in self._visualizers:
+            try:
+                # Check if visualizer is still running
+                if not visualizer.is_running():
+                    visualizers_to_remove.append(visualizer)
+                    continue
+
+                # Handle training pause - block until resumed
+                while visualizer.is_training_paused() and visualizer.is_running():
+                    if isinstance(visualizer, NewtonVisualizer):
+                        # Update state before rendering during pause
+                        visualizer.update_state(NewtonManager._state_0)
+                    visualizer.step(0.0)  # Step with 0 dt during pause
+
+                # Skip rendering if visualizer has rendering paused
+                if visualizer.is_rendering_paused():
+                    continue
+
+                # Normal step: update state and visualizer
+                if isinstance(visualizer, NewtonVisualizer):
+                    visualizer.update_state(NewtonManager._state_0)
+
+                visualizer.step(dt)
+
+            except Exception as e:
+                omni.log.error(f"Error stepping visualizer '{type(visualizer).__name__}': {e}")
+                visualizers_to_remove.append(visualizer)
+
+        # Remove closed visualizers
+        for visualizer in visualizers_to_remove:
+            try:
+                visualizer.close()
+                self._visualizers.remove(visualizer)
+                omni.log.info(f"Removed visualizer: {type(visualizer).__name__}")
+            except Exception as e:
+                omni.log.error(f"Error closing visualizer: {e}")
+
+    def close_visualizers(self) -> None:
+        """Close all active visualizers and clean up resources."""
+        for visualizer in self._visualizers:
+            try:
+                visualizer.close()
+            except Exception as e:
+                omni.log.error(f"Error closing visualizer '{type(visualizer).__name__}': {e}")
+
+        self._visualizers.clear()
+        omni.log.info("All visualizers closed")
+
     def get_initial_stage(self) -> Usd.Stage:
         """Returns stage handle used during scene creation.
 
@@ -577,6 +694,11 @@ def reset(self, soft: bool = False):
         if not soft:
             for _ in range(2):
                 self.render()
+
+        # Initialize visualizers after simulation is set up (only on first reset)
+        if not soft and not self._visualizers:
+            self.initialize_visualizers()
+
         self._disable_app_control_on_stop_handle = False
 
     def step(self, render: bool = True):
@@ -628,9 +750,8 @@ def step(self, render: bool = True):
             if self.is_playing():
                 NewtonManager.step()
 
-        # Use the NewtonManager to render the scene if enabled
-        if self.cfg.enable_newton_rendering:
-            NewtonManager.render()
+        # Update visualizers
+        self.step_visualizers(self.cfg.dt)
 
         # app.update() may be changing the cuda device in step, so we force it back to our desired device here
         if "cuda" in self.device:
@@ -728,6 +849,9 @@ def clear_instance(cls):
             if cls._instance._app_control_on_stop_handle is not None:
                 cls._instance._app_control_on_stop_handle.unsubscribe()
                 cls._instance._app_control_on_stop_handle = None
+            # close all visualizers
+            if hasattr(cls._instance, '_visualizers'):
+                cls._instance.close_visualizers()
         # call parent to clear the instance
         super().clear_instance()
         NewtonManager.clear()