Dev/hougantc/teleop controllers

CONTRIBUTORS.md

@@ -72,6 +72,7 @@ Guidelines for modifications:
 * HoJin Jeon
 * Hongwei Xiong
 * Hongyu Li
+* Hougant Chen
 * Huihua Zhao
 * Iretiayo Akinola
 * Jack Zeng

docs/source/how-to/cloudxr_teleoperation.rst

@@ -721,11 +721,11 @@ Here's an example of setting up hand tracking:
 
    # Create retargeters
    position_retargeter = Se3AbsRetargeter(
-       bound_hand=OpenXRDevice.TrackingTarget.HAND_RIGHT,
+       bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT,
        zero_out_xy_rotation=True,
        use_wrist_position=False  # Use pinch position (thumb-index midpoint) instead of wrist
    )
-   gripper_retargeter = GripperRetargeter(bound_hand=OpenXRDevice.TrackingTarget.HAND_RIGHT)
+   gripper_retargeter = GripperRetargeter(bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT)
 
    # Create OpenXR device with hand tracking and both retargeters
    device = OpenXRDevice(
@@ -919,34 +919,31 @@ The retargeting system is designed to be extensible. You can create custom retar
                Any: The transformed control commands for the robot.
            """
            # Access hand tracking data using TrackingTarget enum
-           right_hand_data = data[OpenXRDevice.TrackingTarget.HAND_RIGHT]
+           right_hand_data = data[DeviceBase.TrackingTarget.HAND_RIGHT]
 
            # Extract specific joint positions and orientations
            wrist_pose = right_hand_data.get("wrist")
            thumb_tip_pose = right_hand_data.get("thumb_tip")
            index_tip_pose = right_hand_data.get("index_tip")
 
            # Access head tracking data
-           head_pose = data[OpenXRDevice.TrackingTarget.HEAD]
+           head_pose = data[DeviceBase.TrackingTarget.HEAD]
 
            # Process the tracking data and apply your custom logic
            # ...
 
            # Return control commands in appropriate format
            return torch.tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0])  # Example output
 
-3. Register your retargeter with the factory by adding it to the ``RETARGETER_MAP``:
+3. Register your retargeter by setting ``retargeter_type`` on the config class:
 
 .. code-block:: python
 
    # Import your retargeter at the top of your module
    from my_package.retargeters import MyCustomRetargeter, MyCustomRetargeterCfg
 
-   # Add your retargeter to the factory
-   from isaaclab.devices.teleop_device_factory import RETARGETER_MAP
-
-   # Register your retargeter type with its constructor
-   RETARGETER_MAP[MyCustomRetargeterCfg] = MyCustomRetargeter
+   # Link the config to the implementation for factory construction
+   MyCustomRetargeterCfg.retargeter_type = MyCustomRetargeter
 
 4. Now you can use your custom retargeter in teleop device configurations:
 

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -3,7 +3,11 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Script to run a keyboard teleoperation with Isaac Lab manipulation environments."""
+"""Script to run teleoperation with Isaac Lab manipulation environments.
+
+Supports multiple input devices (e.g., keyboard, spacemouse, gamepad) and devices
+configured within the environment (including OpenXR-based hand tracking or motion
+controllers)."""
 
 """Launch Isaac Sim Simulator first."""
 
@@ -13,7 +17,7 @@
 from isaaclab.app import AppLauncher
 
 # add argparse arguments
-parser = argparse.ArgumentParser(description="Keyboard teleoperation for Isaac Lab environments.")
+parser = argparse.ArgumentParser(description="Teleoperation for Isaac Lab environments.")
 parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.")
 parser.add_argument(
     "--teleop_device",
@@ -76,7 +80,7 @@
 
 def main() -> None:
     """
-    Run keyboard teleoperation with Isaac Lab manipulation environment.
+    Run teleoperation with an Isaac Lab manipulation environment.
 
     Creates the environment, sets up teleoperation interfaces and callbacks,
     and runs the main simulation loop until the application is closed.
@@ -96,8 +100,6 @@ def main() -> None:
         env_cfg.terminations.object_reached_goal = DoneTerm(func=mdp.object_reached_goal)
 
     if args_cli.xr:
-        # External cameras are not supported with XR teleop
-        # Check for any camera configs and disable them
         env_cfg = remove_camera_configs(env_cfg)
         env_cfg.sim.render.antialiasing_mode = "DLSS"
 
@@ -200,7 +202,7 @@ def stop_teleoperation() -> None:
                 )
             else:
                 omni.log.error(f"Unsupported teleop device: {args_cli.teleop_device}")
-                omni.log.error("Supported devices: keyboard, spacemouse, gamepad, handtracking")
+                omni.log.error("Configure the teleop device in the environment config.")
                 env.close()
                 simulation_app.close()
                 return
@@ -250,6 +252,7 @@ def stop_teleoperation() -> None:
 
                 if should_reset_recording_instance:
                     env.reset()
+                    teleop_interface.reset()
                     should_reset_recording_instance = False
                     print("Environment reset complete")
         except Exception as e:

source/isaaclab/config/extension.toml

@@ -1,7 +1,7 @@
 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.47.4"
+version = "0.47.5"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/isaaclab/docs/CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 ---------
 
+0.47.5 (2025-10-30)
+~~~~~~~~~~~~~~~~~~~
+
+Changed
+^^^^^^^
+
+* Moved retargeter and device declaration out of factory and into the devices/retargeters themselves.
+
+
 0.47.4 (2025-10-30)
 ~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/devices/device_base.py

@@ -9,6 +9,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import Any
 
 from isaaclab.devices.retargeter_base import RetargeterBase, RetargeterCfg
@@ -18,8 +19,14 @@
 class DeviceCfg:
     """Configuration for teleoperation devices."""
 
+    # Whether teleoperation should start active by default
+    teleoperation_active_default: bool = True
+    # Torch device string to place output tensors on
     sim_device: str = "cpu"
+    # Retargeters that transform device data into robot commands
     retargeters: list[RetargeterCfg] = field(default_factory=list)
+    # Concrete device class to construct for this config. Set by each device module.
+    device_type: type["DeviceBase"] | None = None
 
 
 @dataclass
@@ -52,9 +59,13 @@ def __init__(self, retargeters: list[RetargeterBase] | None = None):
         """
         # Initialize empty list if None is provided
         self._retargeters = retargeters or []
+        # Aggregate required features across all retargeters
+        self._required_features = set()
+        for retargeter in self._retargeters:
+            self._required_features.update(retargeter.GetRequirements())
 
     def __str__(self) -> str:
-        """Returns: A string containing the information of joystick."""
+        """Returns: A string identifier for the device."""
         return f"{self.__class__.__name__}"
 
     """
@@ -117,3 +128,32 @@ def advance(self) -> torch.Tensor:
         # With multiple retargeters, return a tuple of outputs
         # Concatenate retargeted outputs into a single tensor
         return torch.cat([retargeter.retarget(raw_data) for retargeter in self._retargeters], dim=-1)
+
+    # -----------------------------
+    # Shared data layout helpers (for retargeters across devices)
+    # -----------------------------
+    class TrackingTarget(Enum):
+        """Standard tracking targets shared across devices."""
+
+        HAND_LEFT = 0
+        HAND_RIGHT = 1
+        HEAD = 2
+        CONTROLLER_LEFT = 3
+        CONTROLLER_RIGHT = 4
+
+    class MotionControllerDataRowIndex(Enum):
+        """Rows in the motion-controller 2x7 array."""
+
+        POSE = 0
+        INPUTS = 1
+
+    class MotionControllerInputIndex(Enum):
+        """Indices in the motion-controller input row."""
+
+        THUMBSTICK_X = 0
+        THUMBSTICK_Y = 1
+        TRIGGER = 2
+        SQUEEZE = 3
+        BUTTON_0 = 4
+        BUTTON_1 = 5
+        PADDING = 6

source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py

@@ -5,6 +5,8 @@
 
 """Gamepad controller for SE(2) control."""
 
+from __future__ import annotations
+
 import numpy as np
 import torch
 import weakref
@@ -18,16 +20,6 @@
 from ..device_base import DeviceBase, DeviceCfg
 
 
-@dataclass
-class Se2GamepadCfg(DeviceCfg):
-    """Configuration for SE2 gamepad devices."""
-
-    v_x_sensitivity: float = 1.0
-    v_y_sensitivity: float = 1.0
-    omega_z_sensitivity: float = 1.0
-    dead_zone: float = 0.01
-
-
 class Se2Gamepad(DeviceBase):
     r"""A gamepad controller for sending SE(2) commands as velocity commands.
 
@@ -209,3 +201,14 @@ def _resolve_command_buffer(self, raw_command: np.ndarray) -> np.ndarray:
         command[command_sign] *= -1
 
         return command
+
+
+@dataclass
+class Se2GamepadCfg(DeviceCfg):
+    """Configuration for SE2 gamepad devices."""
+
+    v_x_sensitivity: float = 1.0
+    v_y_sensitivity: float = 1.0
+    omega_z_sensitivity: float = 1.0
+    dead_zone: float = 0.01
+    device_type: type[DeviceBase] = Se2Gamepad

source/isaaclab/isaaclab/devices/gamepad/se3_gamepad.py

@@ -5,6 +5,8 @@
 
 """Gamepad controller for SE(3) control."""
 
+from __future__ import annotations
+
 import numpy as np
 import torch
 import weakref
@@ -18,17 +20,6 @@
 from ..device_base import DeviceBase, DeviceCfg
 
 
-@dataclass
-class Se3GamepadCfg(DeviceCfg):
-    """Configuration for SE3 gamepad devices."""
-
-    gripper_term: bool = True
-    dead_zone: float = 0.01  # For gamepad devices
-    pos_sensitivity: float = 1.0
-    rot_sensitivity: float = 1.6
-    retargeters: None = None
-
-
 class Se3Gamepad(DeviceBase):
     """A gamepad controller for sending SE(3) commands as delta poses and binary command (open/close).
 
@@ -264,3 +255,14 @@ def _resolve_command_buffer(self, raw_command: np.ndarray) -> np.ndarray:
         delta_command[delta_command_sign] *= -1
 
         return delta_command
+
+
+@dataclass
+class Se3GamepadCfg(DeviceCfg):
+    """Configuration for SE3 gamepad devices."""
+
+    gripper_term: bool = True
+    dead_zone: float = 0.01  # For gamepad devices
+    pos_sensitivity: float = 1.0
+    rot_sensitivity: float = 1.6
+    device_type: type[DeviceBase] = Se3Gamepad

source/isaaclab/isaaclab/devices/keyboard/se2_keyboard.py

@@ -5,6 +5,8 @@
 
 """Keyboard controller for SE(2) control."""
 
+from __future__ import annotations
+
 import numpy as np
 import torch
 import weakref
@@ -17,15 +19,6 @@
 from ..device_base import DeviceBase, DeviceCfg
 
 
-@dataclass
-class Se2KeyboardCfg(DeviceCfg):
-    """Configuration for SE2 keyboard devices."""
-
-    v_x_sensitivity: float = 0.8
-    v_y_sensitivity: float = 0.4
-    omega_z_sensitivity: float = 1.0
-
-
 class Se2Keyboard(DeviceBase):
     r"""A keyboard controller for sending SE(2) commands as velocity commands.
 
@@ -178,3 +171,13 @@ def _create_key_bindings(self):
             "NUMPAD_9": np.asarray([0.0, 0.0, -1.0]) * self.omega_z_sensitivity,
             "X": np.asarray([0.0, 0.0, -1.0]) * self.omega_z_sensitivity,
         }
+
+
+@dataclass
+class Se2KeyboardCfg(DeviceCfg):
+    """Configuration for SE2 keyboard devices."""
+
+    v_x_sensitivity: float = 0.8
+    v_y_sensitivity: float = 0.4
+    omega_z_sensitivity: float = 1.0
+    device_type: type[DeviceBase] = Se2Keyboard

source/isaaclab/isaaclab/devices/keyboard/se3_keyboard.py

@@ -5,6 +5,8 @@
 
 """Keyboard controller for SE(3) control."""
 
+from __future__ import annotations
+
 import numpy as np
 import torch
 import weakref
@@ -18,16 +20,6 @@
 from ..device_base import DeviceBase, DeviceCfg
 
 
-@dataclass
-class Se3KeyboardCfg(DeviceCfg):
-    """Configuration for SE3 keyboard devices."""
-
-    gripper_term: bool = True
-    pos_sensitivity: float = 0.4
-    rot_sensitivity: float = 0.8
-    retargeters: None = None
-
-
 class Se3Keyboard(DeviceBase):
     """A keyboard controller for sending SE(3) commands as delta poses and binary command (open/close).
 
@@ -206,3 +198,14 @@ def _create_key_bindings(self):
             "C": np.asarray([0.0, 0.0, 1.0]) * self.rot_sensitivity,
             "V": np.asarray([0.0, 0.0, -1.0]) * self.rot_sensitivity,
         }
+
+
+@dataclass
+class Se3KeyboardCfg(DeviceCfg):
+    """Configuration for SE3 keyboard devices."""
+
+    gripper_term: bool = True
+    pos_sensitivity: float = 0.4
+    rot_sensitivity: float = 0.8
+    retargeters: None = None
+    device_type: type[DeviceBase] = Se3Keyboard

source/isaaclab/isaaclab/devices/openxr/__init__.py

@@ -7,4 +7,4 @@
 
 from .manus_vive import ManusVive, ManusViveCfg
 from .openxr_device import OpenXRDevice, OpenXRDeviceCfg
-from .xr_cfg import XrCfg, remove_camera_configs
+from .xr_cfg import XrAnchorRotationMode, XrCfg, remove_camera_configs