From e83e59a71dd77e85d9ebabb0fbc70124a5c0d474 Mon Sep 17 00:00:00 2001 From: Mohammad Haghighipanah Date: Mon, 3 Nov 2025 15:04:29 -0800 Subject: [PATCH 1/5] added lerobot act notebook --- notebooks/lerobot_act/README.md | 64 ++ notebooks/lerobot_act/act_to_openvino.ipynb | 598 ++++++++++++++++++ .../lerobot_act/setup_unitree_lerobot_env.sh | 155 +++++ 3 files changed, 817 insertions(+) create mode 100644 notebooks/lerobot_act/README.md create mode 100644 notebooks/lerobot_act/act_to_openvino.ipynb create mode 100644 notebooks/lerobot_act/setup_unitree_lerobot_env.sh diff --git a/notebooks/lerobot_act/README.md b/notebooks/lerobot_act/README.md new file mode 100644 index 00000000000..f38f85bfadf --- /dev/null +++ b/notebooks/lerobot_act/README.md @@ -0,0 +1,64 @@ +# ACT Policy → OpenVINO IR Conversion (Notebook Guide) + +This README documents the current workflow implemented in `act_to_openvino.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. The notebook presently performs FP32 export (Model Optimizer invoked with FP16 compression flag but output standardized to `act_model_fp32.xml/bin`). + + +## Required Checkpoint Files (`act_checkpoint/`) +Place these next to the notebook: +* `model.safetensors` – ACT weights +* `config.json` – architecture + feature definitions +* `train_config.json` – optional (reproducibility record) +* `stats.json` – optional normalization statistics + +## Minimal Installation & Launch +Recommended (Conda environment): +```bash +conda create -n unitree_lerobot python=3.10 -y +conda activate unitree_lerobot +# Optional helper script if present in this folder +bash setup_unitree_lerobot_env.sh +# Launch notebook with correct kernel +jupyter lab act_to_openvino.ipynb --NotebookApp.kernel_name=unitree_lerobot +``` + +If you skip creating a dedicated environment, the dependency cell will install core packages (torch, openvino, nncf, etc.) into the current kernel. You MUST still install `lerobot` manually; the notebook will not auto‑install it. + + +## Key Configuration Variables +| Variable | Meaning | +|-------------------|----------------------------------------------------------| +| `CKPT_DIR` | Relative checkpoint folder (`act_checkpoint`) | +| `CHECKPOINT_PATH` | Path to `model.safetensors` (env‑overrideable) | +| `IR_OUTPUT_DIR` | Destination for `model.onnx` & IR artifacts | +| `STATS_PATH` | Path to `stats.json` if present | +| `PRECISIONS` | Currently `['FP32']` | +| `TARGET_DEVICE` | Default runtime device | + +## ONNX Export +Wrapper (`ONNXWrapper`) mirrors ACT forward usage by constructing a batch dict. Input ordering: +`observation.state`, each camera image (`observation.images.*`), `action_is_pad`, `action`, optional `observation.environment_state`. +* Output name: `output` +Exports only if `openvino_ir_outputs/model.onnx` does not already exist. + +## Model Optimizer Conversion +Executed command: +``` +mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --compress_to_fp16=False +``` +Artifacts are copied / renamed to: +* `act_model_fp32.xml` +* `act_model_fp32.bin` + +## Directory Layout After Successful Conversion +``` +act_to_openvino.ipynb +act_checkpoint/ + model.safetensors + config.json + train_config.json + stats.json (optional) +openvino_ir_outputs/ + model.onnx + act_model_fp32.xml + act_model_fp32.bin +``` \ No newline at end of file diff --git a/notebooks/lerobot_act/act_to_openvino.ipynb b/notebooks/lerobot_act/act_to_openvino.ipynb new file mode 100644 index 00000000000..dd8413725ac --- /dev/null +++ b/notebooks/lerobot_act/act_to_openvino.ipynb @@ -0,0 +1,598 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c1c0cf06", + "metadata": {}, + "source": [ + "# ACT Model to OpenVINO IR Conversion\n", + "\n", + "The Action Chunking Transformer (ACT) is a model that learns a generative model over action sequences for bimanual manipulation. See the original paper for details: [Action Chunking Transformer](https://arxiv.org/pdf/2304.13705).\n", + "\n", + "In this tutorial, we show how to convert a Unitree ACT policy (stored in the LeRobot format) to the OpenVINO Intermediate Representation (IR), producing FP32 artifacts.\n" + ] + }, + { + "cell_type": "markdown", + "id": "f4d34230", + "metadata": {}, + "source": [ + "## Dependency and Core Installation Verification\n", + "Run the next cell to verify all thre required packages are installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16781bc1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CHECK] Core package presence (excluding lerobot)...\n", + " [OK] openvino\n", + " [OK] nncf\n", + " [OK] torch\n", + " [OK] torchvision\n", + " [OK] accelerate\n", + " [OK] safetensors\n", + " [OK] numpy\n", + " [OK] pandas\n", + " [OK] matplotlib\n", + " [OK] tqdm\n", + " [OK] h5py\n", + " [OK] onnx\n", + " [OK] onnxruntime\n", + " [OK] rich\n", + "[INFO] All core packages already installed.\n", + "\n", + "[RECHECK] Core imports after installation:\n", + " [OK] openvino\n", + " [OK] nncf\n", + " [OK] torch\n", + " [OK] torchvision\n", + " [OK] accelerate\n", + " [OK] safetensors\n", + " [OK] numpy\n", + " [OK] pandas\n", + " [OK] matplotlib\n", + " [OK] tqdm\n", + " [OK] h5py\n", + " [OK] onnx\n", + " [OK] onnxruntime\n", + " [OK] rich\n", + "\n", + "[CHECK] lerobot availability...\n", + "[OK] lerobot present.\n", + "\n", + "[SUMMARY] Core dependencies verified; lerobot present. Proceed to environment variable export cell.\n" + ] + } + ], + "source": [ + "# Dependency Verification & Core Installation (no auto-install of lerobot)\n", + "\"\"\"\n", + "This cell:\n", + " * Verifies core packages (torch, openvino, nncf + utilities)\n", + " * Installs only missing core packages\n", + " * Checks for lerobot and EXITS with instructions if it's not present\n", + "\n", + "\"\"\"\n", + "import sys, subprocess, importlib, pathlib, os\n", + "\n", + "CORE_SPECS = [\n", + " 'openvino-dev[pot]>=2024.4.0',\n", + " 'nncf>=2.14.0',\n", + " 'torch>=2.1', 'torchvision', 'accelerate',\n", + " 'safetensors', 'numpy', 'pandas', 'matplotlib', 'tqdm', 'h5py',\n", + " 'onnx', 'onnxruntime', 'rich'\n", + "]\n", + "CORE_IMPORTS = {\n", + " 'openvino-dev[pot]>=2024.4.0': 'openvino',\n", + " 'nncf>=2.14.0': 'nncf',\n", + " 'torch>=2.1': 'torch',\n", + " 'torchvision': 'torchvision',\n", + " 'accelerate': 'accelerate',\n", + " 'safetensors': 'safetensors',\n", + " 'numpy': 'numpy',\n", + " 'pandas': 'pandas',\n", + " 'matplotlib': 'matplotlib',\n", + " 'tqdm': 'tqdm',\n", + " 'h5py': 'h5py',\n", + " 'onnx': 'onnx',\n", + " 'onnxruntime': 'onnxruntime',\n", + " 'rich': 'rich'\n", + "}\n", + "SETUP_SCRIPT = pathlib.Path('setup_unitree_lerobot_env.sh')\n", + "README_PATH = pathlib.Path('README.md')\n", + "\n", + "\n", + "def pip_install(*packages):\n", + " cmd = [sys.executable, '-m', 'pip', 'install'] + list(packages)\n", + " print('[INSTALL]', ' '.join(packages))\n", + " subprocess.check_call(cmd)\n", + "\n", + "print('[CHECK] Core package presence (excluding lerobot)...')\n", + "missing = []\n", + "for spec, name in CORE_IMPORTS.items():\n", + " try:\n", + " importlib.import_module(name)\n", + " print(f' [OK] {name}')\n", + " except Exception:\n", + " print(f' [MISSING] {name} (spec: {spec})')\n", + " missing.append(spec)\n", + "\n", + "if missing:\n", + " print('\\n[PHASE] Installing missing core packages...')\n", + " for spec in missing:\n", + " pip_install(spec)\n", + "else:\n", + " print('[INFO] All core packages already installed.')\n", + "\n", + "print('\\n[RECHECK] Core imports after installation:')\n", + "still_missing = []\n", + "for spec, name in CORE_IMPORTS.items():\n", + " try:\n", + " importlib.import_module(name)\n", + " print(f' [OK] {name}')\n", + " except Exception:\n", + " still_missing.append(name)\n", + " print(f' [FAIL] {name} still missing')\n", + "if still_missing:\n", + " print('\\n[WARN] Remaining missing core packages:', still_missing)\n", + " print('Try restarting the kernel or checking version conflicts before proceeding.')\n", + "\n", + "print('\\n[CHECK] lerobot availability...')\n", + "try:\n", + " import lerobot\n", + " print('[OK] lerobot present.')\n", + "except Exception as e:\n", + " print('[ERROR] lerobot not importable:', e)\n", + " print('\\nACTION REQUIRED:')\n", + " print(f' Run setup script: bash {SETUP_SCRIPT}') \n", + " raise SystemExit(1)\n", + "\n", + "print('\\n[SUMMARY] Core dependencies verified; lerobot present. Proceed to environment variable export cell.')" + ] + }, + { + "cell_type": "markdown", + "id": "cf92faf8", + "metadata": {}, + "source": [ + "Next cell configures all the paths." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "878e6d73", + "metadata": {}, + "outputs": [], + "source": [ + "# Configuration Parameters (Paths, Precision, Device)\n", + "import os, pathlib\n", + "\n", + "CKPT_DIR = pathlib.Path('act_checkpoint') \n", + "NOTEBOOK_DIR = pathlib.Path('.').resolve()\n", + "MODEL_DIR = pathlib.Path(os.getenv('ACT_PROJECT_ROOT', NOTEBOOK_DIR))\n", + "CHECKPOINT_PATH = pathlib.Path(os.getenv('ACT_CHECKPOINT', str(CKPT_DIR / 'model.safetensors')))\n", + "IR_OUTPUT_DIR = pathlib.Path(os.getenv('ACT_IR_OUTPUT_DIR', 'openvino_ir_outputs'))\n", + "IR_OUTPUT_DIR.mkdir(exist_ok=True)\n", + "DATASET_ROOT = pathlib.Path(os.getenv('ACT_DATASET_ROOT', str(MODEL_DIR / 'dataset')))\n", + "STATS_PATH = pathlib.Path(os.getenv('ACT_STATS_PATH', str(CKPT_DIR / 'stats.json')))\n", + "\n", + "PRECISIONS = ['FP32', 'FP16']\n", + "TARGET_DEVICE = os.getenv('ACT_TARGET_DEVICE', 'CPU')\n", + "\n", + "print('Notebook directory:', NOTEBOOK_DIR)\n", + "print('Relative checkpoint dir:', CKPT_DIR)\n", + "print('Resolved checkpoint file path:', CHECKPOINT_PATH)\n", + "print('Dataset root:', DATASET_ROOT)\n", + "print('Stats path (may not exist yet):', STATS_PATH)\n", + "print('Output directory:', IR_OUTPUT_DIR)\n", + "print('Target device:', TARGET_DEVICE)" + ] + }, + { + "cell_type": "markdown", + "id": "7f8c1482", + "metadata": {}, + "source": [ + "## Acquire ACT Checkpoint Assets\n", + "Download the ACT model artifacts: `model.safetensors`, `config.json`, and `train_config.json` into `act_checkpoint/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "141847a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[INFO] Checkpoint directory (relative): act_checkpoint\n", + "[INFO] Environment variables:\n", + " ACT_CHECKPOINT = act_checkpoint/model.safetensors\n", + " ACT_CONFIG_PATH = act_checkpoint/config.json\n", + " ACT_TRAIN_CONFIG_PATH = act_checkpoint/train_config.json\n", + " ACT_STATS_PATH = act_checkpoint/stats.json\n" + ] + } + ], + "source": [ + "# Export environment variables\n", + "import os, pathlib\n", + "CKPT_DIR = pathlib.Path('act_checkpoint')\n", + "os.environ['ACT_CHECKPOINT'] = str(CKPT_DIR / 'model.safetensors')\n", + "os.environ['ACT_CONFIG_PATH'] = str(CKPT_DIR / 'config.json')\n", + "os.environ['ACT_TRAIN_CONFIG_PATH'] = str(CKPT_DIR / 'train_config.json')\n", + "stats_path = CKPT_DIR / 'stats.json'\n", + "if stats_path.exists():\n", + " os.environ['ACT_STATS_PATH'] = str(stats_path)\n", + "print('[INFO] Checkpoint directory (relative):', CKPT_DIR)\n", + "print('[INFO] Environment variables:')\n", + "for k in ['ACT_CHECKPOINT','ACT_CONFIG_PATH','ACT_TRAIN_CONFIG_PATH','ACT_STATS_PATH']:\n", + " if k in os.environ:\n", + " print(' ', k, '=', os.environ[k])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aceed7e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[LOAD] CHECKPOINT_PATH = act_checkpoint/model.safetensors\n", + "[LOAD] CONFIG_PATH = act_checkpoint/config.json\n", + "Loaded ACTPolicy from safetensors. Params: 34246684\n", + "Detected cameras: ['observation.images.cam_left_high', 'observation.images.cam_left_wrist', 'observation.images.cam_right_high', 'observation.images.cam_right_wrist']\n" + ] + } + ], + "source": [ + "# Load Original ACT Model\n", + "import os, json, inspect, pathlib, sys, importlib\n", + "from safetensors.torch import load_file\n", + "from lerobot.policies.act.modeling_act import ACTPolicy\n", + "from lerobot.policies.act.configuration_act import ACTConfig\n", + "from lerobot.configs.types import PolicyFeature, FeatureType, NormalizationMode\n", + "\n", + "CHECKPOINT_PATH = pathlib.Path(os.getenv('ACT_CHECKPOINT', 'act_checkpoint/model.safetensors'))\n", + "CONFIG_PATH = pathlib.Path(os.getenv('ACT_CONFIG_PATH', str(CHECKPOINT_PATH.parent / 'config.json')))\n", + "\n", + "print('[LOAD] CHECKPOINT_PATH =', CHECKPOINT_PATH)\n", + "print('[LOAD] CONFIG_PATH =', CONFIG_PATH)\n", + "\n", + "if not CHECKPOINT_PATH.exists():\n", + " raise FileNotFoundError(\n", + " f\"Checkpoint file not found at {CHECKPOINT_PATH}.\\n\"\n", + " \"Ensure you have: (1) placed model.safetensors in act_checkpoint/, or (2) set ACT_CHECKPOINT env var, then re-run this cell.\"\n", + " )\n", + "if not CONFIG_PATH.exists():\n", + " raise FileNotFoundError(\n", + " f\"config.json not found at {CONFIG_PATH}.\\n\"\n", + " \"Place config.json next to the checkpoint (act_checkpoint/config.json) or set ACT_CONFIG_PATH.\"\n", + " )\n", + "\n", + "with open(CONFIG_PATH, 'r') as f:\n", + " cfg_dict = json.load(f)\n", + "\n", + "# Filter config keys to ACTConfig signature\n", + "valid_keys = set(inspect.signature(ACTConfig.__init__).parameters.keys()); valid_keys.discard('self')\n", + "filtered_cfg = {k: v for k, v in cfg_dict.items() if k in valid_keys}\n", + "\n", + "# Helper wrappers\n", + "def wrap_features(feat_dict):\n", + " return {k: PolicyFeature(type=FeatureType(v['type']), shape=tuple(v['shape'])) for k, v in feat_dict.items()}\n", + "\n", + "def wrap_norm_map(norm_map):\n", + " return {FeatureType(k): NormalizationMode(v) for k, v in norm_map.items()}\n", + "\n", + "if 'input_features' in filtered_cfg:\n", + " filtered_cfg['input_features'] = wrap_features(filtered_cfg['input_features'])\n", + "if 'output_features' in filtered_cfg:\n", + " filtered_cfg['output_features'] = wrap_features(filtered_cfg['output_features'])\n", + "if 'normalization_mapping' in filtered_cfg:\n", + " filtered_cfg['normalization_mapping'] = wrap_norm_map(filtered_cfg['normalization_mapping'])\n", + "\n", + "act_config = ACTConfig(**filtered_cfg)\n", + "act_config.use_vae = False\n", + "policy = ACTPolicy(act_config)\n", + "weights = load_file(str(CHECKPOINT_PATH))\n", + "policy.load_state_dict(weights, strict=False)\n", + "policy.eval()\n", + "print('Loaded ACTPolicy from safetensors. Params:', sum(p.numel() for p in policy.parameters()))\n", + "\n", + "# Extract dimensions\n", + "action_dim = filtered_cfg['output_features']['action'].shape[0]\n", + "chunk_size = filtered_cfg.get('chunk_size', 100)\n", + "# Camera keys\n", + "camera_keys = sorted([k for k in cfg_dict['input_features'] if k.startswith('observation.images.')])\n", + "print('Detected cameras:', camera_keys)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "667d3f45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "State shape: torch.Size([1, 28])\n", + "Image shapes: [torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640])]\n", + "Action pad shape: torch.Size([1, 100])\n", + "Action seq shape: torch.Size([1, 100, 28])\n" + ] + } + ], + "source": [ + "# Inspect Model Architecture and Construct Full Dummy Inputs\n", + "import torch \n", + "\n", + "state_dim = policy.config.input_features['observation.state'].shape[0]\n", + "chunk_size = chunk_size # from previous cell\n", + "H, W = 480, 640\n", + "cams = camera_keys\n", + "\n", + "# Use shapes from config if specified\n", + "image_tensors = []\n", + "for cam in cams:\n", + " shape = policy.config.input_features[cam].shape # e.g. [3, H, W]\n", + " img = torch.zeros(1, *shape, dtype=torch.float32)\n", + " image_tensors.append(img)\n", + "\n", + "state = torch.zeros(1, state_dim, dtype=torch.float32)\n", + "action_is_pad = torch.zeros(1, chunk_size, dtype=torch.bool)\n", + "action_seq = torch.zeros(1, chunk_size, action_dim, dtype=torch.float32)\n", + "\n", + "env_state = None\n", + "if 'observation.environment_state' in policy.config.input_features:\n", + " env_dim = policy.config.input_features['observation.environment_state'].shape[0]\n", + " env_state = torch.zeros(1, env_dim, dtype=torch.float32)\n", + "\n", + "print('State shape:', state.shape)\n", + "print('Image shapes:', [t.shape for t in image_tensors])\n", + "print('Action pad shape:', action_is_pad.shape)\n", + "print('Action seq shape:', action_seq.shape)\n", + "if env_state is not None:\n", + " print('Environment state shape:', env_state.shape)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c301d9af", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ordered input tensor shapes: [torch.Size([1, 28]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 100]), torch.Size([1, 100, 28])]\n" + ] + } + ], + "source": [ + "# Prepare Ordered Inputs\n", + "# Order: observation.state, each camera image, action_is_pad, action, optional environment_state\n", + "ordered_inputs = [state] + image_tensors + [action_is_pad, action_seq] + ([env_state] if env_state is not None else [])\n", + "print('Ordered input tensor shapes:', [t.shape for t in ordered_inputs])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe4a496c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_tokens.extend(list(cam_features))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_tokens.extend(list(cam_features))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_pos_embed.extend(list(cam_pos_embed))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_pos_embed.extend(list(cam_pos_embed))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ONNX model exported to openvino_ir_outputs/model.onnx\n" + ] + } + ], + "source": [ + "# Export ACT Model to ONNX\n", + "import torch, inspect, os, pathlib\n", + "\n", + "# Ensure IR_OUTPUT_DIR is available\n", + "try:\n", + " IR_OUTPUT_DIR\n", + "except NameError:\n", + " IR_OUTPUT_DIR = pathlib.Path(os.getenv('ACT_IR_OUTPUT_DIR', 'openvino_ir_outputs'))\n", + " print('[ONNX] IR_OUTPUT_DIR was undefined; set to', IR_OUTPUT_DIR)\n", + "\n", + "# Create directory if missing\n", + "IR_OUTPUT_DIR.mkdir(exist_ok=True)\n", + "\n", + "# Build ONNXWrapper\n", + "class ONNXWrapper(torch.nn.Module):\n", + " def __init__(self, model, camera_keys):\n", + " super().__init__()\n", + " self.model = model\n", + " self.camera_keys = camera_keys\n", + " def forward(self, observation_state, *cam_inputs_and_rest):\n", + " num_cams = len(self.camera_keys)\n", + " cam_inputs = cam_inputs_and_rest[:num_cams]\n", + " action_is_pad_local = cam_inputs_and_rest[num_cams]\n", + " action_local = cam_inputs_and_rest[num_cams + 1]\n", + " observation_environment_state = None\n", + " if len(cam_inputs_and_rest) > num_cams + 2:\n", + " observation_environment_state = cam_inputs_and_rest[num_cams + 2]\n", + " batch = {'observation.state': observation_state}\n", + " for i, cam_key in enumerate(self.camera_keys):\n", + " batch[cam_key] = cam_inputs[i]\n", + " batch['action_is_pad'] = action_is_pad_local\n", + " batch['action'] = action_local\n", + " batch['observation.images'] = list(cam_inputs)\n", + " if observation_environment_state is not None:\n", + " batch['observation.environment_state'] = observation_environment_state\n", + " prediction = self.model.model(batch)\n", + " if isinstance(prediction, tuple):\n", + " prediction = prediction[0]\n", + " return prediction\n", + "\n", + "onnx_path = IR_OUTPUT_DIR / 'model.onnx'\n", + "if not onnx_path.exists():\n", + " # Construct dummy args & input names\n", + " dummy_args = [torch.randn_like(state)]\n", + " input_names = ['observation_state']\n", + " for i, cam_key in enumerate(camera_keys):\n", + " cam_tensor = torch.randn_like(image_tensors[i])\n", + " dummy_args.append(cam_tensor)\n", + " input_names.append(f'observation_images_{i}')\n", + " dummy_args.append(torch.zeros_like(action_is_pad))\n", + " dummy_args.append(torch.zeros_like(action_seq))\n", + " input_names += ['action_is_pad', 'action']\n", + " if env_state is not None:\n", + " dummy_args.append(torch.randn_like(env_state))\n", + " input_names.append('observation_environment_state')\n", + "\n", + " torch.onnx.export(\n", + " ONNXWrapper(policy, camera_keys),\n", + " tuple(dummy_args),\n", + " str(onnx_path),\n", + " export_params=True,\n", + " opset_version=11,\n", + " do_constant_folding=True,\n", + " input_names=input_names,\n", + " output_names=['output']\n", + " )\n", + " print('ONNX model exported to', onnx_path)\n", + "else:\n", + " print('ONNX already exists:', onnx_path)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "540875e8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running Model Optimizer: mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --compress_to_fp16=False\n", + "[ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release.\n", + "In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. \n", + "Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html\n", + "Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2023-1&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino\n", + "[ SUCCESS ] Generated IR version 11 model.\n", + "[ SUCCESS ] XML file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/model.xml\n", + "[ SUCCESS ] BIN file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/model.bin\n", + "MO conversion complete. Standardized IR files: openvino_ir_outputs/act_model_fp32.xml openvino_ir_outputs/act_model_fp32.bin\n" + ] + } + ], + "source": [ + "# Convert ONNX to OpenVINO IR\n", + "import subprocess, shlex, os, pathlib, shutil\n", + "MO_OUT_DIR = IR_OUTPUT_DIR\n", + "IR_FP32_XML = MO_OUT_DIR / 'act_model_fp32.xml'\n", + "IR_FP32_BIN = MO_OUT_DIR / 'act_model_fp32.bin'\n", + "\n", + "if IR_FP32_XML.exists() and IR_FP32_BIN.exists():\n", + " print('IR already present, skipping MO conversion:', IR_FP32_XML)\n", + "else:\n", + " cmd = f\"mo --input_model {IR_OUTPUT_DIR / 'model.onnx'} --output_dir {MO_OUT_DIR} --compress_to_fp16=False\"\n", + " print('Running Model Optimizer:', cmd)\n", + " try:\n", + " subprocess.run(shlex.split(cmd), check=True)\n", + " src_xml = MO_OUT_DIR / 'model.xml'\n", + " src_bin = MO_OUT_DIR / 'model.bin'\n", + " if src_xml.exists():\n", + " shutil.copy(src_xml, IR_FP32_XML)\n", + " if src_bin.exists():\n", + " shutil.copy(src_bin, IR_FP32_BIN)\n", + " print('MO conversion complete. Standardized IR files:', IR_FP32_XML, IR_FP32_BIN)\n", + " except Exception as e:\n", + " raise RuntimeError(f'Model Optimizer failed. Ensure openvino-dev installed. Original error: {e}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23dcc638", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The following IR files present:\n", + "- act_model_fp32.xml MISSING | size: 0\n", + "- act_model_fp32.bin MISSING | size: 0\n" + ] + } + ], + "source": [ + "# Confirm IR Files Produced by MO\n", + "print('The following IR files present:')\n", + "for f in [IR_OUTPUT_DIR / 'act_model_fp32.xml', IR_OUTPUT_DIR / 'act_model_fp32.bin']:\n", + " print('-', f.name, 'exists' if f.exists() else 'MISSING', '| size:', f.stat().st_size if f.exists() else 0)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01bab214-c42b-4388-8b39-2227f75640c0", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/lerobot_act/setup_unitree_lerobot_env.sh b/notebooks/lerobot_act/setup_unitree_lerobot_env.sh new file mode 100644 index 00000000000..5b07088fc1c --- /dev/null +++ b/notebooks/lerobot_act/setup_unitree_lerobot_env.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Automated setup for Conda-based Python 3.10 environment integrating OpenVINO notebooks + Unitree LeRobot fork. +# Usage: +# bash setup_unitree_lerobot_env.sh +set -euo pipefail + +ENV_NAME="unitree_lerobot" +LEROBOT_DIR="unitree_IL_lerobot" +PY_VERSION="3.10" + +log() { printf "\n[setup] %s\n" "$*"; } +err() { printf "\n[error] %s\n" "$*" >&2; } + +check_conda() { + if ! command -v conda >/dev/null 2>&1; then + err "Conda not found. Install Miniconda: https://docs.conda.io/en/latest/miniconda.html" + exit 1 + fi +} + +create_env() { + if conda env list | grep -E "^${ENV_NAME}[[:space:]]" >/dev/null; then + log "Conda env '${ENV_NAME}' already exists (skipping creation)." + else + log "Creating conda env '${ENV_NAME}' (python=${PY_VERSION})" + conda create -y -n "${ENV_NAME}" python="${PY_VERSION}" + fi +} + +activate_env() { + log "Activating env '${ENV_NAME}'" + source "$(conda info --base)/etc/profile.d/conda.sh" + conda activate "${ENV_NAME}" +} + +upgrade_tooling() { + log "Upgrading pip/wheel/setuptools" + python -m pip install --upgrade pip wheel setuptools +} + +install_openvino_requirements() { + # We are already inside the openvino_notebooks tree; + # find the root (parent directories) containing requirements.txt. + if [ -f ../../requirements.txt ]; then + ROOT_REQ=../../requirements.txt + elif [ -f ../requirements.txt ]; then + ROOT_REQ=../requirements.txt + elif [ -f requirements.txt ]; then + ROOT_REQ=requirements.txt + else + err "Could not locate requirements.txt for OpenVINO notebooks. Ensure script resides under openvino_notebooks/." + return 1 + fi + log "Installing OpenVINO notebooks requirements from ${ROOT_REQ}" + python -m pip install -r "${ROOT_REQ}" +} + +register_kernel() { + if jupyter kernelspec list 2>/dev/null | grep -q "${ENV_NAME}"; then + log "Jupyter kernel '${ENV_NAME}' already registered." + else + log "Registering Jupyter kernel '${ENV_NAME}'" + python -m ipykernel install --user --name "${ENV_NAME}" --display-name "Python ${PY_VERSION} (${ENV_NAME})" + fi +} + +clone_lerobot_repo() { + if [ -d "${LEROBOT_DIR}" ]; then + log "LeRobot repo '${LEROBOT_DIR}' already exists (skipping clone)." + else + log "Cloning Unitree LeRobot fork with submodules" + git clone --recurse-submodules https://github.com/unitreerobotics/unitree_IL_lerobot.git "${LEROBOT_DIR}" + fi + pushd "${LEROBOT_DIR}" >/dev/null + log "Updating submodules" + git submodule update --init --recursive + popd >/dev/null +} + +install_pinocchio() { + log "Installing pinocchio via conda-forge (if not installed)" + if python -c "import pinocchio" 2>/dev/null; then + log "pinocchio already installed." + else + conda install -y pinocchio -c conda-forge || err "pinocchio install failed; continue without if unused." + fi +} + +install_lerobot_editable() { + pushd "${LEROBOT_DIR}" >/dev/null + if python -c "import lerobot" 2>/dev/null; then + log "lerobot already importable (skipping editable install)." + else + if [ -d unitree_lerobot/lerobot ]; then + log "Editable install of upstream component" + pushd unitree_lerobot/lerobot >/dev/null + python -m pip install -e . || err "Editable install (lerobot) failed" + popd >/dev/null + fi + log "Editable install of root fork extras" + python -m pip install -e .[dev] || python -m pip install -e . || err "Editable install root failed" + fi + popd >/dev/null +} + +install_extra_packages() { + log "Installing auxiliary packages (if missing)" + python - <<'EOF' +import importlib, subprocess, sys +packages = [ + 'safetensors','onnx','onnxruntime','openvino-dev[pot]','nncf','hydra-core','rich','tqdm','pandas','h5py' +] +for pkg in packages: + base = pkg.split('[')[0].split('==')[0].split('>=')[0] + try: + importlib.import_module(base.replace('-','_')) + print(f"[extra] OK: {pkg}") + except Exception: + print(f"[extra] Installing: {pkg}") + subprocess.check_call([sys.executable,'-m','pip','install',pkg]) +EOF +} + +sanity_check() { + log "Running sanity check" + python - <<'EOF' +import sys, importlib +print('Python:', sys.version) +for pkg in ['torch','openvino','nncf','lerobot']: + try: + importlib.import_module(pkg) + print('OK:', pkg) + except Exception as e: + print('FAIL:', pkg, e) +EOF +} + +main() { + check_conda + create_env + activate_env + upgrade_tooling + install_openvino_requirements + register_kernel + clone_lerobot_repo + install_pinocchio + install_lerobot_editable + install_extra_packages + sanity_check + log "Setup complete. Launch notebook with:" + echo " conda activate ${ENV_NAME}" + echo " jupyter lab notebooks/lerobot_act/act_to_openvino.ipynb --NotebookApp.kernel_name=${ENV_NAME}" +} + +main From 80df5e0b062563afcbd189732e2080710098da99 Mon Sep 17 00:00:00 2001 From: Mohammad Haghighipanah Date: Fri, 7 Nov 2025 14:01:26 -0800 Subject: [PATCH 2/5] added quantization, direct conversion and evaluation script to lerobot act --- notebooks/lerobot_act/README.md | 76 +- notebooks/lerobot_act/act_to_openvino.ipynb | 790 ++++++++++++++++-- .../lerobot_act/eval_openvino_model_helper.py | 501 +++++++++++ notebooks/lerobot_act/quantize_int8_helper.py | 344 ++++++++ 4 files changed, 1649 insertions(+), 62 deletions(-) create mode 100644 notebooks/lerobot_act/eval_openvino_model_helper.py create mode 100644 notebooks/lerobot_act/quantize_int8_helper.py diff --git a/notebooks/lerobot_act/README.md b/notebooks/lerobot_act/README.md index f38f85bfadf..707a925e1d8 100644 --- a/notebooks/lerobot_act/README.md +++ b/notebooks/lerobot_act/README.md @@ -1,5 +1,4 @@ # ACT Policy → OpenVINO IR Conversion (Notebook Guide) - This README documents the current workflow implemented in `act_to_openvino.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. The notebook presently performs FP32 export (Model Optimizer invoked with FP16 compression flag but output standardized to `act_model_fp32.xml/bin`). @@ -10,13 +9,17 @@ Place these next to the notebook: * `train_config.json` – optional (reproducibility record) * `stats.json` – optional normalization statistics +## Required dataset Files (`dataset/G1_BlockStacking_Dataset/`) +Download the G1_BlockStacking_Dataset from hugging face: +https://huggingface.co/datasets/unitreerobotics/G1_Dex3_BlockStacking_Dataset + + ## Minimal Installation & Launch Recommended (Conda environment): ```bash +bash setup_unitree_lerobot_env.sh conda create -n unitree_lerobot python=3.10 -y conda activate unitree_lerobot -# Optional helper script if present in this folder -bash setup_unitree_lerobot_env.sh # Launch notebook with correct kernel jupyter lab act_to_openvino.ipynb --NotebookApp.kernel_name=unitree_lerobot ``` @@ -49,16 +52,75 @@ Artifacts are copied / renamed to: * `act_model_fp32.xml` * `act_model_fp32.bin` -## Directory Layout After Successful Conversion +## Direct PyTorch FX Conversion +Instead of exporting full temporal tensors via ONNX you can generate a smaller IR directly from PyTorch using OpenVINO's FX path. The wrapper internally creates placeholder temporal inputs (`action`, `action_is_pad`, history) so the IR exposes only observation features: +* `observation_state` +* `observation_images_0..N` (one input per camera) + +Resulting files: +* `act_model_direct_fp32.xml/bin` +* `act_model_direct_fp16.xml/bin` + +## INT8 Quantization (NNCF) +You can produce an INT8 version for reduced size / latency using NNCF post‑training quantization. + +Prerequisites: +* Direct FP32 IR: `act_model_direct_fp32.xml` +* Representative dataset root (`ACT_DATASET_ROOT`) with episodes +* Normalization stats: `stats.json` + +Generated files: +* `openvino_ir_outputs/int8/model_int8.xml/bin` + +Tips: +* Increase calibration samples for better accuracy. +* Use `preset='accuracy'` if performance preset degrades results too much. +* Ensure OpenVINO and NNCF versions are compatible (>= 2025.0.0 for OpenVINO runtime if using latest NNCF). + + +## Evaluation of Variants +The notebook / helper script can compare PyTorch baseline vs IR variants (Direct FP32, FP16, MO FP32, INT8). + +Environment variables (set before running evaluation cell): +| Var | Purpose | +|-----|---------| +| `OPENVINO_MODEL_PATH` | Path to IR `.xml` file to evaluate | +| `STATS_PATH` | Path to `stats.json` for normalization | +| `OPENVINO_DEVICE` | `CPU|GPU|NPU|AUTO` (compile target) | +| `OPENVINO_PRECISION_HINT` | Optional override (`FP32|FP16|INT8`) | + + +Evaluation pipeline steps: +1. Load PyTorch ACT and normalization stats. +2. Compile OpenVINO model. +3. Run action predictions over dataset episodes. +4. Apply optional temporal smoothing ensemble. +5. Plot per‑joint trajectories & error statistics (saved as `actions_comparison_.png`). + + +## Directory Layout (Example After Conversion, FP16 & INT8 Quantization) ``` act_to_openvino.ipynb act_checkpoint/ model.safetensors config.json train_config.json - stats.json (optional) + stats.json # normalization (recommended, required for eval & INT8) +dataset/ + G1_BlockStacking_Dataset/ openvino_ir_outputs/ - model.onnx - act_model_fp32.xml + model.onnx # ONNX baseline export + act_model_fp32.xml # MO baseline IR (full inputs) act_model_fp32.bin + act_model_direct_fp32.xml # Direct minimal-input IR + act_model_direct_fp32.bin + int8/ + model_int8.xml # Post-training quantized INT8 IR + model_int8.bin + +figure_actions_comparison_direct_fp32.png +figure_actions_comparison_direct_fp16.png +figure_actions_comparison_mo_fp32.png +figure_actions_comparison_int8.png + ``` \ No newline at end of file diff --git a/notebooks/lerobot_act/act_to_openvino.ipynb b/notebooks/lerobot_act/act_to_openvino.ipynb index dd8413725ac..f6d59adbcad 100644 --- a/notebooks/lerobot_act/act_to_openvino.ipynb +++ b/notebooks/lerobot_act/act_to_openvino.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "16781bc1", "metadata": {}, "outputs": [ @@ -31,7 +31,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "[CHECK] Core package presence (excluding lerobot)...\n", + "[PIP] /home/case/miniforge3/envs/unitree_lerobot/bin/python -m pip install -U openvino>=2025.0.0\n", + "Requirement already satisfied: openvino>=2025.0.0 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (2025.3.0)\n", + "Requirement already satisfied: numpy<2.3.0,>=1.16.6 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (1.26.4)\n", + "Requirement already satisfied: openvino-telemetry>=2023.2.1 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (2025.2.0)\n", + "Requirement already satisfied: packaging in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (25.0)\n", + "\n", + "[CHECK] Other core package presence (excluding openvino)...\n", " [OK] openvino\n", " [OK] nncf\n", " [OK] torch\n", @@ -46,7 +52,7 @@ " [OK] onnx\n", " [OK] onnxruntime\n", " [OK] rich\n", - "[INFO] All core packages already installed.\n", + "[INFO] All other core packages already installed.\n", "\n", "[RECHECK] Core imports after installation:\n", " [OK] openvino\n", @@ -67,30 +73,35 @@ "[CHECK] lerobot availability...\n", "[OK] lerobot present.\n", "\n", - "[SUMMARY] Core dependencies verified; lerobot present. Proceed to environment variable export cell.\n" + "[SUMMARY] Dependencies verified. openvino runtime >= 2025.0.0 (installed: 2025.3.0). Proceed to path configuration cell.\n" ] } ], "source": [ - "# Dependency Verification & Core Installation (no auto-install of lerobot)\n", + "# Dependency Verification & Core Installation (auto-upgrade openvino runtime)\n", "\"\"\"\n", "This cell:\n", " * Verifies core packages (torch, openvino, nncf + utilities)\n", - " * Installs only missing core packages\n", + " * Auto-installs or upgrades openvino runtime to >=2025.0.0 if current version is older\n", " * Checks for lerobot and EXITS with instructions if it's not present\n", - "\n", "\"\"\"\n", "import sys, subprocess, importlib, pathlib, os\n", + "from importlib import metadata\n", + "\n", + "TARGET_OV_VERSION = '2025.0.0' # Minimum required runtime version\n", + "SETUP_SCRIPT = pathlib.Path('setup_unitree_lerobot_env.sh')\n", + "README_PATH = pathlib.Path('README.md')\n", "\n", + "# Core specs excluding openvino (handled separately for upgrade logic)\n", "CORE_SPECS = [\n", - " 'openvino-dev[pot]>=2024.4.0',\n", + " 'openvino>=2025.0.0',\n", " 'nncf>=2.14.0',\n", " 'torch>=2.1', 'torchvision', 'accelerate',\n", " 'safetensors', 'numpy', 'pandas', 'matplotlib', 'tqdm', 'h5py',\n", " 'onnx', 'onnxruntime', 'rich'\n", "]\n", "CORE_IMPORTS = {\n", - " 'openvino-dev[pot]>=2024.4.0': 'openvino',\n", + " 'openvino>=2025.0.0': 'openvino',\n", " 'nncf>=2.14.0': 'nncf',\n", " 'torch>=2.1': 'torch',\n", " 'torchvision': 'torchvision',\n", @@ -105,16 +116,18 @@ " 'onnxruntime': 'onnxruntime',\n", " 'rich': 'rich'\n", "}\n", - "SETUP_SCRIPT = pathlib.Path('setup_unitree_lerobot_env.sh')\n", - "README_PATH = pathlib.Path('README.md')\n", - "\n", "\n", "def pip_install(*packages):\n", - " cmd = [sys.executable, '-m', 'pip', 'install'] + list(packages)\n", - " print('[INSTALL]', ' '.join(packages))\n", + " cmd = [sys.executable, '-m', 'pip', 'install', '-U'] + list(packages)\n", + " print('[PIP]', ' '.join(cmd))\n", " subprocess.check_call(cmd)\n", "\n", - "print('[CHECK] Core package presence (excluding lerobot)...')\n", + "# --- Upgrade OpenVINO to target version requied for INT8 quantization ---\n", + "pip_install(f'openvino>={TARGET_OV_VERSION}')\n", + "\n", + "\n", + "# --- Remaining core packages ---\n", + "print('\\n[CHECK] Other core package presence (excluding openvino)...')\n", "missing = []\n", "for spec, name in CORE_IMPORTS.items():\n", " try:\n", @@ -125,11 +138,11 @@ " missing.append(spec)\n", "\n", "if missing:\n", - " print('\\n[PHASE] Installing missing core packages...')\n", + " print('\\n[PHASE] Installing missing packages...')\n", " for spec in missing:\n", " pip_install(spec)\n", "else:\n", - " print('[INFO] All core packages already installed.')\n", + " print('[INFO] All other core packages already installed.')\n", "\n", "print('\\n[RECHECK] Core imports after installation:')\n", "still_missing = []\n", @@ -141,9 +154,10 @@ " still_missing.append(name)\n", " print(f' [FAIL] {name} still missing')\n", "if still_missing:\n", - " print('\\n[WARN] Remaining missing core packages:', still_missing)\n", - " print('Try restarting the kernel or checking version conflicts before proceeding.')\n", + " print('\\n[WARN] Remaining missing packages:', still_missing)\n", + " print('Consider restarting the kernel or resolving version conflicts.')\n", "\n", + "# --- lerobot presence ---\n", "print('\\n[CHECK] lerobot availability...')\n", "try:\n", " import lerobot\n", @@ -151,10 +165,13 @@ "except Exception as e:\n", " print('[ERROR] lerobot not importable:', e)\n", " print('\\nACTION REQUIRED:')\n", - " print(f' Run setup script: bash {SETUP_SCRIPT}') \n", + " print(f' Run setup script: bash {SETUP_SCRIPT}')\n", " raise SystemExit(1)\n", "\n", - "print('\\n[SUMMARY] Core dependencies verified; lerobot present. Proceed to environment variable export cell.')" + "# --- Summary ---\n", + "from importlib import metadata as md\n", + "ov_ver = md.version('openvino') if 'openvino' in sys.modules or metadata.version('openvino') else 'UNKNOWN'\n", + "print(f\"\\n[SUMMARY] Dependencies verified. openvino runtime >= {TARGET_OV_VERSION} (installed: {ov_ver}). Proceed to path configuration cell.\")" ] }, { @@ -167,10 +184,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "878e6d73", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Notebook directory: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act\n", + "Relative checkpoint dir: act_checkpoint\n", + "Resolved checkpoint file path: act_checkpoint/model.safetensors\n", + "Dataset root: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/dataset\n", + "Stats path (may not exist yet): act_checkpoint/stats.json\n", + "Output directory: openvino_ir_outputs\n", + "Target device (OpenVINO): CPU\n" + ] + } + ], "source": [ "# Configuration Parameters (Paths, Precision, Device)\n", "import os, pathlib\n", @@ -193,7 +224,7 @@ "print('Dataset root:', DATASET_ROOT)\n", "print('Stats path (may not exist yet):', STATS_PATH)\n", "print('Output directory:', IR_OUTPUT_DIR)\n", - "print('Target device:', TARGET_DEVICE)" + "print('Target device (OpenVINO):', TARGET_DEVICE)" ] }, { @@ -207,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "141847a0", "metadata": {}, "outputs": [ @@ -243,7 +274,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "aceed7e2", "metadata": {}, "outputs": [ @@ -322,7 +353,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "667d3f45", "metadata": {}, "outputs": [ @@ -372,7 +403,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "c301d9af", "metadata": {}, "outputs": [ @@ -393,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "fe4a496c", "metadata": {}, "outputs": [ @@ -497,7 +528,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "540875e8", "metadata": {}, "outputs": [ @@ -505,70 +536,719 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running Model Optimizer: mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --compress_to_fp16=False\n", + "Running Model Optimizer: mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --model_name act_model_fp32 --compress_to_fp16=False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n", + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n", + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n", + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "[ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release.\n", "In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. \n", - "Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html\n", + "Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2023-1&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino\n", "[ SUCCESS ] Generated IR version 11 model.\n", - "[ SUCCESS ] XML file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/model.xml\n", - "[ SUCCESS ] BIN file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/model.bin\n", - "MO conversion complete. Standardized IR files: openvino_ir_outputs/act_model_fp32.xml openvino_ir_outputs/act_model_fp32.bin\n" + "[ SUCCESS ] XML file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/act_model_fp32.xml\n", + "[ SUCCESS ] BIN file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/act_model_fp32.bin\n", + "MO conversion complete. IR files: openvino_ir_outputs/act_model_fp32.xml openvino_ir_outputs/act_model_fp32.bin\n", + "The following IR files present:\n", + "- act_model_fp32.xml exists | size: 2725658\n", + "- act_model_fp32.bin exists | size: 134347484\n" ] } ], "source": [ "# Convert ONNX to OpenVINO IR\n", - "import subprocess, shlex, os, pathlib, shutil\n", - "MO_OUT_DIR = IR_OUTPUT_DIR\n", + "import subprocess, shlex, os, pathlib\n", + "MO_OUT_DIR = IR_OUTPUT_DIR # assumes IR_OUTPUT_DIR defined earlier\n", "IR_FP32_XML = MO_OUT_DIR / 'act_model_fp32.xml'\n", "IR_FP32_BIN = MO_OUT_DIR / 'act_model_fp32.bin'\n", "\n", "if IR_FP32_XML.exists() and IR_FP32_BIN.exists():\n", " print('IR already present, skipping MO conversion:', IR_FP32_XML)\n", "else:\n", - " cmd = f\"mo --input_model {IR_OUTPUT_DIR / 'model.onnx'} --output_dir {MO_OUT_DIR} --compress_to_fp16=False\"\n", + " cmd = (\n", + " f\"mo --input_model {IR_OUTPUT_DIR / 'model.onnx'} \"\n", + " f\"--output_dir {MO_OUT_DIR} --model_name act_model_fp32 --compress_to_fp16=False\"\n", + " )\n", " print('Running Model Optimizer:', cmd)\n", " try:\n", " subprocess.run(shlex.split(cmd), check=True)\n", - " src_xml = MO_OUT_DIR / 'model.xml'\n", - " src_bin = MO_OUT_DIR / 'model.bin'\n", - " if src_xml.exists():\n", - " shutil.copy(src_xml, IR_FP32_XML)\n", - " if src_bin.exists():\n", - " shutil.copy(src_bin, IR_FP32_BIN)\n", - " print('MO conversion complete. Standardized IR files:', IR_FP32_XML, IR_FP32_BIN)\n", + " # Verify artifacts\n", + " if not IR_FP32_XML.exists() or not IR_FP32_BIN.exists():\n", + " raise FileNotFoundError('Expected act_model_fp32.xml/bin not produced. Check MO logs above.')\n", + " print('MO conversion complete. IR files:', IR_FP32_XML, IR_FP32_BIN)\n", + " except Exception as e:\n", + " raise RuntimeError(f'Model Optimizer failed. Ensure openvino (and optionally openvino-dev) installed. Original error: {e}')\n", + "\n", + "print('The following IR files present:')\n", + "for f in [IR_FP32_XML, IR_FP32_BIN]:\n", + " print('-', f.name, 'exists' if f.exists() else 'MISSING', '| size:', f.stat().st_size if f.exists() else 0)" + ] + }, + { + "cell_type": "markdown", + "id": "2dbc80c7", + "metadata": {}, + "source": [ + "### Optional: Direct PyTorch → OpenVINO IR (No ONNX)\n", + "Convert the loaded ACT policy directly from PyTorch to OpenVINO IR using the FX frontend. This bypasses ONNX export.\n", + "\n", + "Precision notes:\n", + "- Default below is FP32 (weights stored as 32-bit floating point).\n", + "- To get FP16 (half-precision weights) via the direct path you can call `ov.save_model(..., compress_to_fp16=True)`.\n", + "\n", + "How to produce FP16 here (two approaches):\n", + "Add after FP32 save:\n", + " ```python\n", + " ov.save_model(ov_model, str(IR_OUTPUT_DIR / 'act_model_direct_fp16.xml'), compress_to_fp16=True)\n", + " ```\n", + " This emits `act_model_direct_fp16.xml/bin`.\n", + "\n", + "What gets converted when using `compress_to_fp16=True`?\n", + "- Parameter tensors (weights) are stored in FP16.\n", + "- Graph topology and layer semantics stay the same." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "326ad8df", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[DIRECT] Converting via ov.convert_model (FX)...\n", + "WARNING:nncf:NNCF provides best results with torch==2.8.*, while current torch version is 2.7.1+cu126. If you encounter issues, consider switching to torch==2.8.*\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_tokens.extend(list(cam_features))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_tokens.extend(list(cam_features))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", + " encoder_in_pos_embed.extend(list(cam_pos_embed))\n", + "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", + " encoder_in_pos_embed.extend(list(cam_pos_embed))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[DIRECT] Final IR input ports (friendly_name / tensor names / partial shape):\n", + " - observation_state | tensor_names=['observation_state'] | partial_shape=[?,?]\n", + " - observation_images_0 | tensor_names=['observation_images_0'] | partial_shape=[?,3,?,?]\n", + " - observation_images_1 | tensor_names=['observation_images_1'] | partial_shape=[?,3,?,?]\n", + " - observation_images_2 | tensor_names=['observation_images_2'] | partial_shape=[?,3,?,?]\n", + " - observation_images_3 | tensor_names=['observation_images_3'] | partial_shape=[?,3,?,?]\n", + "[DIRECT] Saved FP32 XML: openvino_ir_outputs/act_model_direct_fp32.xml | size: 4011117\n", + "[DIRECT] Saved FP32 BIN : openvino_ir_outputs/act_model_direct_fp32.bin | size: 68503430\n", + "\n", + "[HINT] In evaluation build input dict using: observation_state, observation_images_0..N, (optional) observation_environment_state.\n", + "[DONE] Direct conversion complete. (See comments above for FP16 save).\n" + ] + } + ], + "source": [ + "# Direct PyTorch to OpenVINO IR (FP32 by default, FP16 instructions included)\n", + "import torch, pathlib, openvino as ov\n", + "\n", + "# Required objects from previous cells.\n", + "required = ['policy', 'camera_keys', 'state', 'image_tensors', 'action_dim', 'chunk_size']\n", + "for sym in required:\n", + " if sym not in globals():\n", + " raise RuntimeError(f'Missing `{sym}`. Run earlier cells first.')\n", + "\n", + "env_present = 'env_state' in globals() and env_state is not None\n", + "\n", + "class DirectWrapper(torch.nn.Module):\n", + " \"\"\"Expose only observation_state, per-camera images, optional env.\n", + " Internal temporal tensors (action_is_pad, action) are synthesized so they do NOT become IR inputs.\n", + " This keeps the IR minimal and matches evaluation expectations.\n", + " \"\"\"\n", + " def __init__(self, act_policy, camera_keys, chunk_size, action_dim, env_present=False):\n", + " super().__init__()\n", + " self.model = act_policy\n", + " self.camera_keys = camera_keys\n", + " self.chunk_size = chunk_size\n", + " self.action_dim = action_dim\n", + " self.env_present = env_present\n", + " def forward(self, observation_state, *cams_and_env):\n", + " num_cams = len(self.camera_keys)\n", + " cam_tensors = cams_and_env[:num_cams]\n", + " env_tensor = cams_and_env[num_cams] if self.env_present and len(cams_and_env) > num_cams else None\n", + " B = observation_state.shape[0]\n", + " device = observation_state.device\n", + " action_is_pad_local = torch.zeros(B, self.chunk_size, dtype=torch.bool, device=device)\n", + " action_local = torch.zeros(B, self.chunk_size, self.action_dim, dtype=torch.float32, device=device)\n", + " batch = {\n", + " 'observation.state': observation_state,\n", + " 'action_is_pad': action_is_pad_local,\n", + " 'action': action_local,\n", + " 'observation.images': list(cam_tensors)\n", + " }\n", + " for i, key in enumerate(self.camera_keys):\n", + " batch[key] = cam_tensors[i]\n", + " if env_tensor is not None:\n", + " batch['observation.environment_state'] = env_tensor\n", + " out = self.model.model(batch)\n", + " if isinstance(out, tuple):\n", + " out = out[0]\n", + " return out\n", + "\n", + "# Construct example inputs for tracing\n", + "example_inputs = [torch.randn_like(state)] + [torch.randn_like(t) for t in image_tensors]\n", + "if env_present:\n", + " example_inputs.append(torch.randn_like(env_state))\n", + "\n", + "wrapper = DirectWrapper(policy, camera_keys, chunk_size, action_dim, env_present).eval()\n", + "print('[DIRECT] Converting via ov.convert_model (FX)...')\n", + "ov_model = ov.convert_model(wrapper, example_input=tuple(example_inputs))\n", + "\n", + "# Rename ports to evaluation expectations\n", + "inputs = ov_model.inputs\n", + "expected = 1 + len(camera_keys) + (1 if env_present else 0)\n", + "if len(inputs) != expected:\n", + " raise RuntimeError(f'Unexpected IR input count {len(inputs)} vs expected {expected}.')\n", + "\n", + "def _set_names(inp, desired: str):\n", + " node = inp.get_node()\n", + " try:\n", + " node.set_friendly_name(desired)\n", + " except Exception:\n", + " pass\n", + " try:\n", + " inp.get_tensor().set_names({desired})\n", " except Exception as e:\n", - " raise RuntimeError(f'Model Optimizer failed. Ensure openvino-dev installed. Original error: {e}')" + " print('[WARN] Failed to set tensor name for', desired, ':', e)\n", + "\n", + "_set_names(inputs[0], 'observation_state')\n", + "for i in range(len(camera_keys)):\n", + " _set_names(inputs[i+1], f'observation_images_{i}')\n", + "if env_present:\n", + " _set_names(inputs[-1], 'observation_environment_state')\n", + "\n", + "print('[DIRECT] Final IR input ports (friendly_name / tensor names / partial shape):')\n", + "dynamic_present = False\n", + "for inp in ov_model.inputs:\n", + " node = inp.get_node()\n", + " try:\n", + " tnames = list(inp.get_tensor().get_names())\n", + " except Exception:\n", + " tnames = []\n", + " try:\n", + " ps = inp.get_partial_shape()\n", + " except Exception:\n", + " ps = None\n", + " if ps is not None and ps.is_static:\n", + " try:\n", + " concrete = ps.to_shape()\n", + " shape_repr = '[' + ', '.join(str(d) for d in concrete) + ']'\n", + " except Exception:\n", + " shape_repr = str(ps)\n", + " else:\n", + " dynamic_present = True\n", + " shape_repr = str(ps) if ps is not None else 'Unknown(dynamic)'\n", + " print(f\" - {tnames[0] if tnames else node.get_friendly_name()} | tensor_names={tnames} | partial_shape={shape_repr}\")\n", + "\n", + "IR_OUTPUT_DIR = pathlib.Path(globals().get('IR_OUTPUT_DIR', 'openvino_ir_outputs'))\n", + "IR_OUTPUT_DIR.mkdir(exist_ok=True)\n", + "\n", + "# Save FP32 IR\n", + "xml_fp32 = IR_OUTPUT_DIR / 'act_model_direct_fp32.xml'\n", + "ov.save_model(ov_model, str(xml_fp32))\n", + "print('[DIRECT] Saved FP32 XML:', xml_fp32, '| size:', xml_fp32.stat().st_size if xml_fp32.exists() else 0)\n", + "print('[DIRECT] Saved FP32 BIN :', xml_fp32.with_suffix('.bin'), '| size:', xml_fp32.with_suffix('.bin').stat().st_size if xml_fp32.with_suffix('.bin').exists() else 0)\n", + "\n", + "# --- FP16 Guidance ---\n", + "# To also emit an FP16 version (weights compressed to half precision) uncomment:\n", + "# xml_fp16 = IR_OUTPUT_DIR / 'act_model_direct_fp16.xml'\n", + "# ov.save_model(ov_model, str(xml_fp16), compress_to_fp16=True)\n", + "# print('[DIRECT] Saved FP16 XML:', xml_fp16)\n", + "# print('[DIRECT] Saved FP16 BIN :', xml_fp16.with_suffix('.bin'))\n", + "\n", + "print('\\n[HINT] In evaluation build input dict using: observation_state, observation_images_0..N, (optional) observation_environment_state.')\n", + "print('[DONE] Direct conversion complete. (See comments above for FP16 save).')\n" + ] + }, + { + "cell_type": "markdown", + "id": "f107d7b8", + "metadata": {}, + "source": [ + "### Optional: INT8 Quantization (Post-Training)\n", + "This section generates an INT8 (quantized) OpenVINO model using the helper script `quantize_int8_helper.py` found in this folder.\n", + "\n", + "Why INT8?\n", + "- Smaller binary size.\n", + "- Potential throughput / latency gains (depends on CPU / GPU / VPU).\n", + "- Usually minimal accuracy drop if calibration data is representative.\n", + "\n", + "What you need first:\n", + "1. A FP32 IR (e.g. `act_model_direct_fp32.xml` created above).\n", + "2. `stats.json` from training (already exported earlier or placed into `act_checkpoint/`).\n", + "3. A local LeRobot dataset root with episode data (env var `ACT_DATASET_ROOT` or edit path below).\n", + "4. Packages: `openvino-dev` and `nncf` installed.\n", + "\n", + "Calibration parameters:\n", + "- `num_calib_samples`: how many sequential steps to sample (default 300). Increase if quality degrades.\n", + "- `preset`: `performance` (aggressive compression) or `accuracy` (more conservative).\n", + "\n", + "Outputs:\n", + "- `int8/model_int8.xml` and `int8/model_int8.bin` in the IR output directory.\n", + "\n", + "Note:\n", + "- Typical runtime: ~2–15 minutes for 300 samples on CPU; faster on a modern GPU." ] }, { "cell_type": "code", "execution_count": null, - "id": "23dcc638", + "id": "e173d050", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "The following IR files present:\n", - "- act_model_fp32.xml MISSING | size: 0\n", - "- act_model_fp32.bin MISSING | size: 0\n" + "[INT8] Resolved paths:\n", + " IR_OUTPUT_DIR = openvino_ir_outputs\n", + " FP32_XML = openvino_ir_outputs/act_model_direct_fp32.xml\n", + " STATS_JSON = act_checkpoint/stats.json\n", + " DATASET_ROOT = dataset/G1_BlockStacking_Dataset\n", + " SCRIPT_PATH = quantize_int8_helper.py\n", + " OUT_INT8_DIR = openvino_ir_outputs/int8\n", + " CALIB_SAMPLES = 300\n", + " PRESET = performance\n", + "\n", + "[INT8] Starting quantization at 2025-11-07T21:16:17.988208Z\n", + "[INT8] Using FP32 model: act_model_direct_fp32.xml\n", + "[INT8] Stats file : stats.json\n", + "[INT8] Dataset root : dataset/G1_BlockStacking_Dataset\n", + "[INT8] Output directory : openvino_ir_outputs/int8\n", + "[INT8] Calibration samples=300 preset=performance\n", + "[INT8] Running helper script with args:\n", + " quantize_int8_helper.py --model_xml openvino_ir_outputs/act_model_direct_fp32.xml --stats_path act_checkpoint/stats.json --dataset_root dataset/G1_BlockStacking_Dataset --output_dir openvino_ir_outputs/int8 --num_calib_samples 300 --preset performance\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n", + "WARNING:root:\n", + "The dataset you requested (None) is in 2.0 format.\n", + "While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global\n", + "stats instead of per-episode stats. Update your dataset stats to the new format using this command:\n", + "```\n", + "python -m lerobot.datasets.v21.convert_dataset_v20_to_v21 --repo-id=None\n", + "```\n", + "\n", + "If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)\n", + "or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).\n", + "\n" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bf5ca5d1fc4f48f78fabad537fde9ece", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Resolving data files: 0%| | 0/301 [00:00 openvino_ir_outputs/int8/model_int8.xml (device=CPU, precision=INT8)\n", + "[EVAL] CMD: /home/case/miniforge3/envs/unitree_lerobot/bin/python eval_openvino_model_helper.py --repo_id=None --root=dataset/G1_BlockStacking_Dataset --policy.path=act_checkpoint --episodes=0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", + " warnings.warn(\n", + "INFO 2025-11-07 11:25:52 l_helper.py:376 {'arm': 'G1_29',\n", + " 'ee': 'dex3',\n", + " 'episodes': 0,\n", + " 'frequency': 30.0,\n", + " 'headless': False,\n", + " 'motion': False,\n", + " 'policy': {'chunk_size': 100,\n", + " 'device': 'cuda',\n", + " 'dim_feedforward': 3200,\n", + " 'dim_model': 512,\n", + " 'dropout': 0.1,\n", + " 'feedforward_activation': 'relu',\n", + " 'input_features': {'observation.images.cam_left_high': {'shape': (3,\n", + " 480,\n", + " 640),\n", + " 'type': },\n", + " 'observation.images.cam_left_wrist': {'shape': (3,\n", + " 480,\n", + " 640),\n", + " 'type': },\n", + " 'observation.images.cam_right_high': {'shape': (3,\n", + " 480,\n", + " 640),\n", + " 'type': },\n", + " 'observation.images.cam_right_wrist': {'shape': (3,\n", + " 480,\n", + " 640),\n", + " 'type': },\n", + " 'observation.state': {'shape': (28,),\n", + " 'type': }},\n", + " 'kl_weight': 10.0,\n", + " 'latent_dim': 32,\n", + " 'license': None,\n", + " 'n_action_steps': 100,\n", + " 'n_decoder_layers': 1,\n", + " 'n_encoder_layers': 4,\n", + " 'n_heads': 8,\n", + " 'n_obs_steps': 1,\n", + " 'n_vae_encoder_layers': 4,\n", + " 'normalization_mapping': {'ACTION': ,\n", + " 'STATE': ,\n", + " 'VISUAL': },\n", + " 'optimizer_lr': 1e-05,\n", + " 'optimizer_lr_backbone': 1e-05,\n", + " 'optimizer_weight_decay': 0.0001,\n", + " 'output_features': {'action': {'shape': (28,),\n", + " 'type': }},\n", + " 'pre_norm': False,\n", + " 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1',\n", + " 'private': None,\n", + " 'push_to_hub': False,\n", + " 'replace_final_stride_with_dilation': 0,\n", + " 'repo_id': None,\n", + " 'tags': None,\n", + " 'temporal_ensemble_coeff': None,\n", + " 'use_amp': False,\n", + " 'use_vae': True,\n", + " 'vision_backbone': 'resnet18'},\n", + " 'repo_id': 'None',\n", + " 'root': 'dataset/G1_BlockStacking_Dataset',\n", + " 'send_real_robot': False,\n", + " 'use_dataset': False,\n", + " 'visualization': False}\n", + "WARNING 2025-11-07 11:25:52 ts/utils.py:302 \n", + "The dataset you requested (None) is in 2.0 format.\n", + "While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global\n", + "stats instead of per-episode stats. Update your dataset stats to the new format using this command:\n", + "```\n", + "python -m lerobot.datasets.v21.convert_dataset_v20_to_v21 --repo-id=None\n", + "```\n", + "\n", + "If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)\n", + "or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).\n", + "\n", + "INFO 2025-11-07 11:25:59 l_helper.py:392 Using OpenVINO model path: openvino_ir_outputs/int8/model_int8.xml; stats path: act_checkpoint/stats.json (env model=True, env stats=True)\n", + "INFO 2025-11-07 11:25:59 l_helper.py:452 Compiling OpenVINO model for device=CPU with INFERENCE_PRECISION_HINT=INT8\n", + "INFO 2025-11-07 11:26:06 l_helper.py:477 OpenVINO model loaded: openvino_ir_outputs/int8/model_int8.xml (device=CPU, precision=INT8, cameras=['cam_left_high', 'cam_left_wrist', 'cam_right_high', 'cam_right_wrist'], state_dim=28, temporal_ensemble=on, chunk_size=100, coeff=0.0100)\n", + " 0%| | 0/1173 [00:00 actions_comparison_int8.png\n", + "\n", + "[EVAL] Summary:\n", + " - actions_comparison_int8.png\n", + "[EVAL] Done.\n" + ] + } + ], + "source": [ + "# Evaluation & Comparison Plotting (ENV VAR path & device + precision hints)\n", + "\"\"\"\n", + "Sets OPENVINO_MODEL_PATH, STATS_PATH, OPENVINO_DEVICE, and OPENVINO_PRECISION_HINT per variant.\n", + "\n", + "\"\"\"\n", + "import sys, pathlib, subprocess, os, datetime, shutil\n", + "\n", + "for sym in ['IR_OUTPUT_DIR', 'CHECKPOINT_PATH', 'STATS_PATH', 'TARGET_DEVICE']:\n", + " if sym not in globals():\n", + " raise RuntimeError(f'Missing required symbol `{sym}`; rerun earlier cells.')\n", + "\n", + "EVAL_SCRIPT = pathlib.Path('eval_openvino_model_helper.py')\n", + "if not EVAL_SCRIPT.exists():\n", + " raise FileNotFoundError(f'Helper script missing: {EVAL_SCRIPT}')\n", + "\n", + "# Resolve stats path (fallback to dataset meta)\n", + "stats_path = pathlib.Path(STATS_PATH)\n", + "DATASET_ROOT = pathlib.Path('dataset/G1_BlockStacking_Dataset')\n", + "if not stats_path.exists():\n", + " fallback = DATASET_ROOT / 'meta' / 'stats.json'\n", + " if fallback.exists():\n", + " stats_path = fallback\n", + " else:\n", + " raise FileNotFoundError(f'stats.json not found at {STATS_PATH} or {fallback}')\n", + "\n", + "# Gather variants dynamically (include fp16 if present)\n", + "MODEL_VARIANTS = [\n", + " ('direct_fp32', IR_OUTPUT_DIR / 'act_model_direct_fp32.xml'),\n", + " ('direct_fp16', IR_OUTPUT_DIR / 'act_model_direct_fp16.xml'),\n", + " ('mo_fp32', IR_OUTPUT_DIR / 'act_model_fp32.xml'),\n", + " ('int8', IR_OUTPUT_DIR / 'int8' / 'model_int8.xml'),\n", + "]\n", + "MODEL_VARIANTS = [(lbl, p) for lbl, p in MODEL_VARIANTS if p.exists()]\n", + "if not MODEL_VARIANTS:\n", + " raise RuntimeError('No model variants found. Convert / quantize first.')\n", + "\n", + "print('[EVAL] Variants:', ', '.join(lbl for lbl,_ in MODEL_VARIANTS))\n", + "print('[EVAL] Stats path:', stats_path)\n", + "print('[EVAL] Root path :', DATASET_ROOT)\n", + "print('[EVAL] Policy path:', CHECKPOINT_PATH.parent)\n", + "print('[EVAL] Device :', TARGET_DEVICE)\n", + "\n", + "base_env = os.environ.copy()\n", + "base_env['PYTHONWARNINGS'] = 'ignore'\n", + "base_env['OPENVINO_DEVICE'] = TARGET_DEVICE\n", + "figures = []\n", + "\n", + "def infer_precision(label, path):\n", + " lp = label.lower()\n", + " fname = str(path).lower()\n", + " if 'int8' in lp or 'int8' in fname:\n", + " return 'INT8'\n", + " if 'fp16' in lp or 'fp16' in fname:\n", + " return 'FP16'\n", + " return 'FP32'\n", + "\n", + "for label, model_xml in MODEL_VARIANTS:\n", + " precision_hint = infer_precision(label, model_xml)\n", + " print(f\"\\n[EVAL] Variant '{label}' -> {model_xml} (device={TARGET_DEVICE}, precision={precision_hint})\")\n", + " run_env = base_env.copy()\n", + " run_env['OPENVINO_MODEL_PATH'] = str(model_xml)\n", + " run_env['STATS_PATH'] = str(stats_path)\n", + " run_env['OPENVINO_PRECISION_HINT'] = precision_hint\n", + " cmd = [\n", + " sys.executable,\n", + " str(EVAL_SCRIPT),\n", + " '--repo_id=None',\n", + " f'--root={DATASET_ROOT}',\n", + " f'--policy.path={CHECKPOINT_PATH.parent}',\n", + " '--episodes=0',\n", + " ]\n", + " print('[EVAL] CMD:', ' '.join(cmd))\n", + " try:\n", + " subprocess.run(cmd, check=True, env=run_env)\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"[EVAL][ERROR] {label} failed: {e}\")\n", + " continue\n", + "\n", + " candidates = [pathlib.Path('actions_comparison.png')]\n", + " fig_src = next((c for c in candidates if c.exists()), None)\n", + " if fig_src:\n", + " timestamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')\n", + " fig_dst = pathlib.Path(f'{fig_src.stem}_{label}.png')\n", + " if fig_dst.exists():\n", + " fig_dst = pathlib.Path(f'{fig_src.stem}_{label}_{timestamp}.png')\n", + " shutil.move(str(fig_src), str(fig_dst))\n", + " figures.append(fig_dst)\n", + " print('[EVAL] Saved figure ->', fig_dst)\n", + " else:\n", + " print('[EVAL][WARN] No comparison figure produced for', label)\n", + "\n", + "print('\\n[EVAL] Summary:')\n", + "for f in figures:\n", + " print(' -', f)\n", + "if not figures:\n", + " print('[EVAL] No figures generated.')\n", + "print('[EVAL] Done.')" ] }, { "cell_type": "code", "execution_count": null, - "id": "01bab214-c42b-4388-8b39-2227f75640c0", + "id": "e8229fcb-4af9-49bf-801a-4deb4a65a572", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/lerobot_act/eval_openvino_model_helper.py b/notebooks/lerobot_act/eval_openvino_model_helper.py new file mode 100644 index 00000000000..9b39d8504db --- /dev/null +++ b/notebooks/lerobot_act/eval_openvino_model_helper.py @@ -0,0 +1,501 @@ +"""Model action comparison.""" + +import logging +import time +import json +import sys +import os +from dataclasses import asdict +from pprint import pformat + +import matplotlib.pyplot as plt +import numpy as np +import torch +import tqdm +from openvino.runtime import Core + +from lerobot.configs import parser +from lerobot.datasets.lerobot_dataset import LeRobotDataset +from lerobot.policies.factory import make_policy +from lerobot.policies.normalize import Normalize, Unnormalize +from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature +from lerobot.utils.utils import get_safe_torch_device, init_logging +from unitree_lerobot.eval_robot.utils.utils import ( + extract_observation, + predict_action, + EvalRealConfig, +) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +DEFAULT_STATS_PATH = "/G1_BlockStacking_Dataset/meta/stats.json" +DEFAULT_OPENVINO_MODEL_PATH = "/path_to_model.xml" + +DEFAULT_CHUNK_STRATEGY = "first" # options: first, mean +DEFAULT_TEMPORAL_ENSEMBLE_COEFF = 0.01 +DEFAULT_CHUNK_SIZE = 100 + +# Set OPENVINO_MODEL_PATH and STATS_PATH before invoking this script. +# Falls back to defaults if env vars not present. +OPENVINO_MODEL_ENV = os.getenv("OPENVINO_MODEL_PATH") +STATS_PATH_ENV = os.getenv("STATS_PATH") +OPENVINO_DEVICE_ENV = os.getenv("OPENVINO_DEVICE") or os.getenv("OV_DEVICE") or "CPU" +ALLOWED_OPENVINO_DEVICES = {"CPU", "GPU", "NPU", "AUTO"} + +######################### +# OpenVINO Helper Logic # +######################### +def load_norm_stats(stats_path: str): + with open(stats_path, "r") as f: + return json.load(f) + + +def detect_camera_keys(norm_stats: dict): + return sorted({k.split(".")[-1] for k in norm_stats.keys() if k.startswith("observation.images.")}) + + +def build_normalizer(norm_stats: dict, camera_names, state_dim: int): + features = {"observation.state": PolicyFeature(type=FeatureType.STATE, shape=(state_dim,))} + for cam in camera_names: + features[f"observation.images.{cam}"] = PolicyFeature(type=FeatureType.VISUAL, shape=(3, 480, 640)) + norm_map = {FeatureType.STATE: NormalizationMode.MEAN_STD, FeatureType.VISUAL: NormalizationMode.MEAN_STD} + stats = { + "observation.state": { + "mean": torch.tensor(norm_stats["observation.state"]["mean"], dtype=torch.float32), + "std": torch.tensor(norm_stats["observation.state"]["std"], dtype=torch.float32), + } + } + for cam in camera_names: + stats[f"observation.images.{cam}"] = { + "mean": torch.tensor(norm_stats[f"observation.images.{cam}"]["mean"], dtype=torch.float32).reshape(3, 1, 1), + "std": torch.tensor(norm_stats[f"observation.images.{cam}"]["std"], dtype=torch.float32).reshape(3, 1, 1), + } + return Normalize(features, norm_map, stats) + + +def build_unnormalizer(norm_stats: dict): + action_shape = (len(norm_stats["action"]["mean"]),) + features = {"action": PolicyFeature(type=FeatureType.ACTION, shape=action_shape)} + norm_map = {FeatureType.ACTION: NormalizationMode.MEAN_STD} + stats = { + "action": { + "mean": torch.tensor(norm_stats["action"]["mean"], dtype=torch.float32), + "std": torch.tensor(norm_stats["action"]["std"], dtype=torch.float32), + } + } + return Unnormalize(features, norm_map, stats) + +def predict_action_openvino(observation, compiled_model, input_names, normalizer, camera_names, state_dim): + inputs = {} + # Prepare input dict for normalization + state = observation.get("observation.state", None) + input_dict = {} + if state is not None: + if hasattr(state, 'cpu'): + state = state.cpu().numpy().astype(np.float32) + state = state.reshape(1, -1) + input_dict["observation.state"] = torch.from_numpy(state) + for cam in camera_names: + img = observation.get(f"observation.images.{cam}", None) + if img is not None: + if hasattr(img, 'cpu'): + img = img.cpu().numpy().astype(np.float32) + if img.ndim == 3 and img.shape[2] == 3: + img = np.transpose(img, (2, 0, 1)) + if img.shape[-2:] != (480, 640): + from cv2 import resize + img = np.transpose(img, (1, 2, 0)) if img.shape[0] == 3 else img + img = resize(img, (640, 480)) + img = np.transpose(img, (2, 0, 1)) + img = img.astype(np.float32) + input_dict[f"observation.images.{cam}"] = torch.from_numpy(img) + # Normalize + normed = normalizer(input_dict) + if "observation.state" in normed: + inputs["observation_state"] = normed["observation.state"].numpy() + else: + inputs["observation_state"] = np.zeros((1, state_dim), dtype=np.float32) + for i, cam in enumerate(camera_names): + key = f"observation.images.{cam}" + if key in normed: + img = normed[key] + img = img[None, ...].numpy() + inputs[f"observation_images_{i}"] = img + else: + inputs[f"observation_images_{i}"] = np.zeros((1, 3, 480, 640), dtype=np.float32) + result = compiled_model(inputs) + action = result[list(result.keys())[0]] + return action.squeeze(0) + +# ---------------- Prediction Wrapper ---------------- # +def predict_action_safetensor(observation, policy, device, use_amp, use_dataset=True): + """Return numpy action predicted by the PyTorch (safetensor) policy.""" + act = predict_action(observation, policy, device, use_amp, use_dataset=use_dataset) + return act.detach().cpu().numpy() + +############################# +# (Optional) Temporal Smoothing +############################# +class ACTTemporalEnsembler: + def __init__(self, temporal_ensemble_coeff: float, chunk_size: int): + self.chunk_size = chunk_size + self.ensemble_weights = torch.exp(-temporal_ensemble_coeff * torch.arange(chunk_size)) + self.ensemble_weights_cumsum = torch.cumsum(self.ensemble_weights, dim=0) + self.reset() + + def reset(self): + self.ensembled_actions = None + self.ensembled_actions_count = None + + def update(self, actions: np.ndarray) -> np.ndarray: + actions = torch.from_numpy(actions) + self.ensemble_weights = self.ensemble_weights.to(actions.device) + self.ensemble_weights_cumsum = self.ensemble_weights_cumsum.to(actions.device) + if self.ensembled_actions is None: + self.ensembled_actions = actions.clone() + self.ensembled_actions_count = torch.ones((self.chunk_size, 1), dtype=torch.long, device=actions.device) + else: + self.ensembled_actions *= self.ensemble_weights_cumsum[self.ensembled_actions_count - 1] + self.ensembled_actions += actions[:, :-1] * self.ensemble_weights[self.ensembled_actions_count] + self.ensembled_actions /= self.ensemble_weights_cumsum[self.ensembled_actions_count] + self.ensembled_actions_count = torch.clamp(self.ensembled_actions_count + 1, max=self.chunk_size) + self.ensembled_actions = torch.cat([self.ensembled_actions, actions[:, -1:]], dim=1) + self.ensembled_actions_count = torch.cat([ + self.ensembled_actions_count, torch.ones_like(self.ensembled_actions_count[-1:]) + ]) + action, self.ensembled_actions, self.ensembled_actions_count = ( + self.ensembled_actions[:, 0], + self.ensembled_actions[:, 1:], + self.ensembled_actions_count[1:], + ) + return action.cpu().numpy() + + +# ---------------- Core Loop ---------------- # +def run_comparison_loop(cfg, dataset, policy, ov_ctx=None): + """Iterate over dataset steps and collect model + ground truth actions. + + Returns (actions_dict, ground_truth_actions). + """ + device = get_safe_torch_device(policy.config.device) + use_amp = getattr(policy.config, "use_amp", False) + + from_idx = dataset.episode_data_index["from"][0].item() + to_idx = dataset.episode_data_index["to"][0].item() + + safetensor_actions = [] + openvino_actions = [] if ov_ctx is not None else None + if ov_ctx is not None: + compiled_model = ov_ctx["compiled_model"] + input_names = ov_ctx.get("input_names", []) + normalizer = ov_ctx["normalizer"] + unnormalizer = ov_ctx["unnormalizer"] + camera_names = ov_ctx["camera_names"] + state_dim = ov_ctx["state_dim"] + chunk_size = ov_ctx.get("chunk_size") + ensembler = ov_ctx.get("ensembler") + chunk_strategy = ov_ctx.get("chunk_strategy", DEFAULT_CHUNK_STRATEGY) + gt_actions = [] + + for i in tqdm.tqdm(range(from_idx, to_idx)): + loop_start = time.perf_counter() + step = dataset[i] + obs = extract_observation(step) + safetensor_actions.append(predict_action_safetensor(obs, policy, device, use_amp, use_dataset=True)) + if ov_ctx is not None: + # New signature call path + ov_pred = predict_action_openvino( + obs, + compiled_model=compiled_model, + input_names=input_names, + normalizer=normalizer, + camera_names=camera_names, + state_dim=state_dim, + ) + # Ensure numpy array + if isinstance(ov_pred, torch.Tensor): + norm_arr = ov_pred.detach().cpu().numpy() + else: + norm_arr = np.asarray(ov_pred) + + # If model outputs a temporal chunk, optionally ensemble or reduce before unnormalization + if norm_arr.ndim == 2 and norm_arr.shape[0] > 1: # (chunk, A) + if ensembler is not None and chunk_size and norm_arr.shape[0] == chunk_size: + try: + norm_arr = ensembler.update(norm_arr[None, ...]) # returns (1, A) typically + except Exception: + pass # fall back to strategy below + if norm_arr.ndim == 2 and norm_arr.shape[0] > 1: # still unresolved (chunk, A) + if chunk_strategy == "mean": + norm_arr = norm_arr.mean(axis=0) + else: + norm_arr = norm_arr[0] + + # Unnormalize (expects dict with action tensor) + norm_tensor = torch.from_numpy(norm_arr.astype(np.float32)) + try: + unnorm = unnormalizer({"action": norm_tensor})["action"].numpy() + except Exception: + unnorm = norm_tensor.numpy() + + # Squeeze potential leading singleton dimensions + while unnorm.ndim > 1 and unnorm.shape[0] == 1: + unnorm = unnorm.squeeze(0) + if unnorm.ndim > 2: # unexpected extra dims -> flatten last + unnorm = unnorm.reshape(-1)[: norm_tensor.shape[-1]] + openvino_actions.append(unnorm) + gt = step["action"] + if hasattr(gt, "cpu"): + gt = gt.cpu().numpy() + gt_actions.append(gt) + if getattr(cfg, "frequency", None): + time.sleep(max(0, (1.0 / cfg.frequency) - (time.perf_counter() - loop_start))) + gt_np = np.asarray(gt_actions) + actions_dict = {"safetensor": np.asarray(safetensor_actions)} + if openvino_actions is not None: + ov_arr = np.asarray(openvino_actions, dtype=np.float32) + # Collapse shapes like (T,1,chunk,D) or (T,1,D) + if ov_arr.ndim == 4 and ov_arr.shape[1] == 1: # (T,1,chunk,D) + ov_arr = ov_arr[:, 0, :, :] + if ov_arr.ndim == 3: + # (T,chunk,D) -> reduce chunk + strategy = ov_ctx.get("chunk_strategy", DEFAULT_CHUNK_STRATEGY) if ov_ctx else DEFAULT_CHUNK_STRATEGY + if ov_arr.shape[1] > 1: + ov_arr = ov_arr.mean(axis=1) if strategy == "mean" else ov_arr[:, 0, :] + else: # (T,1,D) + ov_arr = ov_arr[:, 0, :] + actions_dict["openvino"] = ov_arr + return actions_dict, gt_np + + +# ---------------- Plotting ---------------- # +def plot_comparison(actions_dict, ground_truth_actions, out_path="actions_comparison.png"): + """Plot actions side-by-side: Left Arm joints in left column, Right Arm joints in right column. + + - Joint names + per-model μ/σ in each subplot. + - If joint names don't include left/right, first half assumed left, second half right. + - Handles unequal counts by leaving blank cells. + """ + if not actions_dict: + logger.warning("No actions to plot.") + return + sample = next(iter(actions_dict.values())) + _, n_dims = sample.shape + try: + from unitree_lerobot.utils.constants import G1_INSPIRE_CONFIG + joint_names = G1_INSPIRE_CONFIG.motors + if len(joint_names) != n_dims: + joint_names = [f"Joint {i+1}" for i in range(n_dims)] + except Exception: + joint_names = [f"Joint {i+1}" for i in range(n_dims)] + + preferred_order = [m for m in ["safetensor", "openvino"] if m in actions_dict] + others = [m for m in actions_dict.keys() if m not in preferred_order] + models = preferred_order + others + + colors = ["red", "green", "orange", "purple", "brown", "cyan"] + styles = [":", "--", "-.", "-", (0, (3,1,1,1)), (0, (5,2))] + + stats = {} + for m in models: + err = actions_dict[m] - ground_truth_actions + stats[m] = (np.mean(err, axis=0), np.std(err, axis=0)) + + left_indices, right_indices = [], [] + for idx, name in enumerate(joint_names): + lname = name.lower() + if "left" in lname: + left_indices.append(idx) + elif "right" in lname: + right_indices.append(idx) + if not left_indices and not right_indices: + half = n_dims // 2 + left_indices = list(range(half)) + right_indices = list(range(half, n_dims)) + if not left_indices: + left_indices = [i for i in range(n_dims) if i not in right_indices] + if not right_indices: + right_indices = [i for i in range(n_dims) if i not in left_indices] + + n_rows = max(len(left_indices), len(right_indices)) + n_cols = 2 if right_indices else 1 + fig, axes = plt.subplots(n_rows, n_cols, figsize=(14 if n_cols==2 else 7, 3.0 * n_rows), sharex=False) + if n_rows == 1 and n_cols == 1: + axes = [[axes]] + elif n_rows == 1: + axes = [axes] + + fig.suptitle("Action Comparison (Left vs Right Arm)", fontsize=12) + + TITLE_FS = 10 + LABEL_FS = 12 + TICK_FS = 12 + + def _plot_side(side_indices, col, side_name): + for row in range(n_rows): + ax = axes[row][col] if n_cols == 2 else axes[row][0] + if row >= len(side_indices): + ax.axis('off') + continue + j_idx = side_indices[row] + ax.plot(ground_truth_actions[:, j_idx], label="Ground Truth", color="blue", linewidth=1.2) + for k, m in enumerate(models): + ax.plot( + actions_dict[m][:, j_idx], + label=f"{m} (μ={stats[m][0][j_idx]:.3f}, σ={stats[m][1][j_idx]:.3f})", + color=colors[k % len(colors)], + linestyle=styles[k % len(styles)], + linewidth=1.0, + ) + summary_parts = [f"{m}:μ={stats[m][0][j_idx]:.2f} σ={stats[m][1][j_idx]:.2f}" for m in models[:2]] + ax.set_title(f"{side_name} - {joint_names[j_idx]} ({' | '.join(summary_parts)})", fontsize=TITLE_FS) + ax.set_ylabel("Val", fontsize=LABEL_FS) + ax.grid(alpha=0.25, linestyle=":") + ax.tick_params(axis='both', labelsize=TICK_FS) + if row == 0: + ax.legend(fontsize=7, ncol=2, loc="upper right") + if row == n_rows - 1: + ax.set_xlabel("Timestep", fontsize=LABEL_FS) + ax.tick_params(axis='both', labelsize=TICK_FS) + + _plot_side(left_indices, 0, "Left Arm") + if n_cols == 2: + _plot_side(right_indices, 1, "Right Arm") + + plt.tight_layout(rect=[0, 0, 1, 0.96]) + plt.savefig(out_path) + logger.info( + f"Saved comparison figure to {out_path} (grid {n_rows}x{n_cols}; left joints={len(left_indices)}, right joints={len(right_indices)})" + ) + + +@parser.wrap() +def eval_main(cfg: EvalRealConfig): + logging.info(pformat(asdict(cfg))) + + dataset = LeRobotDataset( + repo_id=None if (cfg.repo_id is None or str(cfg.repo_id).lower() == "none") else cfg.repo_id, + root=cfg.root, + ) + policy = make_policy(cfg=cfg.policy, ds_meta=dataset.meta) + policy.eval() + if hasattr(policy, "reset"): + policy.reset() + + + # Optional OpenVINO setup + import os + ov_model_path = OPENVINO_MODEL_ENV or DEFAULT_OPENVINO_MODEL_PATH + stats_path = STATS_PATH_ENV or DEFAULT_STATS_PATH + logging.info(f"Using OpenVINO model path: {ov_model_path}; stats path: {stats_path} (env model={OPENVINO_MODEL_ENV is not None}, env stats={STATS_PATH_ENV is not None})") + if not os.path.exists(ov_model_path): + logging.warning(f"OpenVINO model path does not exist: {ov_model_path}. Skipping OpenVINO.") + if not os.path.exists(stats_path): + logging.warning(f"Stats path does not exist: {stats_path}. Skipping OpenVINO.") + state_dim_arg = parser.parse_arg("state_dim") + + ov_ctx = None + if os.path.exists(ov_model_path) and os.path.exists(stats_path): + try: + if state_dim_arg: + state_dim = int(state_dim_arg) + else: + # Try infer from stats + try: + stats_preview = load_norm_stats(stats_path) + state_dim = len(stats_preview["observation.state"]["mean"]) if "observation.state" in stats_preview else None + except Exception: + state_dim = None + if state_dim is None: + # Try dataset meta + try: + meta_state = dataset.meta["observation"]["state"] + if isinstance(meta_state, dict) and "shape" in meta_state: + shape = meta_state["shape"] + if isinstance(shape, (list, tuple)): + state_dim = shape[0] + except Exception: + pass + if state_dim is None: + raise ValueError("Could not determine state_dim (provide --state_dim).") + norm_stats = load_norm_stats(stats_path) + camera_names = detect_camera_keys(norm_stats) + normalizer = build_normalizer(norm_stats, camera_names, state_dim) + unnormalizer = build_unnormalizer(norm_stats) + core = Core() + model = core.read_model(ov_model_path) + ov_device = OPENVINO_DEVICE_ENV.upper() + if ov_device not in ALLOWED_OPENVINO_DEVICES: + logging.warning( + "Requested OPENVINO_DEVICE %s not in allowed %s; falling back to CPU.", + ov_device, + sorted(ALLOWED_OPENVINO_DEVICES), + ) + ov_device = "CPU" + precision_env = os.getenv("OPENVINO_PRECISION_HINT") or os.getenv("OV_PRECISION") + if precision_env is None: + lower_path = ov_model_path.lower() + if "int8" in lower_path: + precision_env = "INT8" + elif "fp16" in lower_path: + precision_env = "FP16" + else: + precision_env = "FP32" + precision_env = precision_env.upper() + allowed_precisions = {"FP32", "FP16", "INT8"} + if precision_env not in allowed_precisions: + logging.warning("Invalid OPENVINO_PRECISION_HINT=%s; falling back to FP32.", precision_env) + precision_env = "FP32" + compile_config = {"INFERENCE_PRECISION_HINT": precision_env} + logging.info( + "Compiling OpenVINO model for device=%s with INFERENCE_PRECISION_HINT=%s", ov_device, precision_env + ) + try: + compiled_model = core.compile_model(model, ov_device, config=compile_config) + except Exception as e: + logging.warning( + "Precision-specific compile failed (%s). Retrying without config: %s", compile_config, e + ) + compiled_model = core.compile_model(model, ov_device) + try: + input_names = [inp.get_any_name() for inp in model.inputs] + except Exception: + input_names = [] + ov_ctx = { + "compiled_model": compiled_model, + "camera_names": camera_names, + "normalizer": normalizer, + "unnormalizer": unnormalizer, + "state_dim": state_dim, + "input_names": input_names, + "chunk_size": DEFAULT_CHUNK_SIZE, + "ensembler": ACTTemporalEnsembler(DEFAULT_TEMPORAL_ENSEMBLE_COEFF, DEFAULT_CHUNK_SIZE), + "chunk_strategy": DEFAULT_CHUNK_STRATEGY, + } + logging.info( + "OpenVINO model loaded: %s (device=%s, precision=%s, cameras=%s, state_dim=%s, temporal_ensemble=on, chunk_size=%d, coeff=%.4f)", + ov_model_path, + ov_device, + precision_env, + camera_names, + state_dim, + DEFAULT_CHUNK_SIZE, + DEFAULT_TEMPORAL_ENSEMBLE_COEFF, + ) + except Exception as e: + logging.warning(f"Failed to initialize OpenVINO path '{ov_model_path}': {e}") + + actions_dict, gt_actions = run_comparison_loop(cfg, dataset, policy, ov_ctx=ov_ctx) + if "openvino" not in actions_dict: + logging.warning("OpenVINO actions not collected; only plotting sensorsafe model.") + else: + logging.info("Models plotted: %s", list(actions_dict.keys())) + plot_comparison(actions_dict, gt_actions) + logging.info("Evaluation complete") + + +if __name__ == "__main__": + init_logging() + eval_main() \ No newline at end of file diff --git a/notebooks/lerobot_act/quantize_int8_helper.py b/notebooks/lerobot_act/quantize_int8_helper.py new file mode 100644 index 00000000000..4438ad9c745 --- /dev/null +++ b/notebooks/lerobot_act/quantize_int8_helper.py @@ -0,0 +1,344 @@ +"""Post-training INT8 quantization for an OpenVINO ACT model. + +This script performs INT8 post‑training quantization (static) using NNCF's + +Requirements: + pip install --upgrade openvino-dev nncf + +Example: + python quantize_int8_helper.py \ + --model_xml /path/to/model.xml \ + --stats_path /path/to/stats.json \ + --dataset_root /path/to/dataset/root \ + --output_dir /path/to/out_int8 --num_calib_samples 300 + +After success you will have model_int8.xml / model_int8.bin. +You can evaluate using your existing eval script by pointing to the INT8 model. +""" +from __future__ import annotations +import argparse +import os +import json +import numpy as np +from typing import List, Dict +import importlib.metadata as md +import warnings + +from openvino.runtime import Core, serialize +import openvino as ov_pkg # keep reference to top-level package for compatibility shims + +# --- Early compatibility probe for openvino.op (required by newer NNCF OpenVINO backend) --- +def _check_openvino_op_module(): + """Detect presence of openvino.op module and provide actionable guidance if missing. + + Newer NNCF releases expect `import openvino.op as op`. This is available only in newer + OpenVINO Python packages (2025.x or certain late 2024 builds). If absent we should abort + before deep inside NNCF with a clear remediation path. + """ + ov_version = getattr(ov_pkg, '__version__', 'unknown') + try: + import importlib + importlib.import_module('openvino.op') # noqa: F401 + return True, ov_version + except Exception: + return False, ov_version + +_has_op, _ov_ver = _check_openvino_op_module() +if not _has_op: + print('[compat] Missing module `openvino.op` (OpenVINO version:', _ov_ver, ')') + print('[compat] Your installed NNCF likely requires a newer OpenVINO Python API exposing `openvino.op`.') + print('[action] Choose ONE option and re-run this script:') + print(' Option A (Recommended): Upgrade OpenVINO stack:') + print(" pip install -U 'openvino>=2025.0.0'") + print(' Option B: Downgrade NNCF to a version compatible with current OpenVINO (e.g. 2.16.0):') + print(" pip install 'nncf<2.18' # example: pip install nncf==2.16.0") + print('[hint] After adjusting packages restart the kernel / environment, then run INT8 cell again.') + # Abort early to avoid ModuleNotFoundError deeper inside NNCF + raise SystemExit('Aborting INT8 quantization due to missing openvino.op module.') + +# We will import nncf after applying an OpenVINO compatibility shim. +def _nncf_versions_report(): + try: + nncf_ver = md.version('nncf') + except md.PackageNotFoundError: + nncf_ver = 'not-installed' + ov_ver = getattr(ov_pkg, '__version__', 'unknown') + return ov_ver, nncf_ver + + +def _apply_openvino_node_shim(): + """Provide openvino.Node alias if missing (newer NNCF expects it). + + """ + try: + import openvino.runtime as ovrt + if not hasattr(ov_pkg, 'Node') and hasattr(ovrt, 'Node'): + ov_pkg.Node = ovrt.Node # type: ignore[attr-defined] + print("[compat] Injected openvino.Node alias -> openvino.runtime.Node") + except Exception as exc: # pragma: no cover + warnings.warn(f"Failed to apply openvino.Node shim: {exc}") + + +def _ensure_version_alignment(): + ov_ver, nncf_ver = _nncf_versions_report() + print(f"[info] OpenVINO version: {ov_ver} | NNCF version: {nncf_ver}") + # Basic heuristics: if nncf >=2.18 but OpenVINO still 2024.*, warn user. + try: + from packaging.version import Version + if nncf_ver not in ('not-installed', 'unknown'): + if Version(nncf_ver) >= Version('2.18') and ov_ver.startswith('2024.'): + print("[warn] Detected nncf >=2.18 with OpenVINO 2024.*. Consider either:\n" + " Upgrade OpenVINO: pip install -U 'openvino-dev>=2025.3.0' 'openvino>=2025.3.0'\n" + " OR downgrade NNCF: pip install 'nncf<2.18' (e.g. nncf==2.16.0)") + except Exception: + pass + + +def _import_nncf(): + try: + from nncf import quantize, Dataset + return quantize, Dataset + except ImportError as e: + raise SystemExit("nncf not installed. Install with: pip install nncf") from e + +# LeRobot utilities (assuming project layout already on PYTHONPATH when run from repo root) +try: + from lerobot.datasets.lerobot_dataset import LeRobotDataset + from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature + from lerobot.policies.normalize import Normalize +except Exception as e: # pragma: no cover + raise SystemExit(f"Failed to import LeRobot packages: {e}") + + +def load_json(path: str): + with open(path, 'r') as f: + return json.load(f) + + +def detect_camera_names(stats: dict) -> List[str]: + return sorted({k.split('.')[-1] for k in stats.keys() if k.startswith('observation.images.')}) + + +def build_normalizer(norm_stats: dict, camera_names: List[str], state_dim: int) -> Normalize: + features = {"observation.state": PolicyFeature(type=FeatureType.STATE, shape=(state_dim,))} + for cam in camera_names: + features[f"observation.images.{cam}"] = PolicyFeature(type=FeatureType.VISUAL, shape=(3, 480, 640)) + norm_map = { + FeatureType.STATE: NormalizationMode.MEAN_STD, + FeatureType.VISUAL: NormalizationMode.MEAN_STD, + } + stats = { + "observation.state": { + "mean": np.asarray(norm_stats["observation.state"]["mean"], dtype=np.float32), + "std": np.asarray(norm_stats["observation.state"]["std"], dtype=np.float32), + } + } + for cam in camera_names: + stats[f"observation.images.{cam}"] = { + "mean": np.asarray(norm_stats[f"observation.images.{cam}"]["mean"], dtype=np.float32).reshape(3, 1, 1), + "std": np.asarray(norm_stats[f"observation.images.{cam}"]["std"], dtype=np.float32).reshape(3, 1, 1), + } + return Normalize(features, norm_map, stats) + + +def infer_state_dim(stats: dict) -> int: + return len(stats["observation.state"]["mean"]) + + +from typing import Tuple + + +def derive_action_and_chunk(model, stats: dict) -> Tuple[int, int, bool]: + """Infer (action_dim, chunk_size, has_action_inputs) with dynamic-shape safety. + """ + action_dim = None + chunk_size = None + has_action_inputs = False + + # Pass 1: direct match on name == 'action' + for inp in model.inputs: + name = inp.get_any_name() + try: + pshape = inp.get_partial_shape() + except Exception: + continue + if name == 'action' and pshape.is_static and len(pshape) == 3: + shape = pshape.to_shape() + chunk_size = int(shape[1]) + action_dim = int(shape[2]) + has_action_inputs = True + break + + # Pass 2: generic 3D search (batch, chunk, action_dim) + if action_dim is None: + for inp in model.inputs: + try: + pshape = inp.get_partial_shape() + except Exception: + continue + if not pshape.is_static or len(pshape) != 3: + continue + shape = pshape.to_shape() + if shape[0] == 1: # likely (1, chunk, action_dim) + chunk_size = int(shape[1]) + action_dim = int(shape[2]) + # Determine if action_is_pad is present + for i2 in model.inputs: + if i2.get_any_name() == 'action_is_pad': + has_action_inputs = True + break + break + + # Fallback: stats.json + if action_dim is None: + if 'action' in stats and 'mean' in stats['action']: + action_dim = len(stats['action']['mean']) + # Prefer explicit chunk_size in stats if provided else 1 + chunk_size = int(stats.get('chunk_size', 1) or 1) + has_action_inputs = False + print(f"[info] No action inputs found in IR. Using stats fallback action_dim={action_dim}, chunk_size={chunk_size}.") + else: + raise ValueError( + "Cannot infer action_dim: stats.json lacks action.mean and model inputs provide no static 3D tensor." + ) + return action_dim, chunk_size, has_action_inputs + + +def build_sample(model_inputs, sample_step, normalizer: Normalize, camera_names: List[str], action_dim: int, chunk_size: int, has_action_inputs: bool) -> Dict[str, np.ndarray]: + # Prepare observation dict for normalization + obs = {} + if "observation.state" in sample_step: + state = sample_step["observation.state"].cpu().numpy().astype(np.float32) + obs["observation.state"] = state + for cam in camera_names: + key = f"observation.images.{cam}" + if key in sample_step: + img = sample_step[key].cpu().numpy().astype(np.float32) + if img.ndim == 3 and img.shape[0] == 3: + pass + elif img.ndim == 3 and img.shape[2] == 3: # HWC -> CHW + img = np.transpose(img, (2, 0, 1)) + obs[key] = img + + import torch + tensor_input = {k: torch.from_numpy(v) for k, v in obs.items()} + normed = normalizer(tensor_input) + + feed = {} + # Map normalization results back to model input names + for inp in model_inputs: + name = inp.get_any_name() + if name == 'observation_state' and 'observation.state' in normed: + feed[name] = normed['observation.state'].unsqueeze(0).numpy() + elif name.startswith('observation_images_'): + idx = int(name.split('_')[-1]) + if idx < len(camera_names): + cam_key = f"observation.images.{camera_names[idx]}" + if cam_key in normed: + feed[name] = normed[cam_key].unsqueeze(0).numpy() + elif has_action_inputs and name == 'action_is_pad': + feed[name] = np.zeros((1, chunk_size), dtype=bool) + elif has_action_inputs and name == 'action': + feed[name] = np.zeros((1, chunk_size, action_dim), dtype=np.float32) + elif name == 'observation_environment_state': + # Dynamic-shape safe extraction of env dim + env_dim = 1 + try: + pshape = inp.get_partial_shape() + if pshape.is_static: + shape = pshape.to_shape() + if len(shape) > 1: + env_dim = int(shape[1]) + except Exception: + pass + feed[name] = np.zeros((1, env_dim), dtype=np.float32) + return feed + + +def collect_calibration_samples(core_model, dataset: LeRobotDataset, normalizer: Normalize, camera_names: List[str], action_dim: int, chunk_size: int, has_action_inputs: bool, num: int): + inputs = core_model.inputs + samples = [] + from_idx = dataset.episode_data_index['from'][0].item() + to_idx = dataset.episode_data_index['to'][0].item() + end = min(to_idx, from_idx + num) + for idx in range(from_idx, end): + step = dataset[idx] + sample = build_sample(inputs, step, normalizer, camera_names, action_dim, chunk_size, has_action_inputs) + samples.append(sample) + return samples + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--model_xml', required=True, help='Path to FP32 OpenVINO model XML') + ap.add_argument('--stats_path', required=True, help='Path to stats.json used for normalization') + ap.add_argument('--dataset_root', required=True, help='Root of LeRobot dataset (local)') + ap.add_argument('--output_dir', required=True, help='Directory to save INT8 model') + ap.add_argument('--num_calib_samples', type=int, default=300, help='Number of calibration samples') + ap.add_argument('--preset', choices=['performance', 'accuracy'], default='performance', help='Quantization preset') + ap.add_argument('--action_dim', type=int, default=None, help='Override action dimension if inference fails') + ap.add_argument('--chunk_size', type=int, default=None, help='Override chunk size if inference fails') + ap.add_argument('--subset_size', type=int, default=None, help='Override subset size (defaults to num_calib_samples)') + args = ap.parse_args() + + os.makedirs(args.output_dir, exist_ok=True) + + core = Core() + model = core.read_model(args.model_xml) + + # Load stats & dataset + stats = load_json(args.stats_path) + camera_names = detect_camera_names(stats) + state_dim = infer_state_dim(stats) + normalizer = build_normalizer(stats, camera_names, state_dim) + dataset = LeRobotDataset(repo_id=None, root=args.dataset_root) + + action_dim, chunk_size, has_action_inputs = derive_action_and_chunk(model, stats) + # CLI overrides + if args.action_dim is not None: + action_dim = args.action_dim + if args.chunk_size is not None: + chunk_size = args.chunk_size + print(f"[info] has_action_inputs={has_action_inputs} action_dim={action_dim} chunk_size={chunk_size}") + + print(f"[info] Cameras={camera_names} state_dim={state_dim} action_dim={action_dim} chunk_size={chunk_size}") + print(f"[info] Collecting {args.num_calib_samples} calibration samples ...") + samples = collect_calibration_samples(model, dataset, normalizer, camera_names, action_dim, chunk_size, has_action_inputs, args.num_calib_samples) + + # Version alignment & compatibility shim before touching nncf internals + _apply_openvino_node_shim() + _ensure_version_alignment() + quantize, Dataset = _import_nncf() + + # Wrap samples for NNCF Dataset (expects iterator over input dicts) + nncf_dataset = Dataset(samples) + subset = args.subset_size or len(samples) + print(f"[info] Quantizing (preset={args.preset}, subset_size={subset}) ...") + try: + # Map string preset to QuantizationPreset enum for NNCF versions that expect enum (avoids AttributeError) + preset_arg = args.preset + try: # lightweight, safe attempt + from nncf.quantization import QuantizationPreset as _QPreset # type: ignore + if isinstance(preset_arg, str): + preset_arg = _QPreset.PERFORMANCE if args.preset == 'performance' else _QPreset.ACCURACY + except Exception: + preset_arg = args.preset # fall back to raw string + quantized_model = quantize(model, nncf_dataset, preset=preset_arg, subset_size=subset) + except AttributeError as attr_err: + if 'openvino' in str(attr_err) and 'Node' in str(attr_err): + print("[error] NNCF encountered missing openvino.Node despite shim. This indicates a deeper version mismatch.") + print("[hint] Fix options: \n" + " 1) Upgrade OpenVINO stack: pip install -U 'openvino>=2025.3.0'\n" + " 2) Downgrade NNCF: pip install 'nncf<2.18' (e.g. nncf==2.16.0)\n" + "Re-run this script after adjusting versions.") + raise + + out_xml = os.path.join(args.output_dir, 'model_int8.xml') + out_bin = os.path.join(args.output_dir, 'model_int8.bin') + serialize(quantized_model, out_xml, out_bin) + print(f"[done] INT8 model saved to: {out_xml} / {out_bin}") + print("Evaluate it with your evaluation script pointing to model_int8.xml") + + +if __name__ == '__main__': + main() From ccaf1c684f758e594be737d0cdcebcd5ca2a1bf6 Mon Sep 17 00:00:00 2001 From: Mohammad Haghighipanah Date: Fri, 7 Nov 2025 16:45:53 -0800 Subject: [PATCH 3/5] clearing cell outputs --- notebooks/lerobot_act/README.md | 8 +- notebooks/lerobot_act/act_to_openvino.ipynb | 493 +------------------- 2 files changed, 25 insertions(+), 476 deletions(-) diff --git a/notebooks/lerobot_act/README.md b/notebooks/lerobot_act/README.md index 707a925e1d8..1cefb44e28e 100644 --- a/notebooks/lerobot_act/README.md +++ b/notebooks/lerobot_act/README.md @@ -118,9 +118,9 @@ openvino_ir_outputs/ model_int8.xml # Post-training quantized INT8 IR model_int8.bin -figure_actions_comparison_direct_fp32.png -figure_actions_comparison_direct_fp16.png -figure_actions_comparison_mo_fp32.png -figure_actions_comparison_int8.png +actions_comparison_direct_fp32.png +actions_comparison_direct_fp16.png +actions_comparison_mo_fp32.png +actions_comparison_int8.png ``` \ No newline at end of file diff --git a/notebooks/lerobot_act/act_to_openvino.ipynb b/notebooks/lerobot_act/act_to_openvino.ipynb index f6d59adbcad..364b89caa4d 100644 --- a/notebooks/lerobot_act/act_to_openvino.ipynb +++ b/notebooks/lerobot_act/act_to_openvino.ipynb @@ -23,60 +23,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "16781bc1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[PIP] /home/case/miniforge3/envs/unitree_lerobot/bin/python -m pip install -U openvino>=2025.0.0\n", - "Requirement already satisfied: openvino>=2025.0.0 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (2025.3.0)\n", - "Requirement already satisfied: numpy<2.3.0,>=1.16.6 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (1.26.4)\n", - "Requirement already satisfied: openvino-telemetry>=2023.2.1 in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (2025.2.0)\n", - "Requirement already satisfied: packaging in /home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages (from openvino>=2025.0.0) (25.0)\n", - "\n", - "[CHECK] Other core package presence (excluding openvino)...\n", - " [OK] openvino\n", - " [OK] nncf\n", - " [OK] torch\n", - " [OK] torchvision\n", - " [OK] accelerate\n", - " [OK] safetensors\n", - " [OK] numpy\n", - " [OK] pandas\n", - " [OK] matplotlib\n", - " [OK] tqdm\n", - " [OK] h5py\n", - " [OK] onnx\n", - " [OK] onnxruntime\n", - " [OK] rich\n", - "[INFO] All other core packages already installed.\n", - "\n", - "[RECHECK] Core imports after installation:\n", - " [OK] openvino\n", - " [OK] nncf\n", - " [OK] torch\n", - " [OK] torchvision\n", - " [OK] accelerate\n", - " [OK] safetensors\n", - " [OK] numpy\n", - " [OK] pandas\n", - " [OK] matplotlib\n", - " [OK] tqdm\n", - " [OK] h5py\n", - " [OK] onnx\n", - " [OK] onnxruntime\n", - " [OK] rich\n", - "\n", - "[CHECK] lerobot availability...\n", - "[OK] lerobot present.\n", - "\n", - "[SUMMARY] Dependencies verified. openvino runtime >= 2025.0.0 (installed: 2025.3.0). Proceed to path configuration cell.\n" - ] - } - ], + "outputs": [], "source": [ "# Dependency Verification & Core Installation (auto-upgrade openvino runtime)\n", "\"\"\"\n", @@ -184,24 +134,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "878e6d73", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook directory: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act\n", - "Relative checkpoint dir: act_checkpoint\n", - "Resolved checkpoint file path: act_checkpoint/model.safetensors\n", - "Dataset root: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/dataset\n", - "Stats path (may not exist yet): act_checkpoint/stats.json\n", - "Output directory: openvino_ir_outputs\n", - "Target device (OpenVINO): CPU\n" - ] - } - ], + "outputs": [], "source": [ "# Configuration Parameters (Paths, Precision, Device)\n", "import os, pathlib\n", @@ -238,23 +174,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "141847a0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[INFO] Checkpoint directory (relative): act_checkpoint\n", - "[INFO] Environment variables:\n", - " ACT_CHECKPOINT = act_checkpoint/model.safetensors\n", - " ACT_CONFIG_PATH = act_checkpoint/config.json\n", - " ACT_TRAIN_CONFIG_PATH = act_checkpoint/train_config.json\n", - " ACT_STATS_PATH = act_checkpoint/stats.json\n" - ] - } - ], + "outputs": [], "source": [ "# Export environment variables\n", "import os, pathlib\n", @@ -274,21 +197,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "aceed7e2", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[LOAD] CHECKPOINT_PATH = act_checkpoint/model.safetensors\n", - "[LOAD] CONFIG_PATH = act_checkpoint/config.json\n", - "Loaded ACTPolicy from safetensors. Params: 34246684\n", - "Detected cameras: ['observation.images.cam_left_high', 'observation.images.cam_left_wrist', 'observation.images.cam_right_high', 'observation.images.cam_right_wrist']\n" - ] - } - ], + "outputs": [], "source": [ "# Load Original ACT Model\n", "import os, json, inspect, pathlib, sys, importlib\n", @@ -353,21 +265,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "667d3f45", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "State shape: torch.Size([1, 28])\n", - "Image shapes: [torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640])]\n", - "Action pad shape: torch.Size([1, 100])\n", - "Action seq shape: torch.Size([1, 100, 28])\n" - ] - } - ], + "outputs": [], "source": [ "# Inspect Model Architecture and Construct Full Dummy Inputs\n", "import torch \n", @@ -403,18 +304,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c301d9af", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ordered input tensor shapes: [torch.Size([1, 28]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 3, 480, 640]), torch.Size([1, 100]), torch.Size([1, 100, 28])]\n" - ] - } - ], + "outputs": [], "source": [ "# Prepare Ordered Inputs\n", "# Order: observation.state, each camera image, action_is_pad, action, optional environment_state\n", @@ -424,36 +317,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "fe4a496c", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_tokens.extend(list(cam_features))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_tokens.extend(list(cam_features))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_pos_embed.extend(list(cam_pos_embed))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_pos_embed.extend(list(cam_pos_embed))\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ONNX model exported to openvino_ir_outputs/model.onnx\n" - ] - } - ], + "outputs": [], "source": [ "# Export ACT Model to ONNX\n", "import torch, inspect, os, pathlib\n", @@ -528,63 +395,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "540875e8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running Model Optimizer: mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --model_name act_model_fp32 --compress_to_fp16=False\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n", - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n", - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n", - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release.\n", - "In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. \n", - "Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2023-1&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino\n", - "[ SUCCESS ] Generated IR version 11 model.\n", - "[ SUCCESS ] XML file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/act_model_fp32.xml\n", - "[ SUCCESS ] BIN file: /home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/openvino_ir_outputs/act_model_fp32.bin\n", - "MO conversion complete. IR files: openvino_ir_outputs/act_model_fp32.xml openvino_ir_outputs/act_model_fp32.bin\n", - "The following IR files present:\n", - "- act_model_fp32.xml exists | size: 2725658\n", - "- act_model_fp32.bin exists | size: 134347484\n" - ] - } - ], + "outputs": [], "source": [ "# Convert ONNX to OpenVINO IR\n", "import subprocess, shlex, os, pathlib\n", @@ -640,54 +454,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "326ad8df", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[DIRECT] Converting via ov.convert_model (FX)...\n", - "WARNING:nncf:NNCF provides best results with torch==2.8.*, while current torch version is 2.7.1+cu126. If you encounter issues, consider switching to torch==2.8.*\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:478: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_pos_embed = list(self.encoder_1d_feature_pos_embed.weight.unsqueeze(1))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_tokens.extend(list(cam_features))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:503: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_tokens.extend(list(cam_features))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", - " encoder_in_pos_embed.extend(list(cam_pos_embed))\n", - "/home/case/Mohammad/notebook/openvino_notebooks/notebooks/lerobot_act/unitree_IL_lerobot/unitree_lerobot/lerobot/src/lerobot/policies/act/modeling_act.py:504: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results.\n", - " encoder_in_pos_embed.extend(list(cam_pos_embed))\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[DIRECT] Final IR input ports (friendly_name / tensor names / partial shape):\n", - " - observation_state | tensor_names=['observation_state'] | partial_shape=[?,?]\n", - " - observation_images_0 | tensor_names=['observation_images_0'] | partial_shape=[?,3,?,?]\n", - " - observation_images_1 | tensor_names=['observation_images_1'] | partial_shape=[?,3,?,?]\n", - " - observation_images_2 | tensor_names=['observation_images_2'] | partial_shape=[?,3,?,?]\n", - " - observation_images_3 | tensor_names=['observation_images_3'] | partial_shape=[?,3,?,?]\n", - "[DIRECT] Saved FP32 XML: openvino_ir_outputs/act_model_direct_fp32.xml | size: 4011117\n", - "[DIRECT] Saved FP32 BIN : openvino_ir_outputs/act_model_direct_fp32.bin | size: 68503430\n", - "\n", - "[HINT] In evaluation build input dict using: observation_state, observation_images_0..N, (optional) observation_environment_state.\n", - "[DONE] Direct conversion complete. (See comments above for FP16 save).\n" - ] - } - ], + "outputs": [], "source": [ "# Direct PyTorch to OpenVINO IR (FP32 by default, FP16 instructions included)\n", "import torch, pathlib, openvino as ov\n", @@ -845,91 +615,7 @@ "execution_count": null, "id": "e173d050", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[INT8] Resolved paths:\n", - " IR_OUTPUT_DIR = openvino_ir_outputs\n", - " FP32_XML = openvino_ir_outputs/act_model_direct_fp32.xml\n", - " STATS_JSON = act_checkpoint/stats.json\n", - " DATASET_ROOT = dataset/G1_BlockStacking_Dataset\n", - " SCRIPT_PATH = quantize_int8_helper.py\n", - " OUT_INT8_DIR = openvino_ir_outputs/int8\n", - " CALIB_SAMPLES = 300\n", - " PRESET = performance\n", - "\n", - "[INT8] Starting quantization at 2025-11-07T21:16:17.988208Z\n", - "[INT8] Using FP32 model: act_model_direct_fp32.xml\n", - "[INT8] Stats file : stats.json\n", - "[INT8] Dataset root : dataset/G1_BlockStacking_Dataset\n", - "[INT8] Output directory : openvino_ir_outputs/int8\n", - "[INT8] Calibration samples=300 preset=performance\n", - "[INT8] Running helper script with args:\n", - " quantize_int8_helper.py --model_xml openvino_ir_outputs/act_model_direct_fp32.xml --stats_path act_checkpoint/stats.json --dataset_root dataset/G1_BlockStacking_Dataset --output_dir openvino_ir_outputs/int8 --num_calib_samples 300 --preset performance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n", - "WARNING:root:\n", - "The dataset you requested (None) is in 2.0 format.\n", - "While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global\n", - "stats instead of per-episode stats. Update your dataset stats to the new format using this command:\n", - "```\n", - "python -m lerobot.datasets.v21.convert_dataset_v20_to_v21 --repo-id=None\n", - "```\n", - "\n", - "If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)\n", - "or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).\n", - "\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bf5ca5d1fc4f48f78fabad537fde9ece", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Resolving data files: 0%| | 0/301 [00:00 openvino_ir_outputs/int8/model_int8.xml (device=CPU, precision=INT8)\n", - "[EVAL] CMD: /home/case/miniforge3/envs/unitree_lerobot/bin/python eval_openvino_model_helper.py --repo_id=None --root=dataset/G1_BlockStacking_Dataset --policy.path=act_checkpoint --episodes=0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/case/miniforge3/envs/unitree_lerobot/lib/python3.10/site-packages/openvino/runtime/__init__.py:10: DeprecationWarning: The `openvino.runtime` module is deprecated and will be removed in the 2026.0 release. Please replace `openvino.runtime` with `openvino`.\n", - " warnings.warn(\n", - "INFO 2025-11-07 11:25:52 l_helper.py:376 {'arm': 'G1_29',\n", - " 'ee': 'dex3',\n", - " 'episodes': 0,\n", - " 'frequency': 30.0,\n", - " 'headless': False,\n", - " 'motion': False,\n", - " 'policy': {'chunk_size': 100,\n", - " 'device': 'cuda',\n", - " 'dim_feedforward': 3200,\n", - " 'dim_model': 512,\n", - " 'dropout': 0.1,\n", - " 'feedforward_activation': 'relu',\n", - " 'input_features': {'observation.images.cam_left_high': {'shape': (3,\n", - " 480,\n", - " 640),\n", - " 'type': },\n", - " 'observation.images.cam_left_wrist': {'shape': (3,\n", - " 480,\n", - " 640),\n", - " 'type': },\n", - " 'observation.images.cam_right_high': {'shape': (3,\n", - " 480,\n", - " 640),\n", - " 'type': },\n", - " 'observation.images.cam_right_wrist': {'shape': (3,\n", - " 480,\n", - " 640),\n", - " 'type': },\n", - " 'observation.state': {'shape': (28,),\n", - " 'type': }},\n", - " 'kl_weight': 10.0,\n", - " 'latent_dim': 32,\n", - " 'license': None,\n", - " 'n_action_steps': 100,\n", - " 'n_decoder_layers': 1,\n", - " 'n_encoder_layers': 4,\n", - " 'n_heads': 8,\n", - " 'n_obs_steps': 1,\n", - " 'n_vae_encoder_layers': 4,\n", - " 'normalization_mapping': {'ACTION': ,\n", - " 'STATE': ,\n", - " 'VISUAL': },\n", - " 'optimizer_lr': 1e-05,\n", - " 'optimizer_lr_backbone': 1e-05,\n", - " 'optimizer_weight_decay': 0.0001,\n", - " 'output_features': {'action': {'shape': (28,),\n", - " 'type': }},\n", - " 'pre_norm': False,\n", - " 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1',\n", - " 'private': None,\n", - " 'push_to_hub': False,\n", - " 'replace_final_stride_with_dilation': 0,\n", - " 'repo_id': None,\n", - " 'tags': None,\n", - " 'temporal_ensemble_coeff': None,\n", - " 'use_amp': False,\n", - " 'use_vae': True,\n", - " 'vision_backbone': 'resnet18'},\n", - " 'repo_id': 'None',\n", - " 'root': 'dataset/G1_BlockStacking_Dataset',\n", - " 'send_real_robot': False,\n", - " 'use_dataset': False,\n", - " 'visualization': False}\n", - "WARNING 2025-11-07 11:25:52 ts/utils.py:302 \n", - "The dataset you requested (None) is in 2.0 format.\n", - "While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global\n", - "stats instead of per-episode stats. Update your dataset stats to the new format using this command:\n", - "```\n", - "python -m lerobot.datasets.v21.convert_dataset_v20_to_v21 --repo-id=None\n", - "```\n", - "\n", - "If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)\n", - "or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).\n", - "\n", - "INFO 2025-11-07 11:25:59 l_helper.py:392 Using OpenVINO model path: openvino_ir_outputs/int8/model_int8.xml; stats path: act_checkpoint/stats.json (env model=True, env stats=True)\n", - "INFO 2025-11-07 11:25:59 l_helper.py:452 Compiling OpenVINO model for device=CPU with INFERENCE_PRECISION_HINT=INT8\n", - "INFO 2025-11-07 11:26:06 l_helper.py:477 OpenVINO model loaded: openvino_ir_outputs/int8/model_int8.xml (device=CPU, precision=INT8, cameras=['cam_left_high', 'cam_left_wrist', 'cam_right_high', 'cam_right_wrist'], state_dim=28, temporal_ensemble=on, chunk_size=100, coeff=0.0100)\n", - " 0%| | 0/1173 [00:00 actions_comparison_int8.png\n", - "\n", - "[EVAL] Summary:\n", - " - actions_comparison_int8.png\n", - "[EVAL] Done.\n" - ] - } - ], + "outputs": [], "source": [ "# Evaluation & Comparison Plotting (ENV VAR path & device + precision hints)\n", "\"\"\"\n", @@ -1244,14 +801,6 @@ " print('[EVAL] No figures generated.')\n", "print('[EVAL] Done.')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e8229fcb-4af9-49bf-801a-4deb4a65a572", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 765a7b178441e4133ce96ea4b6af2bff5f0513ec Mon Sep 17 00:00:00 2001 From: Mohammad Haghighipanah Date: Wed, 12 Nov 2025 11:37:25 -0800 Subject: [PATCH 4/5] Renamed act_to_openvino.ipynb to lerobot-act.ipynb --- notebooks/lerobot_act/README.md | 6 +++--- .../{act_to_openvino.ipynb => lerobot-act.ipynb} | 0 notebooks/lerobot_act/setup_unitree_lerobot_env.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename notebooks/lerobot_act/{act_to_openvino.ipynb => lerobot-act.ipynb} (100%) diff --git a/notebooks/lerobot_act/README.md b/notebooks/lerobot_act/README.md index 1cefb44e28e..f8622866b85 100644 --- a/notebooks/lerobot_act/README.md +++ b/notebooks/lerobot_act/README.md @@ -1,5 +1,5 @@ # ACT Policy → OpenVINO IR Conversion (Notebook Guide) -This README documents the current workflow implemented in `act_to_openvino.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. The notebook presently performs FP32 export (Model Optimizer invoked with FP16 compression flag but output standardized to `act_model_fp32.xml/bin`). +This README documents the current workflow implemented in `lerobot-act.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. The notebook presently performs FP32 export (Model Optimizer invoked with FP16 compression flag but output standardized to `act_model_fp32.xml/bin`). ## Required Checkpoint Files (`act_checkpoint/`) @@ -21,7 +21,7 @@ bash setup_unitree_lerobot_env.sh conda create -n unitree_lerobot python=3.10 -y conda activate unitree_lerobot # Launch notebook with correct kernel -jupyter lab act_to_openvino.ipynb --NotebookApp.kernel_name=unitree_lerobot +jupyter lerobot-act.ipynb --NotebookApp.kernel_name=unitree_lerobot ``` If you skip creating a dedicated environment, the dependency cell will install core packages (torch, openvino, nncf, etc.) into the current kernel. You MUST still install `lerobot` manually; the notebook will not auto‑install it. @@ -100,7 +100,7 @@ Evaluation pipeline steps: ## Directory Layout (Example After Conversion, FP16 & INT8 Quantization) ``` -act_to_openvino.ipynb +lerobot-act.ipynb act_checkpoint/ model.safetensors config.json diff --git a/notebooks/lerobot_act/act_to_openvino.ipynb b/notebooks/lerobot_act/lerobot-act.ipynb similarity index 100% rename from notebooks/lerobot_act/act_to_openvino.ipynb rename to notebooks/lerobot_act/lerobot-act.ipynb diff --git a/notebooks/lerobot_act/setup_unitree_lerobot_env.sh b/notebooks/lerobot_act/setup_unitree_lerobot_env.sh index 5b07088fc1c..4da933554d2 100644 --- a/notebooks/lerobot_act/setup_unitree_lerobot_env.sh +++ b/notebooks/lerobot_act/setup_unitree_lerobot_env.sh @@ -149,7 +149,7 @@ main() { sanity_check log "Setup complete. Launch notebook with:" echo " conda activate ${ENV_NAME}" - echo " jupyter lab notebooks/lerobot_act/act_to_openvino.ipynb --NotebookApp.kernel_name=${ENV_NAME}" + echo " jupyter lab notebooks/lerobot_act/lerobot-act.ipynb --NotebookApp.kernel_name=${ENV_NAME}" } main From a6a605fea523096ea0fca23c9c10cca018fa78e0 Mon Sep 17 00:00:00 2001 From: Mohammad Haghighipanah Date: Tue, 25 Nov 2025 11:35:02 -0800 Subject: [PATCH 5/5] added installation cell and removed setup bash script, deleted onnx and two step conversion --- notebooks/lerobot_act/README.md | 37 +- notebooks/lerobot_act/lerobot-act.ipynb | 571 +++++++++--------- .../lerobot_act/setup_unitree_lerobot_env.sh | 155 ----- 3 files changed, 296 insertions(+), 467 deletions(-) delete mode 100644 notebooks/lerobot_act/setup_unitree_lerobot_env.sh diff --git a/notebooks/lerobot_act/README.md b/notebooks/lerobot_act/README.md index f8622866b85..a6661c0dd80 100644 --- a/notebooks/lerobot_act/README.md +++ b/notebooks/lerobot_act/README.md @@ -1,6 +1,8 @@ # ACT Policy → OpenVINO IR Conversion (Notebook Guide) -This README documents the current workflow implemented in `lerobot-act.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. The notebook presently performs FP32 export (Model Optimizer invoked with FP16 compression flag but output standardized to `act_model_fp32.xml/bin`). +This README documents the current workflow implemented in `lerobot-act.ipynb` for converting a LeRobot ACT (Action Chunking Transformer) PyTorch checkpoint into an OpenVINO IR (XML/BIN) model. +# Run +`jupyter lab lerobot-act.ipynb` ## Required Checkpoint Files (`act_checkpoint/`) Place these next to the notebook: @@ -14,19 +16,6 @@ Download the G1_BlockStacking_Dataset from hugging face: https://huggingface.co/datasets/unitreerobotics/G1_Dex3_BlockStacking_Dataset -## Minimal Installation & Launch -Recommended (Conda environment): -```bash -bash setup_unitree_lerobot_env.sh -conda create -n unitree_lerobot python=3.10 -y -conda activate unitree_lerobot -# Launch notebook with correct kernel -jupyter lerobot-act.ipynb --NotebookApp.kernel_name=unitree_lerobot -``` - -If you skip creating a dedicated environment, the dependency cell will install core packages (torch, openvino, nncf, etc.) into the current kernel. You MUST still install `lerobot` manually; the notebook will not auto‑install it. - - ## Key Configuration Variables | Variable | Meaning | |-------------------|----------------------------------------------------------| @@ -37,20 +26,6 @@ If you skip creating a dedicated environment, the dependency cell will install c | `PRECISIONS` | Currently `['FP32']` | | `TARGET_DEVICE` | Default runtime device | -## ONNX Export -Wrapper (`ONNXWrapper`) mirrors ACT forward usage by constructing a batch dict. Input ordering: -`observation.state`, each camera image (`observation.images.*`), `action_is_pad`, `action`, optional `observation.environment_state`. -* Output name: `output` -Exports only if `openvino_ir_outputs/model.onnx` does not already exist. - -## Model Optimizer Conversion -Executed command: -``` -mo --input_model openvino_ir_outputs/model.onnx --output_dir openvino_ir_outputs --compress_to_fp16=False -``` -Artifacts are copied / renamed to: -* `act_model_fp32.xml` -* `act_model_fp32.bin` ## Direct PyTorch FX Conversion Instead of exporting full temporal tensors via ONNX you can generate a smaller IR directly from PyTorch using OpenVINO's FX path. The wrapper internally creates placeholder temporal inputs (`action`, `action_is_pad`, history) so the IR exposes only observation features: @@ -79,7 +54,7 @@ Tips: ## Evaluation of Variants -The notebook / helper script can compare PyTorch baseline vs IR variants (Direct FP32, FP16, MO FP32, INT8). +The notebook / helper script can compare PyTorch baseline vs IR variants (Direct FP32, FP16, INT8). Environment variables (set before running evaluation cell): | Var | Purpose | @@ -109,9 +84,6 @@ act_checkpoint/ dataset/ G1_BlockStacking_Dataset/ openvino_ir_outputs/ - model.onnx # ONNX baseline export - act_model_fp32.xml # MO baseline IR (full inputs) - act_model_fp32.bin act_model_direct_fp32.xml # Direct minimal-input IR act_model_direct_fp32.bin int8/ @@ -120,7 +92,6 @@ openvino_ir_outputs/ actions_comparison_direct_fp32.png actions_comparison_direct_fp16.png -actions_comparison_mo_fp32.png actions_comparison_int8.png ``` \ No newline at end of file diff --git a/notebooks/lerobot_act/lerobot-act.ipynb b/notebooks/lerobot_act/lerobot-act.ipynb index 364b89caa4d..d13eea7c745 100644 --- a/notebooks/lerobot_act/lerobot-act.ipynb +++ b/notebooks/lerobot_act/lerobot-act.ipynb @@ -18,110 +18,167 @@ "metadata": {}, "source": [ "## Dependency and Core Installation Verification\n", - "Run the next cell to verify all thre required packages are installed." + "Run the next cell to install all the required packages." ] }, { "cell_type": "code", "execution_count": null, - "id": "16781bc1", + "id": "0abdb850", "metadata": {}, "outputs": [], "source": [ - "# Dependency Verification & Core Installation (auto-upgrade openvino runtime)\n", - "\"\"\"\n", - "This cell:\n", - " * Verifies core packages (torch, openvino, nncf + utilities)\n", - " * Auto-installs or upgrades openvino runtime to >=2025.0.0 if current version is older\n", - " * Checks for lerobot and EXITS with instructions if it's not present\n", - "\"\"\"\n", - "import sys, subprocess, importlib, pathlib, os\n", - "from importlib import metadata\n", - "\n", - "TARGET_OV_VERSION = '2025.0.0' # Minimum required runtime version\n", - "SETUP_SCRIPT = pathlib.Path('setup_unitree_lerobot_env.sh')\n", - "README_PATH = pathlib.Path('README.md')\n", - "\n", - "# Core specs excluding openvino (handled separately for upgrade logic)\n", - "CORE_SPECS = [\n", - " 'openvino>=2025.0.0',\n", - " 'nncf>=2.14.0',\n", - " 'torch>=2.1', 'torchvision', 'accelerate',\n", - " 'safetensors', 'numpy', 'pandas', 'matplotlib', 'tqdm', 'h5py',\n", - " 'onnx', 'onnxruntime', 'rich'\n", + "# Simple Python 3.10 Environment Setup (lerobot + ACT dependencies)\n", + "import os, sys, subprocess, shutil, pathlib, textwrap\n", + "\n", + "VENV_DIR = pathlib.Path('.py310_venv')\n", + "PY310 = shutil.which('python3.10') or shutil.which('python3')\n", + "if not PY310:\n", + " raise SystemExit('python3.10 not found')\n", + "\n", + "if not VENV_DIR.exists():\n", + " subprocess.check_call([PY310, '-m', 'venv', str(VENV_DIR)])\n", + "VENV_PY = VENV_DIR / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python')\n", + "\n", + "def vrun(args, msg, check=True, env=None):\n", + " print('[STEP]', msg)\n", + " return subprocess.run([str(VENV_PY)] + args, check=check, env=env)\n", + "\n", + "REPO_URL = 'https://github.com/unitreerobotics/unitree_IL_lerobot.git'\n", + "REPO_DIR = pathlib.Path('unitree_IL_lerobot')\n", + "PARENT_DIR = REPO_DIR / 'unitree_lerobot' # submodule / inner repo\n", + "NESTED_DIR = PARENT_DIR / 'lerobot' # nested python package\n", + "\n", + "# Desired commits\n", + "COMMIT_PARENT = '1960b4693024a4439b1c9325e15131130cc1f60a'\n", + "COMMIT_NESTED = '0878c6880fa4fbadf0742751cf7b015f2d63a769'\n", + "\n", + "REQUIRED_PKGS = [\n", + " 'openvino>=2025.0.0','nncf>=2.14.0',\n", + " 'torch>=2.1','torchvision','accelerate',\n", + " 'safetensors','numpy','pandas','matplotlib','tqdm','h5py',\n", + " 'onnx','onnxruntime','rich',\n", + " 'transformers>=4.45.2','tyro>=0.9.10','datasets==3.3.0','meshcat==0.3.2','logging_mp'\n", "]\n", - "CORE_IMPORTS = {\n", - " 'openvino>=2025.0.0': 'openvino',\n", - " 'nncf>=2.14.0': 'nncf',\n", - " 'torch>=2.1': 'torch',\n", - " 'torchvision': 'torchvision',\n", - " 'accelerate': 'accelerate',\n", - " 'safetensors': 'safetensors',\n", - " 'numpy': 'numpy',\n", - " 'pandas': 'pandas',\n", - " 'matplotlib': 'matplotlib',\n", - " 'tqdm': 'tqdm',\n", - " 'h5py': 'h5py',\n", - " 'onnx': 'onnx',\n", - " 'onnxruntime': 'onnxruntime',\n", - " 'rich': 'rich'\n", - "}\n", - "\n", - "def pip_install(*packages):\n", - " cmd = [sys.executable, '-m', 'pip', 'install', '-U'] + list(packages)\n", - " print('[PIP]', ' '.join(cmd))\n", - " subprocess.check_call(cmd)\n", - "\n", - "# --- Upgrade OpenVINO to target version requied for INT8 quantization ---\n", - "pip_install(f'openvino>={TARGET_OV_VERSION}')\n", - "\n", - "\n", - "# --- Remaining core packages ---\n", - "print('\\n[CHECK] Other core package presence (excluding openvino)...')\n", - "missing = []\n", - "for spec, name in CORE_IMPORTS.items():\n", - " try:\n", - " importlib.import_module(name)\n", - " print(f' [OK] {name}')\n", - " except Exception:\n", - " print(f' [MISSING] {name} (spec: {spec})')\n", - " missing.append(spec)\n", "\n", - "if missing:\n", - " print('\\n[PHASE] Installing missing packages...')\n", - " for spec in missing:\n", - " pip_install(spec)\n", + "# Clone top-level repo if missing\n", + "if not REPO_DIR.exists():\n", + " env = os.environ.copy(); env['GIT_LFS_SKIP_SMUDGE'] = '1'\n", + " subprocess.check_call(['git','clone', REPO_URL, str(REPO_DIR)], env=env)\n", "else:\n", - " print('[INFO] All other core packages already installed.')\n", + " print('[INFO] Top-level repo exists.')\n", + "\n", + "# Init / update submodules (ensure presence)\n", + "subprocess.check_call(['git','-C', str(REPO_DIR),'submodule','update','--init','--recursive'])\n", + "\n", + "# Upgrade tooling + install pkgs\n", + "vrun(['-m','pip','install','-U','pip','setuptools','wheel'], 'Upgrade tooling')\n", + "vrun(['-m','pip','install','-U'] + REQUIRED_PKGS, 'Install required packages')\n", + "\n", + "def is_git_dir(path: pathlib.Path):\n", + " g = path / '.git'\n", + " if g.is_dir():\n", + " return True\n", + " if g.is_file():\n", + " # submodule pointer file\n", + " return True\n", + " return False\n", "\n", - "print('\\n[RECHECK] Core imports after installation:')\n", - "still_missing = []\n", - "for spec, name in CORE_IMPORTS.items():\n", + "def ensure_commit(repo_path: pathlib.Path, commit: str, label: str):\n", + " \"\"\"\n", + " Fetch commit into repo_path (handles submodule pointer .git file).\n", + " \"\"\"\n", + " if not is_git_dir(repo_path):\n", + " print(f'[WARN] {label} path {repo_path} is not a git repo.')\n", + " return False\n", + " # For submodule, .git is a file -> still use -C path\n", + " # First try to see if commit exists\n", + " has = subprocess.run(['git','-C',str(repo_path),'cat-file','-e',f'{commit}^{commit}'], capture_output=True)\n", + " if has.returncode != 0:\n", + " print(f'[INFO] Commit {commit[:8]} not present in {label}. Fetching all...')\n", + " # remove shallow restrictions if any\n", + " subprocess.run(['git','-C',str(repo_path),'fetch','--all','--tags','--prune'], check=True)\n", + " # optional unshallow\n", + " subprocess.run(['git','-C',str(repo_path),'fetch','--depth','1000000'], check=False)\n", + " has2 = subprocess.run(['git','-C',str(repo_path),'cat-file','-e',commit], capture_output=True)\n", + " if has2.returncode != 0:\n", + " print(f'[ERROR] Commit {commit} still not found in {label}.')\n", + " return False\n", + " # Checkout in detached HEAD\n", " try:\n", - " importlib.import_module(name)\n", - " print(f' [OK] {name}')\n", - " except Exception:\n", - " still_missing.append(name)\n", - " print(f' [FAIL] {name} still missing')\n", - "if still_missing:\n", - " print('\\n[WARN] Remaining missing packages:', still_missing)\n", - " print('Consider restarting the kernel or resolving version conflicts.')\n", - "\n", - "# --- lerobot presence ---\n", - "print('\\n[CHECK] lerobot availability...')\n", + " subprocess.check_call(['git','-C',str(repo_path),'checkout',commit])\n", + " current = subprocess.check_output(['git','-C',str(repo_path),'rev-parse','HEAD']).decode().strip()\n", + " if current != commit:\n", + " print(f'[WARN] After checkout {label} HEAD={current[:8]} expected {commit[:8]}')\n", + " return False\n", + " print(f'[OK] {label} at commit {commit[:8]}')\n", + " return True\n", + " except subprocess.CalledProcessError as e:\n", + " print(f'[ERROR] Checkout failed for {label}: {e}')\n", + " return False\n", + "\n", + "# Pin parent (inner repo)\n", + "parent_ok = ensure_commit(PARENT_DIR, COMMIT_PARENT, 'parent')\n", + "\n", + "import os, subprocess\n", + "os.chdir('unitree_IL_lerobot/unitree_lerobot')\n", + "subprocess.check_call(['git','fetch','--all','--tags','--prune'])\n", + "subprocess.check_call(['git','checkout',COMMIT_PARENT])\n", + "print('HEAD:', subprocess.check_output(['git','rev-parse','--short','HEAD']).decode().strip())\n", + "os.chdir('../../')\n", + "\n", + "# Pin nested (if it is itself a git repo)\n", + "nested_ok = ensure_commit(NESTED_DIR, COMMIT_NESTED, 'nested')\n", + "\n", + "print(f'[STATUS] parent pinned={parent_ok} nested pinned={nested_ok}')\n", + "\n", + "# Editable installs\n", + "vrun(['-m','pip','install','-e', str(REPO_DIR)], 'Editable install (top-level)')\n", + "if NESTED_DIR.exists():\n", + " vrun(['-m','pip','install','-e', str(NESTED_DIR)], 'Editable install (nested lerobot)', check=True)\n", + "else:\n", + " print('[ERROR] Missing nested path:', NESTED_DIR)\n", + "\n", + "# Clean conflicting distributions\n", + "try:\n", + " import importlib.metadata as md\n", + " dist = md.distribution('lerobot')\n", + " dist_path = pathlib.Path(dist.locate_file('lerobot')).resolve()\n", + " if dist_path != NESTED_DIR.resolve() and NESTED_DIR.exists():\n", + " print('[CLEANUP] Replacing existing lerobot distribution.')\n", + " vrun(['-m','pip','uninstall','-y','lerobot'], 'Uninstall other lerobot', check=False)\n", + " vrun(['-m','pip','install','-e', str(NESTED_DIR)], 'Reinstall target lerobot', check=True)\n", + "except Exception as e:\n", + " print('[CLEANUP][INFO] Skip distribution check:', e)\n", + "\n", + "# Prepend parent path\n", + "parent_str = str(PARENT_DIR.resolve())\n", + "if parent_str not in sys.path:\n", + " sys.path.insert(0, parent_str)\n", + "\n", + "# Import lerobot to verify\n", + "if 'lerobot' in sys.modules:\n", + " del sys.modules['lerobot']\n", + "import lerobot\n", + "print('[IMPORT] lerobot ->', lerobot.__file__)\n", + "\n", + "# Register kernel\n", + "vrun(['-m','pip','install','ipykernel'], 'Ensure ipykernel', check=True)\n", + "vrun(['-m','ipykernel','install','--user','--name','act-py310','--display-name','ACT Py310'], 'Register kernel', check=False)\n", + "\n", + "print('[DONE] Setup complete.')\n", + "print('[INFO] Requested parent commit :', COMMIT_PARENT)\n", + "print('[INFO] Requested nested commit :', COMMIT_NESTED)\n", + "try:\n", + " parent_head = subprocess.check_output(['git','-C',str(PARENT_DIR),'rev-parse','HEAD']).decode().strip()\n", + " print('[INFO] Actual parent HEAD :', parent_head)\n", + "except Exception as e:\n", + " print('[INFO] Parent HEAD unavailable:', e)\n", "try:\n", - " import lerobot\n", - " print('[OK] lerobot present.')\n", + " nested_head = subprocess.check_output(['git','-C',str(NESTED_DIR),'rev-parse','HEAD']).decode().strip()\n", + " print('[INFO] Actual nested HEAD :', nested_head)\n", "except Exception as e:\n", - " print('[ERROR] lerobot not importable:', e)\n", - " print('\\nACTION REQUIRED:')\n", - " print(f' Run setup script: bash {SETUP_SCRIPT}')\n", - " raise SystemExit(1)\n", - "\n", - "# --- Summary ---\n", - "from importlib import metadata as md\n", - "ov_ver = md.version('openvino') if 'openvino' in sys.modules or metadata.version('openvino') else 'UNKNOWN'\n", - "print(f\"\\n[SUMMARY] Dependencies verified. openvino runtime >= {TARGET_OV_VERSION} (installed: {ov_ver}). Proceed to path configuration cell.\")" + " print('[INFO] Nested HEAD unavailable:', e)" ] }, { @@ -129,7 +186,7 @@ "id": "cf92faf8", "metadata": {}, "source": [ - "Next cell configures all the paths." + "The next cell configures all the paths. Before running it, select the ACT Py310 kernel. From the notebook menu, go to Kernel → Change kernel, then select ACT Py310 and click 'Select'." ] }, { @@ -139,6 +196,8 @@ "metadata": {}, "outputs": [], "source": [ + "import sys; print(sys.executable)\n", + "\n", "# Configuration Parameters (Paths, Precision, Device)\n", "import os, pathlib\n", "\n", @@ -169,7 +228,7 @@ "metadata": {}, "source": [ "## Acquire ACT Checkpoint Assets\n", - "Download the ACT model artifacts: `model.safetensors`, `config.json`, and `train_config.json` into `act_checkpoint/`" + "Before running the next code cell, download the ACT model artifacts: `model.safetensors`, `config.json`, and `train_config.json` into `act_checkpoint/`" ] }, { @@ -195,6 +254,22 @@ " print(' ', k, '=', os.environ[k])\n" ] }, + { + "cell_type": "markdown", + "id": "e0de240a", + "metadata": {}, + "source": [ + "# Load ACT Policy (Overview)\n", + "\n", + "Next code cell does the followings:\n", + "- Verifies both model.safetensors and config.json file exist; aborts with clear errors if missing.\n", + "- Parses config.json and filters keys to ACTConfig’s constructor.\n", + "- Wraps feature definitions into PolicyFeature and normalization mapping into NormalizationMode.\n", + "- Instantiates ACTConfig, builds ACTPolicy, loads weights (strict=False), switches to eval().\n", + "- Extracts action dimension, chunk_size (default 100 if absent), and discovers camera feature keys (observation.images.*).\n", + "- Prints parameter count and detected cameras for later conversion steps." + ] + }, { "cell_type": "code", "execution_count": null, @@ -263,6 +338,24 @@ "print('Detected cameras:', camera_keys)\n" ] }, + { + "cell_type": "markdown", + "id": "9d7bd607", + "metadata": {}, + "source": [ + "## Inspect model and build dummy inputs (for conversion)\n", + "\n", + "The next code cell:\n", + "- Reads input feature dimensions from `policy.config` (state, per-camera images, environment_state).\n", + "- Allocates zero tensors with batch size 1 for:\n", + " - `observation.state` [1, state_dim]\n", + " - each camera image [1, C, H, W] using shapes from the config\n", + " - `action_is_pad` [1, chunk_size] (bool)\n", + " - `action` sequence [1, chunk_size, action_dim]\n", + " - optional `observation.environment_state` [1, env_dim]\n", + "- Prints all shapes to verify setup. These tensors will be used for tracing/export in the next steps." + ] + }, { "cell_type": "code", "execution_count": null, @@ -315,141 +408,22 @@ "print('Ordered input tensor shapes:', [t.shape for t in ordered_inputs])\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "fe4a496c", - "metadata": {}, - "outputs": [], - "source": [ - "# Export ACT Model to ONNX\n", - "import torch, inspect, os, pathlib\n", - "\n", - "# Ensure IR_OUTPUT_DIR is available\n", - "try:\n", - " IR_OUTPUT_DIR\n", - "except NameError:\n", - " IR_OUTPUT_DIR = pathlib.Path(os.getenv('ACT_IR_OUTPUT_DIR', 'openvino_ir_outputs'))\n", - " print('[ONNX] IR_OUTPUT_DIR was undefined; set to', IR_OUTPUT_DIR)\n", - "\n", - "# Create directory if missing\n", - "IR_OUTPUT_DIR.mkdir(exist_ok=True)\n", - "\n", - "# Build ONNXWrapper\n", - "class ONNXWrapper(torch.nn.Module):\n", - " def __init__(self, model, camera_keys):\n", - " super().__init__()\n", - " self.model = model\n", - " self.camera_keys = camera_keys\n", - " def forward(self, observation_state, *cam_inputs_and_rest):\n", - " num_cams = len(self.camera_keys)\n", - " cam_inputs = cam_inputs_and_rest[:num_cams]\n", - " action_is_pad_local = cam_inputs_and_rest[num_cams]\n", - " action_local = cam_inputs_and_rest[num_cams + 1]\n", - " observation_environment_state = None\n", - " if len(cam_inputs_and_rest) > num_cams + 2:\n", - " observation_environment_state = cam_inputs_and_rest[num_cams + 2]\n", - " batch = {'observation.state': observation_state}\n", - " for i, cam_key in enumerate(self.camera_keys):\n", - " batch[cam_key] = cam_inputs[i]\n", - " batch['action_is_pad'] = action_is_pad_local\n", - " batch['action'] = action_local\n", - " batch['observation.images'] = list(cam_inputs)\n", - " if observation_environment_state is not None:\n", - " batch['observation.environment_state'] = observation_environment_state\n", - " prediction = self.model.model(batch)\n", - " if isinstance(prediction, tuple):\n", - " prediction = prediction[0]\n", - " return prediction\n", - "\n", - "onnx_path = IR_OUTPUT_DIR / 'model.onnx'\n", - "if not onnx_path.exists():\n", - " # Construct dummy args & input names\n", - " dummy_args = [torch.randn_like(state)]\n", - " input_names = ['observation_state']\n", - " for i, cam_key in enumerate(camera_keys):\n", - " cam_tensor = torch.randn_like(image_tensors[i])\n", - " dummy_args.append(cam_tensor)\n", - " input_names.append(f'observation_images_{i}')\n", - " dummy_args.append(torch.zeros_like(action_is_pad))\n", - " dummy_args.append(torch.zeros_like(action_seq))\n", - " input_names += ['action_is_pad', 'action']\n", - " if env_state is not None:\n", - " dummy_args.append(torch.randn_like(env_state))\n", - " input_names.append('observation_environment_state')\n", - "\n", - " torch.onnx.export(\n", - " ONNXWrapper(policy, camera_keys),\n", - " tuple(dummy_args),\n", - " str(onnx_path),\n", - " export_params=True,\n", - " opset_version=11,\n", - " do_constant_folding=True,\n", - " input_names=input_names,\n", - " output_names=['output']\n", - " )\n", - " print('ONNX model exported to', onnx_path)\n", - "else:\n", - " print('ONNX already exists:', onnx_path)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "540875e8", - "metadata": {}, - "outputs": [], - "source": [ - "# Convert ONNX to OpenVINO IR\n", - "import subprocess, shlex, os, pathlib\n", - "MO_OUT_DIR = IR_OUTPUT_DIR # assumes IR_OUTPUT_DIR defined earlier\n", - "IR_FP32_XML = MO_OUT_DIR / 'act_model_fp32.xml'\n", - "IR_FP32_BIN = MO_OUT_DIR / 'act_model_fp32.bin'\n", - "\n", - "if IR_FP32_XML.exists() and IR_FP32_BIN.exists():\n", - " print('IR already present, skipping MO conversion:', IR_FP32_XML)\n", - "else:\n", - " cmd = (\n", - " f\"mo --input_model {IR_OUTPUT_DIR / 'model.onnx'} \"\n", - " f\"--output_dir {MO_OUT_DIR} --model_name act_model_fp32 --compress_to_fp16=False\"\n", - " )\n", - " print('Running Model Optimizer:', cmd)\n", - " try:\n", - " subprocess.run(shlex.split(cmd), check=True)\n", - " # Verify artifacts\n", - " if not IR_FP32_XML.exists() or not IR_FP32_BIN.exists():\n", - " raise FileNotFoundError('Expected act_model_fp32.xml/bin not produced. Check MO logs above.')\n", - " print('MO conversion complete. IR files:', IR_FP32_XML, IR_FP32_BIN)\n", - " except Exception as e:\n", - " raise RuntimeError(f'Model Optimizer failed. Ensure openvino (and optionally openvino-dev) installed. Original error: {e}')\n", - "\n", - "print('The following IR files present:')\n", - "for f in [IR_FP32_XML, IR_FP32_BIN]:\n", - " print('-', f.name, 'exists' if f.exists() else 'MISSING', '| size:', f.stat().st_size if f.exists() else 0)" - ] - }, { "cell_type": "markdown", "id": "2dbc80c7", "metadata": {}, "source": [ - "### Optional: Direct PyTorch → OpenVINO IR (No ONNX)\n", - "Convert the loaded ACT policy directly from PyTorch to OpenVINO IR using the FX frontend. This bypasses ONNX export.\n", + "## Direct PyTorch to OpenVINO IR (No ONNX)\n", "\n", - "Precision notes:\n", - "- Default below is FP32 (weights stored as 32-bit floating point).\n", - "- To get FP16 (half-precision weights) via the direct path you can call `ov.save_model(..., compress_to_fp16=True)`.\n", - "\n", - "How to produce FP16 here (two approaches):\n", - "Add after FP32 save:\n", - " ```python\n", - " ov.save_model(ov_model, str(IR_OUTPUT_DIR / 'act_model_direct_fp16.xml'), compress_to_fp16=True)\n", - " ```\n", - " This emits `act_model_direct_fp16.xml/bin`.\n", + "Convert the loaded ACT policy directly from PyTorch to OpenVINO IR using the FX frontend. This bypasses ONNX export.\n", "\n", - "What gets converted when using `compress_to_fp16=True`?\n", - "- Parameter tensors (weights) are stored in FP16.\n", - "- Graph topology and layer semantics stay the same." + "The next cell:\n", + "- Wraps ACTPolicy in a DirectWrapper; internal action_is_pad/action are synthesized (not IR inputs).\n", + "- Converts with ov.convert_model using example inputs; includes env if present.\n", + "- Renames IR inputs: observation_state, observation_images_0..N, optional observation_environment_state.\n", + "- Validates input count and prints port names and partial shapes.\n", + "- Saves FP32 IR (act_model_direct_fp32.xml/bin). \n", + "- To produce FP16, call ov.save_model(..., compress_to_fp16=True).\n" ] }, { @@ -585,9 +559,17 @@ "id": "f107d7b8", "metadata": {}, "source": [ - "### Optional: INT8 Quantization (Post-Training)\n", + "## Optional: INT8 Quantization (Post-Training)\n", "This section generates an INT8 (quantized) OpenVINO model using the helper script `quantize_int8_helper.py` found in this folder.\n", "\n", + "The helper runs with the following arguments:\n", + " - `--model_xml` FP32 IR path\n", + " - `--stats_path` training stats\n", + " - `--dataset_root` calibration dataset\n", + " - `--output_dir` `openvino_ir_outputs/int8`\n", + " - `--num_calib_samples` (default 300)\n", + " - `--preset` (`performance` or `accuracy`)\n", + " \n", "Why INT8?\n", "- Smaller binary size.\n", "- Potential throughput / latency gains (depends on CPU / GPU / VPU).\n", @@ -693,7 +675,9 @@ "id": "729f7aaf", "metadata": {}, "source": [ - "Next cell runs evaluation and comparison for each OpenVINO IR model variant (FP32, MO FP32, INT8) using the helper script. It generates action comparison plots for each variant, comparing OpenVINO outputs to the baseline PyTorch model. Results are saved as PNG figures for further analysis." + "## Evaluation & Comparison Plotting\n", + "\n", + "Next cell runs evaluation and comparison for each OpenVINO IR model variant (FP32, INT8) using the helper script. It generates action comparison plots for each variant, comparing OpenVINO outputs to the baseline PyTorch model. Results are saved as PNG figures for further analysis.\n" ] }, { @@ -703,87 +687,116 @@ "metadata": {}, "outputs": [], "source": [ - "# Evaluation & Comparison Plotting (ENV VAR path & device + precision hints)\n", - "\"\"\"\n", - "Sets OPENVINO_MODEL_PATH, STATS_PATH, OPENVINO_DEVICE, and OPENVINO_PRECISION_HINT per variant.\n", + "# Evaluation & Comparison Plotting\n", + "import sys, os, pathlib, datetime, runpy, shutil, traceback\n", "\n", - "\"\"\"\n", - "import sys, pathlib, subprocess, os, datetime, shutil\n", + "REQUIRED_EVAL_PKGS = [\"openvino\", \"torch\", \"numpy\", \"matplotlib\"]\n", + "for mod in REQUIRED_EVAL_PKGS:\n", + " try:\n", + " __import__(mod)\n", + " except Exception as e:\n", + " print(f\"[EVAL][WARN] Missing module '{mod}' ({e}).\")\n", "\n", - "for sym in ['IR_OUTPUT_DIR', 'CHECKPOINT_PATH', 'STATS_PATH', 'TARGET_DEVICE']:\n", + "needed_syms = [\"IR_OUTPUT_DIR\", \"CHECKPOINT_PATH\", \"STATS_PATH\", \"TARGET_DEVICE\"]\n", + "for sym in needed_syms:\n", " if sym not in globals():\n", - " raise RuntimeError(f'Missing required symbol `{sym}`; rerun earlier cells.')\n", + " raise RuntimeError(f\"[EVAL] Missing `{sym}`; rerun earlier cells.\")\n", "\n", - "EVAL_SCRIPT = pathlib.Path('eval_openvino_model_helper.py')\n", + "EVAL_SCRIPT = pathlib.Path(\"eval_openvino_model_helper.py\")\n", "if not EVAL_SCRIPT.exists():\n", - " raise FileNotFoundError(f'Helper script missing: {EVAL_SCRIPT}')\n", + " raise FileNotFoundError(f\"Helper script missing: {EVAL_SCRIPT}\")\n", "\n", - "# Resolve stats path (fallback to dataset meta)\n", "stats_path = pathlib.Path(STATS_PATH)\n", - "DATASET_ROOT = pathlib.Path('dataset/G1_BlockStacking_Dataset')\n", + "DATASET_ROOT = pathlib.Path(\"dataset/G1_BlockStacking_Dataset\")\n", "if not stats_path.exists():\n", - " fallback = DATASET_ROOT / 'meta' / 'stats.json'\n", + " fallback = DATASET_ROOT / \"meta\" / \"stats.json\"\n", " if fallback.exists():\n", " stats_path = fallback\n", + " print(f\"[EVAL] Using fallback stats path: {stats_path}\")\n", " else:\n", - " raise FileNotFoundError(f'stats.json not found at {STATS_PATH} or {fallback}')\n", + " raise FileNotFoundError(f\"stats.json not found at {STATS_PATH} or {fallback}\")\n", "\n", - "# Gather variants dynamically (include fp16 if present)\n", "MODEL_VARIANTS = [\n", - " ('direct_fp32', IR_OUTPUT_DIR / 'act_model_direct_fp32.xml'),\n", - " ('direct_fp16', IR_OUTPUT_DIR / 'act_model_direct_fp16.xml'),\n", - " ('mo_fp32', IR_OUTPUT_DIR / 'act_model_fp32.xml'),\n", - " ('int8', IR_OUTPUT_DIR / 'int8' / 'model_int8.xml'),\n", + " (\"direct_fp32\", IR_OUTPUT_DIR / \"act_model_direct_fp32.xml\"),\n", + " (\"direct_fp16\", IR_OUTPUT_DIR / \"act_model_direct_fp16.xml\"),\n", + " (\"mo_fp32\", IR_OUTPUT_DIR / \"act_model_fp32.xml\"),\n", + " (\"int8\", IR_OUTPUT_DIR / \"int8\" / \"model_int8.xml\"),\n", "]\n", "MODEL_VARIANTS = [(lbl, p) for lbl, p in MODEL_VARIANTS if p.exists()]\n", "if not MODEL_VARIANTS:\n", - " raise RuntimeError('No model variants found. Convert / quantize first.')\n", + " raise RuntimeError(\"[EVAL] No model variants found.\")\n", "\n", - "print('[EVAL] Variants:', ', '.join(lbl for lbl,_ in MODEL_VARIANTS))\n", - "print('[EVAL] Stats path:', stats_path)\n", - "print('[EVAL] Root path :', DATASET_ROOT)\n", - "print('[EVAL] Policy path:', CHECKPOINT_PATH.parent)\n", - "print('[EVAL] Device :', TARGET_DEVICE)\n", + "print('[EVAL] Variants discovered:', ', '.join(lbl for lbl, _ in MODEL_VARIANTS))\n", + "print('[EVAL] Stats path :', stats_path)\n", + "print('[EVAL] Dataset root :', DATASET_ROOT)\n", + "print('[EVAL] Policy directory :', CHECKPOINT_PATH.parent)\n", + "print('[EVAL] Device :', TARGET_DEVICE)\n", "\n", - "base_env = os.environ.copy()\n", - "base_env['PYTHONWARNINGS'] = 'ignore'\n", - "base_env['OPENVINO_DEVICE'] = TARGET_DEVICE\n", + "ENV_KEYS = [\"OPENVINO_MODEL_PATH\", \"STATS_PATH\", \"OPENVINO_PRECISION_HINT\"]\n", + "original_env = {k: os.environ.get(k) for k in ENV_KEYS}\n", "figures = []\n", "\n", - "def infer_precision(label, path):\n", - " lp = label.lower()\n", - " fname = str(path).lower()\n", - " if 'int8' in lp or 'int8' in fname:\n", - " return 'INT8'\n", - " if 'fp16' in lp or 'fp16' in fname:\n", - " return 'FP16'\n", - " return 'FP32'\n", + "def infer_precision(label: str, path: pathlib.Path) -> str:\n", + " ll = label.lower(); fp = str(path).lower()\n", + " if \"int8\" in ll or \"int8\" in fp: return \"INT8\"\n", + " if \"fp16\" in ll or \"fp16\" in fp: return \"FP16\"\n", + " return \"FP32\"\n", + "\n", + "# Episodes logic: if dataset exists use 1, else 0 (synthetic path expected in helper)\n", + "episodes = 1 if DATASET_ROOT.exists() else 0\n", + "if episodes == 0:\n", + " print(\"[EVAL][INFO] Dataset root missing; running with episodes=0 (synthetic / may limit evaluation).\")\n", "\n", "for label, model_xml in MODEL_VARIANTS:\n", " precision_hint = infer_precision(label, model_xml)\n", - " print(f\"\\n[EVAL] Variant '{label}' -> {model_xml} (device={TARGET_DEVICE}, precision={precision_hint})\")\n", - " run_env = base_env.copy()\n", - " run_env['OPENVINO_MODEL_PATH'] = str(model_xml)\n", - " run_env['STATS_PATH'] = str(stats_path)\n", - " run_env['OPENVINO_PRECISION_HINT'] = precision_hint\n", - " cmd = [\n", - " sys.executable,\n", - " str(EVAL_SCRIPT),\n", + " print(f\"\\n[EVAL] Variant '{label}' -> {model_xml.name} (precision={precision_hint}, device={TARGET_DEVICE})\")\n", + "\n", + " os.environ['OPENVINO_MODEL_PATH'] = str(model_xml)\n", + " os.environ['STATS_PATH'] = str(stats_path)\n", + " os.environ['OPENVINO_PRECISION_HINT'] = precision_hint\n", + "\n", + " # Compatibility shim for legacy lerobot missing PolicyAction\n", + " try:\n", + " from lerobot.processor import PolicyAction # noqa\n", + " except Exception:\n", + " try:\n", + " import lerobot.processor as proc\n", + " class PolicyAction: # stub\n", + " pass\n", + " proc.PolicyAction = PolicyAction\n", + " print(\"[EVAL][SHIM] Injected PolicyAction stub.\")\n", + " except Exception as e:\n", + " print(\"[EVAL][SHIM][FAIL] Could not inject PolicyAction stub:\", e)\n", + "\n", + " argv_backup = sys.argv\n", + " sys.argv = [\n", + " 'eval_openvino_model_helper.py',\n", " '--repo_id=None',\n", " f'--root={DATASET_ROOT}',\n", " f'--policy.path={CHECKPOINT_PATH.parent}',\n", - " '--episodes=0',\n", + " '--policy.device=cpu', # force CPU to avoid CUDA mismatch\n", + " f'--episodes={episodes}',\n", + " '--visualization=False',\n", + " '--use_dataset=False' if episodes == 0 else '--use_dataset=True',\n", " ]\n", - " print('[EVAL] CMD:', ' '.join(cmd))\n", + " print('[EVAL] sys.argv ->', ' '.join(sys.argv))\n", " try:\n", - " subprocess.run(cmd, check=True, env=run_env)\n", - " except subprocess.CalledProcessError as e:\n", - " print(f\"[EVAL][ERROR] {label} failed: {e}\")\n", - " continue\n", - "\n", - " candidates = [pathlib.Path('actions_comparison.png')]\n", - " fig_src = next((c for c in candidates if c.exists()), None)\n", - " if fig_src:\n", + " runpy.run_path(str(EVAL_SCRIPT), run_name='__main__')\n", + " except SystemExit as e:\n", + " print(f\"[EVAL][ERROR] SystemExit({e.code}) for {label}.\")\n", + " except Exception as e:\n", + " print(f\"[EVAL][ERROR] Exception during evaluation of {label}: {e}\")\n", + " traceback.print_exc()\n", + " finally:\n", + " sys.argv = argv_backup\n", + " for k, v in original_env.items():\n", + " if v is None:\n", + " os.environ.pop(k, None)\n", + " else:\n", + " os.environ[k] = v\n", + "\n", + " fig_src = pathlib.Path('actions_comparison.png')\n", + " if fig_src.exists():\n", " timestamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')\n", " fig_dst = pathlib.Path(f'{fig_src.stem}_{label}.png')\n", " if fig_dst.exists():\n", @@ -792,9 +805,9 @@ " figures.append(fig_dst)\n", " print('[EVAL] Saved figure ->', fig_dst)\n", " else:\n", - " print('[EVAL][WARN] No comparison figure produced for', label)\n", + " print('[EVAL][WARN] No figure produced for', label)\n", "\n", - "print('\\n[EVAL] Summary:')\n", + "print('\\n[EVAL] Summary of figures:')\n", "for f in figures:\n", " print(' -', f)\n", "if not figures:\n", @@ -805,9 +818,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "ACT Py310", "language": "python", - "name": "python3" + "name": "act-py310" }, "language_info": { "codemirror_mode": { @@ -819,7 +832,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.19" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/notebooks/lerobot_act/setup_unitree_lerobot_env.sh b/notebooks/lerobot_act/setup_unitree_lerobot_env.sh deleted file mode 100644 index 4da933554d2..00000000000 --- a/notebooks/lerobot_act/setup_unitree_lerobot_env.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -# Automated setup for Conda-based Python 3.10 environment integrating OpenVINO notebooks + Unitree LeRobot fork. -# Usage: -# bash setup_unitree_lerobot_env.sh -set -euo pipefail - -ENV_NAME="unitree_lerobot" -LEROBOT_DIR="unitree_IL_lerobot" -PY_VERSION="3.10" - -log() { printf "\n[setup] %s\n" "$*"; } -err() { printf "\n[error] %s\n" "$*" >&2; } - -check_conda() { - if ! command -v conda >/dev/null 2>&1; then - err "Conda not found. Install Miniconda: https://docs.conda.io/en/latest/miniconda.html" - exit 1 - fi -} - -create_env() { - if conda env list | grep -E "^${ENV_NAME}[[:space:]]" >/dev/null; then - log "Conda env '${ENV_NAME}' already exists (skipping creation)." - else - log "Creating conda env '${ENV_NAME}' (python=${PY_VERSION})" - conda create -y -n "${ENV_NAME}" python="${PY_VERSION}" - fi -} - -activate_env() { - log "Activating env '${ENV_NAME}'" - source "$(conda info --base)/etc/profile.d/conda.sh" - conda activate "${ENV_NAME}" -} - -upgrade_tooling() { - log "Upgrading pip/wheel/setuptools" - python -m pip install --upgrade pip wheel setuptools -} - -install_openvino_requirements() { - # We are already inside the openvino_notebooks tree; - # find the root (parent directories) containing requirements.txt. - if [ -f ../../requirements.txt ]; then - ROOT_REQ=../../requirements.txt - elif [ -f ../requirements.txt ]; then - ROOT_REQ=../requirements.txt - elif [ -f requirements.txt ]; then - ROOT_REQ=requirements.txt - else - err "Could not locate requirements.txt for OpenVINO notebooks. Ensure script resides under openvino_notebooks/." - return 1 - fi - log "Installing OpenVINO notebooks requirements from ${ROOT_REQ}" - python -m pip install -r "${ROOT_REQ}" -} - -register_kernel() { - if jupyter kernelspec list 2>/dev/null | grep -q "${ENV_NAME}"; then - log "Jupyter kernel '${ENV_NAME}' already registered." - else - log "Registering Jupyter kernel '${ENV_NAME}'" - python -m ipykernel install --user --name "${ENV_NAME}" --display-name "Python ${PY_VERSION} (${ENV_NAME})" - fi -} - -clone_lerobot_repo() { - if [ -d "${LEROBOT_DIR}" ]; then - log "LeRobot repo '${LEROBOT_DIR}' already exists (skipping clone)." - else - log "Cloning Unitree LeRobot fork with submodules" - git clone --recurse-submodules https://github.com/unitreerobotics/unitree_IL_lerobot.git "${LEROBOT_DIR}" - fi - pushd "${LEROBOT_DIR}" >/dev/null - log "Updating submodules" - git submodule update --init --recursive - popd >/dev/null -} - -install_pinocchio() { - log "Installing pinocchio via conda-forge (if not installed)" - if python -c "import pinocchio" 2>/dev/null; then - log "pinocchio already installed." - else - conda install -y pinocchio -c conda-forge || err "pinocchio install failed; continue without if unused." - fi -} - -install_lerobot_editable() { - pushd "${LEROBOT_DIR}" >/dev/null - if python -c "import lerobot" 2>/dev/null; then - log "lerobot already importable (skipping editable install)." - else - if [ -d unitree_lerobot/lerobot ]; then - log "Editable install of upstream component" - pushd unitree_lerobot/lerobot >/dev/null - python -m pip install -e . || err "Editable install (lerobot) failed" - popd >/dev/null - fi - log "Editable install of root fork extras" - python -m pip install -e .[dev] || python -m pip install -e . || err "Editable install root failed" - fi - popd >/dev/null -} - -install_extra_packages() { - log "Installing auxiliary packages (if missing)" - python - <<'EOF' -import importlib, subprocess, sys -packages = [ - 'safetensors','onnx','onnxruntime','openvino-dev[pot]','nncf','hydra-core','rich','tqdm','pandas','h5py' -] -for pkg in packages: - base = pkg.split('[')[0].split('==')[0].split('>=')[0] - try: - importlib.import_module(base.replace('-','_')) - print(f"[extra] OK: {pkg}") - except Exception: - print(f"[extra] Installing: {pkg}") - subprocess.check_call([sys.executable,'-m','pip','install',pkg]) -EOF -} - -sanity_check() { - log "Running sanity check" - python - <<'EOF' -import sys, importlib -print('Python:', sys.version) -for pkg in ['torch','openvino','nncf','lerobot']: - try: - importlib.import_module(pkg) - print('OK:', pkg) - except Exception as e: - print('FAIL:', pkg, e) -EOF -} - -main() { - check_conda - create_env - activate_env - upgrade_tooling - install_openvino_requirements - register_kernel - clone_lerobot_repo - install_pinocchio - install_lerobot_editable - install_extra_packages - sanity_check - log "Setup complete. Launch notebook with:" - echo " conda activate ${ENV_NAME}" - echo " jupyter lab notebooks/lerobot_act/lerobot-act.ipynb --NotebookApp.kernel_name=${ENV_NAME}" -} - -main