diff --git a/backends/arm/__init__.py b/backends/arm/__init__.py index 36dec85adba..8c9f5aaa3fc 100644 --- a/backends/arm/__init__.py +++ b/backends/arm/__init__.py @@ -19,10 +19,13 @@ logger = logging.getLogger(__name__) if logger.level == logging.NOTSET: logger.setLevel(min(logging.INFO, logging.root.getEffectiveLevel())) - -# Register Arm-specific torch.library ops and MXFP transforms at package -# import time. -import executorch.backends.arm.ao_ext # noqa: F401 + logger.info( + "Using default logging level %s for %s. Override with " + "logging.getLogger(%r).setLevel(logging.DEBUG).", + logging.getLevelName(logger.level), + __name__, + __name__, + ) # Public for tooling (manifest generation and API validation). LAZY_IMPORTS = { diff --git a/backends/arm/scripts/aot_arm_compiler.py b/backends/arm/scripts/aot_arm_compiler.py index 30f775958d0..165c2c4201d 100644 --- a/backends/arm/scripts/aot_arm_compiler.py +++ b/backends/arm/scripts/aot_arm_compiler.py @@ -6,452 +6,14 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. - -# Example script for exporting simple models to flatbuffer +"""Parse CLI options and configure logging before importing the exporter.""" import argparse -import copy import logging -import os -import sys - -from enum import Enum - -from pathlib import Path - -from typing import Any, List, Optional, Tuple -import torch -from executorch.backends.arm.common.arm_compile_spec import ArmCompileSpec -from executorch.backends.arm.ethosu import EthosUCompileSpec -from executorch.backends.arm.quantizer import ( - get_symmetric_a16w8_quantization_config, - get_symmetric_quantization_config, -) -from executorch.backends.arm.tosa import TosaSpecification -from executorch.backends.arm.tosa.compile_spec import TosaCompileSpec -from executorch.backends.arm.util._factory import create_partitioner, create_quantizer - -from executorch.backends.arm.vgf import VgfCompileSpec -from executorch.backends.cortex_m.edge_compile_config import ( - cortex_m_edge_compile_config, -) -from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager - -from executorch.backends.cortex_m.passes.replace_quant_nodes_pass import ( - ReplaceQuantNodesPass, -) -from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer -from executorch.backends.cortex_m.target_config import CortexMTargetConfig -from executorch.devtools import BundledProgram, generate_etrecord -from executorch.devtools.backend_debug import get_delegation_info -from executorch.devtools.bundled_program.config import MethodTestCase, MethodTestSuite -from executorch.devtools.bundled_program.serialize import ( - serialize_from_bundled_program_to_flatbuffer, -) from executorch.examples.models import MODEL_NAME_TO_MODEL -from executorch.examples.models.model_factory import EagerModelFactory - -from executorch.exir import ( - EdgeCompileConfig, - ExecutorchBackendConfig, - to_edge_transform_and_lower, -) - -from executorch.extension.export_util.utils import save_pte_program -from tabulate import tabulate # type: ignore[import-untyped] -from torch.export import ExportedProgram -from torch.fx import GraphModule - -# Quantize model if required using the standard export quantizaion flow. -from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e - -# Maximum number of samples to use for calibration when quantizing. -CALIBRATION_MAX_SAMPLES = 1000 - - -class QuantMode(str, Enum): - INT8 = "INT8" - A16W8 = "A16W8" - - -def _save_bundled_program(executorch_program, method_test_suites, output_path: str): - """Serialize a bundled program to disk.""" - bundled_program = BundledProgram(executorch_program, method_test_suites) - bundled_program_buffer = serialize_from_bundled_program_to_flatbuffer( - bundled_program - ) - - with open(output_path, "wb") as file: - file.write(bundled_program_buffer) - - -def _load_example_inputs(model_input: str | None) -> Any: # nosec B614 - """Load example inputs from a `.pt` file when a path is provided.""" - if model_input is None: - return None - - logging.info(f"Load model input from {model_input}") - - if model_input.endswith(".pt"): - return torch.load( - model_input, weights_only=False - ) # nosec B614 trusted artifacts - - raise RuntimeError( - f"Model input data '{model_input}' is not a valid name. Use --model_input " - ".pt e.g. saved with torch.save()" - ) - - -def _load_internal_model( - model_name: str, example_inputs: Any -) -> Optional[Tuple[torch.nn.Module, Any]]: - """Load a bundled example model from the internal `MODELS` mapping.""" - logging.info( - "Loading internal models is deprecated. Use --model_name .py/.pt " - "or a model from examples/models." - ) - - if model_name not in MODELS: - return None - - logging.info(f"Internal model {model_name}") - - model = MODELS[model_name]() - inputs = ( - example_inputs - if example_inputs is not None - else MODELS[model_name].example_input # type: ignore[attr-defined] - ) - - return model, inputs - - -def _load_registered_model( - model_name: str, example_inputs: Any -) -> Optional[Tuple[torch.nn.Module, Any]]: - """Load a registered example model from `examples.models`.""" - if model_name not in MODEL_NAME_TO_MODEL: - return None - - logging.warning( - "Using a model from examples/models. Not all of these are currently supported." - ) - logging.info( - f"Load {model_name} -> {MODEL_NAME_TO_MODEL[model_name]} from examples/models" - ) - - model, tmp_example_inputs, _, _ = EagerModelFactory.create_model( - *MODEL_NAME_TO_MODEL[model_name] - ) - inputs = example_inputs if example_inputs is not None else tmp_example_inputs - - return model, inputs - - -def _load_python_module_model( - model_name: str, example_inputs: Any -) -> Optional[Tuple[torch.nn.Module, Any]]: - """Load a model and inputs from a Python source file. - - The file must define `ModelUnderTest` and `ModelInputs` attributes. - - """ - if not model_name.endswith(".py"): - return None - - logging.info( - f"Load model file {model_name} " - "Variable ModelUnderTest= ModelInputs=" - ) - - import importlib.util - - spec = importlib.util.spec_from_file_location("tmp_model", model_name) - if spec is None or spec.loader is None: - raise RuntimeError(f"Unable to load model file {model_name}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules["tmp_model"] = module - model = module.ModelUnderTest - inputs = example_inputs if example_inputs is not None else module.ModelInputs - - return model, inputs - - -def _load_serialized_model( - model_name: str, example_inputs: Any -) -> Optional[Tuple[torch.nn.Module, Any]]: # nosec B614 - """Load a serialized Torch model saved via `torch.save`.""" - if not model_name.endswith((".pth", ".pt")): - return None - - logging.info(f"Load model file {model_name}") - - model = torch.load(model_name, weights_only=False) # nosec B614 trusted inputs - if example_inputs is None: - raise RuntimeError( - f"Model '{model_name}' requires input data specify --model_input .pt" - ) - - return model, example_inputs - - -def _apply_replace_quant_nodes(edge, target: str, direct_drive: bool): - """Apply the replace_quant_nodes pass to the edge graph module.""" - - if target != "vgf" and not direct_drive: - edge = edge.transform([ReplaceQuantNodesPass()]) - return edge - - -def get_model_and_inputs_from_name( - model_name: str, model_input: str | None -) -> Tuple[torch.nn.Module, Any]: - """Resolve a model name into a model instance and example inputs. - - Args: - model_name: Identifier for the model. It can be a key in - `MODEL_NAME_TO_MODEL`, a Python module path, or a serialized - model file path. - model_input: Optional path to a `.pt` file containing example inputs. - - Returns: - Tuple of `(model, example_inputs)` ready for compilation. - - Raises: - RuntimeError: If the model cannot be resolved or required inputs are - missing. - - """ - example_inputs = _load_example_inputs(model_input) - - loaders = ( - _load_internal_model, - _load_registered_model, - _load_python_module_model, - _load_serialized_model, - ) - - for loader in loaders: - result = loader(model_name, example_inputs) - if result is not None: - model, example_inputs = result - logging.debug(f"Loaded model: {model}") - logging.debug(f"Loaded input: {example_inputs}") - return model, example_inputs - - raise RuntimeError( - f"Model '{model_name}' is not a valid name. Use --help for a list of available models." - ) - - -def as_input_tuple(sample: object) -> Tuple[torch.Tensor, ...]: - if isinstance(sample, tuple): - return sample - if isinstance(sample, list): - return tuple(sample) - if isinstance(sample, torch.Tensor): - return (sample,) - if isinstance(sample, dict): - if "pixel_values" in sample: - return (sample["pixel_values"],) - raise ValueError("Calibration sample dict must contain 'pixel_values' key.") - raise ValueError( - "Calibration sample must be a Tensor, tuple, list, or dict with " - "'pixel_values'." - ) - - -def load_calibration_sample( - path: str, example_inputs: Tuple[torch.Tensor, ...] -) -> Tuple[torch.Tensor, ...]: - suffix = Path(path).suffix.lower() - if suffix in {".pt", ".pth"}: - sample = torch.load(path, weights_only=False) # nosec B614 trusted inputs - return as_input_tuple(sample) - raise ValueError(f"Unsupported calibration file type: {path}") - - -def load_calibration_samples( - calibration_data: str | None, - example_inputs: Tuple[torch.Tensor, ...], -) -> Optional[List[Tuple[torch.Tensor, ...]]]: - if calibration_data is None: - return None - - path = Path(calibration_data) - if path.is_file(): - return [load_calibration_sample(str(path), example_inputs)] - - if not path.is_dir(): - raise ValueError( - f"Calibration data path '{calibration_data}' is not a file or directory." - ) - - supported_suffixes = {".pt", ".pth"} - candidates = sorted( - str(p) - for p in path.rglob("*") - if p.is_file() and p.suffix.lower() in supported_suffixes - ) - if not candidates: - raise ValueError( - f"No supported calibration files found in directory '{calibration_data}'." - ) - - samples: List[Tuple[torch.Tensor, ...]] = [] - for candidate in candidates[:CALIBRATION_MAX_SAMPLES]: - samples.append(load_calibration_sample(candidate, example_inputs)) - - return samples - - -def _validate_calibration_sample( - calibration_sample: Tuple[torch.Tensor, ...], - example_inputs: Tuple[torch.Tensor, ...], -) -> None: - expected_len = len(example_inputs) - - if len(calibration_sample) != expected_len: - raise ValueError( - "Calibration sample has %d inputs, expected %d." - % (len(calibration_sample), expected_len) - ) - for input_idx, (expected, actual) in enumerate( - zip(example_inputs, calibration_sample) - ): - if isinstance(expected, torch.Tensor) and isinstance(actual, torch.Tensor): - if expected.shape != actual.shape: - raise ValueError( - "Calibration sample input %d shape %s does not match " - "expected shape %s." - % (input_idx, list(actual.shape), list(expected.shape)) - ) - elif type(expected) is not type(actual): - raise ValueError( - "Calibration sample input %d type %s does not match " - "expected type %s." - % (input_idx, type(actual).__name__, type(expected).__name__) - ) - - -def quantize( - model: GraphModule, - model_name: str, - compile_specs: ArmCompileSpec, - example_inputs: Tuple[torch.Tensor], - quant_mode: QuantMode = QuantMode.INT8, - calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]] = None, -) -> GraphModule: - """This is the official recommended flow for quantization in pytorch 2.0 - export. - """ - logging.info("Quantizing Model...") - logging.debug(f"Original model: {model}") - - quantizer = create_quantizer(compile_specs) - - match quant_mode: - case QuantMode.INT8: - operator_config = get_symmetric_quantization_config(is_per_channel=True) - case QuantMode.A16W8: - if compile_specs.tosa_spec.support_extension("int16"): - operator_config = get_symmetric_a16w8_quantization_config( - is_per_channel=True - ) - else: - raise ValueError( - f"Context TOSA spec {compile_specs.tosa_spec} doesn't support int16" - ) - case _: - raise ValueError(f"Unsupported quantization mode: {quant_mode}") - - quantizer.set_global(operator_config) - m = prepare_pt2e(model, quantizer) - - if calibration_samples is None: - calibration_samples = [example_inputs] - - for sample in calibration_samples: - _validate_calibration_sample(sample, example_inputs) - m(*sample) - - m = convert_pt2e(m) - logging.debug(f"Quantized model: {m}") - return m - - -class QuantAddTest(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward(self, a): - return a + a - - example_input = (torch.rand([13, 3], dtype=torch.float32),) # a - normal values - can_delegate = True # when quantized - - -class QuantAddTest2(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward(self, a, b): - p = a + a - q = b + b - r = p + q - return p, q, r - - example_input = ( - torch.randn([13, 7, 3], dtype=torch.float32), - torch.randn([13, 7, 3], dtype=torch.float32), - ) - can_delegate = True # when quantized - - -class QuantOpTest(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward(self, w, x, y, z): - o1 = w - x - o2 = o1 + y - o3 = o2 * z - return o1, o2, o3 - - example_input = ( - torch.randn([3, 1, 2], dtype=torch.float32), # w - normal values - torch.randn([3, 5, 2], dtype=torch.float32), # x - normal values - torch.randn([3, 5, 1], dtype=torch.float32) - * -0.000001, # y - small -ve values, needs to be calibration for tests - torch.randn([3, 5, 2], dtype=torch.float32) * 1000, # z - large values - ) - can_delegate = True # when quantized - -class QuantLinearTest(torch.nn.Module): - def __init__(self): - super().__init__() - # Define a simple linear layer - self.linear = torch.nn.Linear(61, 37) - - def forward(self, x): - return self.linear(x) - - example_input = (torch.randn([8, 61], dtype=torch.float32),) - can_delegate = True - - -MODELS = { - "qadd": QuantAddTest, - "qadd2": QuantAddTest2, - "qops": QuantOpTest, - # TODO: Remove this from here, once we have dedicated MCU test pipeline ready. This is an interim solution. - # See https://github.com/pytorch/executorch/discussions/13944 - "qlinear": QuantLinearTest, -} +logger = logging.getLogger(__name__) TARGETS = [ "ethos-u55-32", @@ -482,114 +44,35 @@ def forward(self, x): ] -def _get_compile_spec(args) -> ArmCompileSpec: - compile_spec: ArmCompileSpec - - if args.target.startswith("TOSA"): - tosa_spec = TosaSpecification.create_from_string(args.target) - compile_spec = TosaCompileSpec(tosa_spec) - elif "ethos-u" in args.target: - extra_flags = ["--verbose-operators", "--verbose-cycle-estimate"] - if args.enable_debug_mode is not None: - extra_flags.append("--enable-debug-db") - if args.direct_drive: - extra_flags.append("--separate-io-regions") - extra_flags.append("--cop-format=COP2") - max_scratch_size = args.max_scratch_size - if ( - max_scratch_size is None - and args.target.startswith("ethos-u55") - and args.system_config in (None, "Ethos_U55_High_End_Embedded") - and args.memory_mode in (None, "Shared_Sram") - and args.config in (None, "Arm/vela.ini") - ): - max_scratch_size = 2 * 1024 * 1024 - compile_spec = EthosUCompileSpec( - args.target, - system_config=args.system_config, - memory_mode=args.memory_mode, - extra_flags=extra_flags, - config_ini=args.config, - max_scratch_size=max_scratch_size, - plan_delegate_scratch=args.plan_delegate_scratch, +def _configure_logging(debug: bool): + logger.setLevel(logging.DEBUG if debug else logging.INFO) + if debug: + logging.basicConfig( + level=logging.DEBUG, + format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s", + force=True, ) - elif "vgf" in args.target: - if args.quantize: - tosa_spec = TosaSpecification.create_from_string("TOSA-1.0+INT") - else: - tosa_spec = TosaSpecification.create_from_string("TOSA-1.0+FP") - compile_spec = VgfCompileSpec(tosa_spec) + logging.getLogger("executorch.backends.arm").setLevel(logging.DEBUG) + logging.getLogger("executorch.backends.cortex_m").setLevel(logging.DEBUG) else: - raise RuntimeError(f"Unkown target {args.target}") - - if args.intermediates is not None: - compile_spec.dump_intermediate_artifacts_to(args.intermediates) - - if args.enable_debug_mode is not None: - mode = ArmCompileSpec.DebugMode[args.enable_debug_mode.upper()] - compile_spec.dump_debug_info(mode) - - return compile_spec - - -def dump_delegation_info(edge, intermediate_files_folder: Optional[str] = None): - graph_module = edge.exported_program().graph_module - delegation_info = get_delegation_info(graph_module) - df = delegation_info.get_operator_delegation_dataframe() - table = tabulate(df, headers="keys", tablefmt="fancy_grid") - delegation_info_string = f"Delegation info:\n{delegation_info.get_summary()}\nDelegation table:\n{table}\n" - logging.info(delegation_info_string) - if intermediate_files_folder is not None: - delegation_file_path = os.path.join( - intermediate_files_folder, "delegation_info.txt" + logging.basicConfig( + level=logging.WARNING, format="%(levelname)s :%(message)s", force=True + ) + logging.getLogger("executorch.backends.arm").setLevel(logging.INFO) + logging.getLogger("executorch.backends.cortex_m").setLevel(logging.INFO) + logger.info( + "Tip - For more verbose logs, use --debug. Use -i to save artifacts to a given directory." ) - with open(delegation_file_path, "w") as file: - file.write(delegation_info_string) - print_delegation_summary(delegation_info, intermediate_files_folder) - - -def print_delegation_summary( - delegation_info, - intermediate_files_folder: Optional[str] = None, -) -> None: - non_delegated_ops = sorted( - ( - (breakdown.op_type, breakdown.non_delegated) - for breakdown in delegation_info.delegation_by_operator.values() - if breakdown.non_delegated > 0 - ), - key=lambda item: (-item[1], item[0]), - ) - - summary_lines = ["Delegation summary:"] - if delegation_info.num_delegated_nodes == 0: - summary_lines.append(" Model was not delegated.") - elif delegation_info.num_non_delegated_nodes == 0: - summary_lines.append(" Model was fully delegated.") - else: - summary_lines.append(" Model was partially delegated.") - summary_lines.append( - f" Delegated partitions for silicon acceleration: {delegation_info.num_delegated_subgraphs}" + logging.getLogger("executorch.exir.backend.backend_api").setLevel(logging.INFO) + logging.getLogger("executorch.backends.cortex_m.quantizer_reporter").setLevel( + logging.DEBUG if debug else logging.INFO ) - summary_lines.append( - f" Non-delegated ops: {delegation_info.num_non_delegated_nodes}" - ) - - if non_delegated_ops: - summary_lines.append(" Non-delegated operators:") - for op_type, count in non_delegated_ops: - summary_lines.append(f" - {op_type}: {count}") - if intermediate_files_folder is not None: - delegation_file_path = os.path.join( - intermediate_files_folder, "delegation_info.txt" - ) - summary_lines.append("") - summary_lines.append("Full delegation report:") - summary_lines.append(f" {delegation_file_path}") - - print("\n".join(summary_lines)) + logging.getLogger( + "executorch.backends.arm.scripts.aot_arm_compiler_backend" + ).setLevel(logging.DEBUG if debug else logging.INFO) + logging.getLogger("torch.utils._pytree").setLevel(logging.ERROR) def _get_args(): @@ -747,457 +230,20 @@ def _get_args(): if args.plan_delegate_scratch and "ethos-u" not in args.target: parser.error("--plan-delegate-scratch requires an Ethos-U target") - LOGGING_FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" - logging_level = logging.DEBUG if args.debug else logging.WARNING - logging.basicConfig(level=logging_level, format=LOGGING_FORMAT, force=True) - if args.calibration_data is not None and not args.quantize: - raise RuntimeError("--calibration_data requires --quantize to be enabled.") - - # if we have custom ops, register them before processing the model - if args.so_library is not None: - logging.info(f"Loading custom ops from {args.so_library}") - torch.ops.load_library(args.so_library) - - if ( - args.model_name in MODELS.keys() - and args.delegate is True - and MODELS[args.model_name].can_delegate is False - ): - raise RuntimeError(f"Model {args.model_name} cannot be delegated.") + parser.error("--calibration_data requires --quantize to be enabled.") return args -def _save_bpte_program( - exec_prog, - original_model: torch.nn.Module, - output_name: str, - example_inputs: Tuple[torch.Tensor, ...], - args, -): - # Construct MethodTestSuite for Each Method - - # Generate Test Suites - method_names = [ - method.name for method in exec_prog.executorch_program.execution_plan - ] - - program_inputs = {m_name: [example_inputs] for m_name in method_names} - - method_test_suites: List[MethodTestSuite] = [] - for m_name in method_names: - method_inputs = program_inputs[m_name] - - # To create a bundled program, we first create every test cases from input. We leverage eager model - # to generate expected output for each test input, and use MethodTestCase to hold the information of - # each test case. We gather all MethodTestCase for same method into one MethodTestSuite, and generate - # bundled program by all MethodTestSuites. - method_test_cases: List[MethodTestCase] = [] - - if args.intermediates: - # Save model.pth - intermediates_path = Path(args.intermediates) - model_path = os.path.join(intermediates_path, "model.pth") - try: - torch.save(original_model, model_path) - except: - logging.warning(f"Could not torch.save(model, {model_path})") - - method_index = 0 - for method_input in method_inputs: - output_ref = original_model(*method_input) - - logging.debug(f"input_{method_index}: {method_input}") - logging.debug(f"output_ref_{method_index}: {output_ref}") - - if args.intermediates: - # Save model input and referece output - input_path = os.path.join( - intermediates_path, f"input_{method_index}.pt" # type: ignore[possibly-undefined] - ) - try: - torch.save(method_input, input_path) - except: - logging.warning( - f"Could not torch.save(input_{method_index}, {input_path})" - ) - refoutput_path = os.path.join( - intermediates_path, f"output_ref_{method_index}.pt" - ) - try: - torch.save(output_ref, refoutput_path) - except: - logging.warning( - f"Could not torch.save(output_ref_{method_index}, {refoutput_path})" - ) - - method_test_cases.append( - MethodTestCase( - inputs=method_input, - expected_outputs=output_ref, - ) - ) - - method_index = method_index + 1 - - method_test_suites.append( - MethodTestSuite( - method_name=m_name, - test_cases=method_test_cases, - ) - ) - - # Generate BundledProgram - output_dir = os.path.dirname(output_name) - if output_dir: - os.makedirs(output_dir, exist_ok=True) - _save_bundled_program(exec_prog, method_test_suites, output_name) - - -def quantize_model( - model: GraphModule, - example_inputs: Tuple[torch.Tensor], - compile_spec, - model_name: str, - strict_export: bool, - quant_mode: QuantMode, - calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], -) -> Tuple[GraphModule, ExportedProgram]: - model_quant = quantize( - model, - model_name, - compile_spec, - example_inputs, - quant_mode, - calibration_samples, - ) - # Wrap quantized model back into an exported_program - exported_program = torch.export.export( - model_quant, example_inputs, strict=strict_export - ) - - return model_quant, exported_program - - -def _to_edge_TOSA_delegate( - target: str, - exported_program: ExportedProgram, - compile_spec, - model: GraphModule, - quant_mode: Optional[QuantMode], - example_inputs: Tuple[torch.Tensor], - model_name: str, - strict_export: bool, - calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], - direct_drive: bool, -): - model_quant = None - if quant_mode is not None: - model_quant, exported_program = quantize_model( - model, - example_inputs, - compile_spec, - model_name, - strict_export, - quant_mode, - calibration_samples, - ) - - partitioner = create_partitioner(compile_spec) - - edge = to_edge_transform_and_lower( - exported_program, - partitioner=[partitioner], - compile_config=EdgeCompileConfig( - _check_ir_validity=False, - ), - ) - - # Replace quantized_decomposed::{quantize,dequantize}_per_tensor nodes - # with cortex_m:: equivalents for int8/int16 QDQ ops remaining outside - # the delegated subgraph. - edge = _apply_replace_quant_nodes(edge, target, direct_drive) - - return model_quant, edge - - -def _to_edge_cortex_m( - exported_program: ExportedProgram, - args, - model: GraphModule, - example_inputs: Tuple[torch.Tensor], - calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], - target_config: CortexMTargetConfig, -): - """Cortex-M/CMSIS-NN compilation path with no delegation.""" - logging.info( - f"Using Cortex-M/CMSIS-NN compilation path for cpu={target_config.cpu.name} " - f"backend={target_config.backend.name} " - f"layout={'explicit' if args.cortex_m_explicit_layout else 'dim-order'}" - ) - - if args.cortex_m_explicit_layout and not args.quantize: - raise RuntimeError( - "--cortex-m-explicit-layout requires --quantize; explicit layout " - "does not fall back to portable float spatial operators." - ) - - def _to_channels_last(x): - if isinstance(x, torch.Tensor): - if x.dim() == 4: - # Singleton channels can satisfy both contiguity checks while - # retaining NCHW strides, so always request the target format. - channels_last = x.to(memory_format=torch.channels_last) - if channels_last.stride() != x.stride(): - logging.warning( - "Converting input tensor with shape %s to channels_last", - list(x.shape), - ) - return channels_last - return x - elif isinstance(x, tuple): - return tuple(_to_channels_last(t) for t in x) - return x - - if not args.quantize: - logging.warning( - "Quantization is DISABLED. Cortex-M typically requires quantization." - ) - model_quant = None - else: - if not args.cortex_m_explicit_layout: - model = model.to(memory_format=torch.channels_last) # type: ignore[call-overload] - example_inputs = tuple(_to_channels_last(x) for x in example_inputs) - # Refresh fake-tensor strides after changing the captured module's - # memory format so the legacy quantizer sees channels-last inputs. - model = torch.export.export( - model, example_inputs, strict=args.strict_export - ).module() - - quantizer = CortexMQuantizer(use_explicit_layout=args.cortex_m_explicit_layout) - - prepared = prepare_pt2e(model, quantizer) - - if calibration_samples is None: - calibration_samples = [example_inputs] - - for sample in calibration_samples: - prepared(*sample) - - model_quant = convert_pt2e(prepared) - - exported_program = torch.export.export( - model_quant, example_inputs, strict=args.strict_export - ) - - edge = to_edge_transform_and_lower( - exported_program, - compile_config=cortex_m_edge_compile_config(), - transform_passes=CortexMPassManager( - target_config=target_config, - use_explicit_layout=args.cortex_m_explicit_layout, - ), - ) - - return model_quant, edge, example_inputs - - -def _to_edge_no_delegate( - args, - exported_program: ExportedProgram, - compile_spec, - model: GraphModule, - quant_mode: Optional[QuantMode], - example_inputs: Tuple[torch.Tensor], - model_name: str, - strict_export: bool, - calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], -): - model_quant = None - if quant_mode is not None: - # As we can target multiple output encodings, one must - # be specified. - model, exported_program = quantize_model( - model, - example_inputs, - compile_spec, - model_name, - strict_export, - quant_mode, - calibration_samples, - ) - model_quant = model - - edge = to_edge_transform_and_lower( - exported_program, - compile_config=EdgeCompileConfig( - _check_ir_validity=False, - ), - ) - - # Replace quantized_decomposed::{quantize,dequantize}_per_tensor nodes - # with cortex_m:: equivalents for int8/int16 QDQ ops remaining outside - # the delegated subgraph. - edge = _apply_replace_quant_nodes(edge, args.target, args.direct_drive) - - return model_quant, edge - - -def main() -> None: # noqa: C901 +def main() -> None: + """Parse arguments, configure logging, then import and run the exporter.""" args = _get_args() + _configure_logging(args.debug) - # Pick model from one of the supported lists - original_model, example_inputs = get_model_and_inputs_from_name( - args.model_name, args.model_input - ) - calibration_samples = load_calibration_samples( - args.calibration_data, example_inputs - ) - model = original_model.eval() - model.requires_grad_(False) - - # export under the assumption we quantize, the exported form also works - # in to_edge if we don't quantize - exported_program = torch.export.export( - model, example_inputs, strict=args.strict_export - ) - - model = exported_program.module() - - model_name = os.path.basename(os.path.splitext(args.model_name)[0]) - if args.intermediates: - os.makedirs(args.intermediates, exist_ok=True) - - # We only support Python3.10 and above, so use a later pickle protocol - torch.export.save( - exported_program, - f"{args.intermediates}/{model_name}_exported_program.pt2", - pickle_protocol=5, - ) - - # Quantize if required - model_quant = None - if args.quantize: - quant_mode = QuantMode.A16W8 if "int16" in args.target else QuantMode.INT8 - else: - quant_mode = None - - if args.target.startswith("cortex-m"): - # Cortex-M path: CMSIS-NN portable kernels, no delegation - target_config = CortexMTargetConfig.from_target_string(args.target) - if args.delegate: - logging.warning( - f"--delegate is ignored for target {args.target!r} " - "(this target does not use delegated ops)." - ) - args.delegate = False - model_quant, edge, example_inputs = _to_edge_cortex_m( - exported_program, - args, - model, - example_inputs, - calibration_samples, - target_config, - ) - elif args.delegate: - # As we can target multiple output encodings, one must - # be specified. - model_quant, edge = _to_edge_TOSA_delegate( - args.target, - exported_program, - _get_compile_spec(args), - model, - quant_mode, - example_inputs, - args.model_name, - args.strict_export, - calibration_samples, - args.direct_drive, - ) - else: - model_quant, edge = _to_edge_no_delegate( - args, - exported_program, - _get_compile_spec(args), - model, - quant_mode, - example_inputs, - args.model_name, - args.strict_export, - calibration_samples, - ) - - dump_delegation_info(edge, args.intermediates) - - try: - exec_prog = edge.to_executorch( - config=ExecutorchBackendConfig(extract_delegate_segments=False) - ) - except RuntimeError as e: - if "Missing out variants" in str(e.args[0]): - raise RuntimeError( - e.args[0] - + ".\nThis likely due to an external so library not being loaded. Supply a path to it with the -s flag." - ).with_traceback(e.__traceback__) from None - else: - raise e - - output_name = f"{model_name}" + ( - f"_arm_delegate_{args.target}" - if args.delegate is True - else f"_arm_{args.target}" - ) - - if args.bundleio: - output_file_name = f"{output_name}.bpte" - else: - output_file_name = f"{output_name}.pte" - - if args.output is not None: - if args.output.endswith(".pte") or args.output.endswith(".bpte"): - # --output is a pte or bundle pte filename use it as output name - if args.bundleio and not args.output.endswith(".bpte"): - raise RuntimeError( - f"--bundleio expects a .bpte file ending to --output and not .pte {args.output}" - ) - if not args.bundleio and not args.output.endswith(".pte"): - raise RuntimeError( - f"When not using --bundleio a .bpte file should not be use as --output {args.output}" - ) - output_file_name = args.output - output_dir = os.path.dirname(output_file_name) - if output_dir: - os.makedirs(output_dir, exist_ok=True) - else: - # --output is a folder - os.makedirs(args.output, exist_ok=True) - output_file_name = os.path.join(args.output, output_file_name) - - if args.bundleio: - # Realize the quantization impact on numerics when generating reference output - reference_model = original_model if not model_quant else model_quant - _save_bpte_program( - exec_prog, reference_model, output_file_name, example_inputs, args - ) - print(f"Bundle PTE file saved as {output_file_name}") - else: - save_pte_program(exec_prog, output_file_name) - print(f"PTE file saved as {output_file_name}") + from executorch.backends.arm.scripts.aot_arm_compiler_backend import export - if args.bundleio or args.etrecord: - etrecord_file_name = os.path.splitext(output_file_name)[0] + "_etrecord.bin" - try: - edge_program_manager_copy = copy.deepcopy(edge) - generate_etrecord(etrecord_file_name, edge_program_manager_copy, exec_prog) - print(f"ETRecord saved as {etrecord_file_name}") - except Exception as e: - # Treat ETRecord failures as non-fatal only when generated as a side-effect - # of --bundleio. When --etrecord is explicitly requested, fail loudly. - if args.bundleio and not args.etrecord: - logging.warning(f"ETRecord generation failed (non-fatal): {e}") - else: - raise + export(args) if __name__ == "__main__": diff --git a/backends/arm/scripts/aot_arm_compiler_backend.py b/backends/arm/scripts/aot_arm_compiler_backend.py new file mode 100644 index 00000000000..7f604ec1f8e --- /dev/null +++ b/backends/arm/scripts/aot_arm_compiler_backend.py @@ -0,0 +1,1034 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# Copyright 2023-2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +"""Export implementation loaded after the AoT CLI configures logging.""" + +import argparse +import copy +import logging +import os +import sys + +from enum import Enum + +from pathlib import Path + +from typing import Any, List, Optional, Tuple + +import torch +from executorch.backends.arm.common.arm_compile_spec import ArmCompileSpec +from executorch.backends.arm.ethosu import EthosUCompileSpec +from executorch.backends.arm.quantizer import ( + get_symmetric_a16w8_quantization_config, + get_symmetric_quantization_config, +) +from executorch.backends.arm.tosa import TosaSpecification +from executorch.backends.arm.tosa.compile_spec import TosaCompileSpec +from executorch.backends.arm.util._factory import create_partitioner, create_quantizer + +from executorch.backends.arm.vgf import VgfCompileSpec +from executorch.backends.cortex_m.edge_compile_config import ( + cortex_m_edge_compile_config, +) +from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager + +from executorch.backends.cortex_m.passes.replace_quant_nodes_pass import ( + ReplaceQuantNodesPass, +) +from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer +from executorch.backends.cortex_m.target_config import CortexMTargetConfig +from executorch.devtools import BundledProgram, generate_etrecord +from executorch.devtools.backend_debug import get_delegation_info +from executorch.devtools.bundled_program.config import MethodTestCase, MethodTestSuite +from executorch.devtools.bundled_program.serialize import ( + serialize_from_bundled_program_to_flatbuffer, +) +from executorch.examples.models import MODEL_NAME_TO_MODEL +from executorch.examples.models.model_factory import EagerModelFactory + +from executorch.exir import ( + EdgeCompileConfig, + ExecutorchBackendConfig, + to_edge_transform_and_lower, +) + +from executorch.extension.export_util.utils import save_pte_program +from tabulate import tabulate # type: ignore[import-untyped] +from torch.export import ExportedProgram +from torch.fx import GraphModule + +# Quantize model if required using the standard export quantizaion flow. +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +# Maximum number of samples to use for calibration when quantizing. +CALIBRATION_MAX_SAMPLES = 1000 + +logger = logging.getLogger(__name__) + + +class QuantMode(str, Enum): + INT8 = "INT8" + A16W8 = "A16W8" + + +def _save_bundled_program(executorch_program, method_test_suites, output_path: str): + """Serialize a bundled program to disk.""" + bundled_program = BundledProgram(executorch_program, method_test_suites) + bundled_program_buffer = serialize_from_bundled_program_to_flatbuffer( + bundled_program + ) + + with open(output_path, "wb") as file: + file.write(bundled_program_buffer) + + +def _load_example_inputs(model_input: str | None) -> Any: # nosec B614 + """Load example inputs from a `.pt` file when a path is provided.""" + if model_input is None: + return None + + logger.info(f"Load model input from {model_input}") + + if model_input.endswith(".pt"): + return torch.load( + model_input, weights_only=False + ) # nosec B614 trusted artifacts + + raise RuntimeError( + f"Model input data '{model_input}' is not a valid name. Use --model_input " + ".pt e.g. saved with torch.save()" + ) + + +def _load_internal_model( + model_name: str, example_inputs: Any +) -> Optional[Tuple[torch.nn.Module, Any]]: + """Load a bundled example model from the internal `MODELS` mapping.""" + if model_name not in MODELS: + return None + + logger.info( + "Loading internal models is deprecated. Use --model_name .py/.pt " + "or a model from examples/models." + ) + + logger.info(f"Internal model {model_name}") + + model = MODELS[model_name]() + inputs = ( + example_inputs + if example_inputs is not None + else MODELS[model_name].example_input # type: ignore[attr-defined] + ) + + return model, inputs + + +def _load_registered_model( + model_name: str, example_inputs: Any +) -> Optional[Tuple[torch.nn.Module, Any]]: + """Load a registered example model from `examples.models`.""" + if model_name not in MODEL_NAME_TO_MODEL: + return None + + logger.warning( + "Using a model from examples/models. Not all of these are currently supported." + ) + logger.info( + f"Load {model_name} -> {MODEL_NAME_TO_MODEL[model_name]} from examples/models" + ) + + model, tmp_example_inputs, _, _ = EagerModelFactory.create_model( + *MODEL_NAME_TO_MODEL[model_name] + ) + inputs = example_inputs if example_inputs is not None else tmp_example_inputs + + return model, inputs + + +def _load_python_module_model( + model_name: str, example_inputs: Any +) -> Optional[Tuple[torch.nn.Module, Any]]: + """Load a model and inputs from a Python source file. + + The file must define `ModelUnderTest` and `ModelInputs` attributes. + + """ + if not model_name.endswith(".py"): + return None + + logger.info( + f"Load model file {model_name} " + "Variable ModelUnderTest= ModelInputs=" + ) + + import importlib.util + + spec = importlib.util.spec_from_file_location("tmp_model", model_name) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load model file {model_name}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules["tmp_model"] = module + model = module.ModelUnderTest + inputs = example_inputs if example_inputs is not None else module.ModelInputs + + return model, inputs + + +def _load_serialized_model( + model_name: str, example_inputs: Any +) -> Optional[Tuple[torch.nn.Module, Any]]: # nosec B614 + """Load a serialized Torch model saved via `torch.save`.""" + if not model_name.endswith((".pth", ".pt")): + return None + + logger.info(f"Load model file {model_name}") + + model = torch.load(model_name, weights_only=False) # nosec B614 trusted inputs + if example_inputs is None: + raise RuntimeError( + f"Model '{model_name}' requires input data specify --model_input .pt" + ) + + return model, example_inputs + + +def _apply_replace_quant_nodes(edge, target: str, direct_drive: bool): + """Apply the replace_quant_nodes pass to the edge graph module.""" + + if target != "vgf" and not direct_drive: + edge = edge.transform([ReplaceQuantNodesPass()]) + return edge + + +def get_model_and_inputs_from_name( + model_name: str, model_input: str | None +) -> Tuple[torch.nn.Module, Any]: + """Resolve a model name into a model instance and example inputs. + + Args: + model_name: Identifier for the model. It can be a key in + `MODEL_NAME_TO_MODEL`, a Python module path, or a serialized + model file path. + model_input: Optional path to a `.pt` file containing example inputs. + + Returns: + Tuple of `(model, example_inputs)` ready for compilation. + + Raises: + RuntimeError: If the model cannot be resolved or required inputs are + missing. + + """ + example_inputs = _load_example_inputs(model_input) + + loaders = ( + _load_internal_model, + _load_registered_model, + _load_python_module_model, + _load_serialized_model, + ) + + for loader in loaders: + result = loader(model_name, example_inputs) + if result is not None: + model, example_inputs = result + logger.debug(f"Loaded input: {example_inputs}") + return model, example_inputs + + raise RuntimeError( + f"Model '{model_name}' is not a valid name. Use --help for a list of available models." + ) + + +def as_input_tuple(sample: object) -> Tuple[torch.Tensor, ...]: + if isinstance(sample, tuple): + return sample + if isinstance(sample, list): + return tuple(sample) + if isinstance(sample, torch.Tensor): + return (sample,) + if isinstance(sample, dict): + if "pixel_values" in sample: + return (sample["pixel_values"],) + raise ValueError("Calibration sample dict must contain 'pixel_values' key.") + raise ValueError( + "Calibration sample must be a Tensor, tuple, list, or dict with " + "'pixel_values'." + ) + + +def load_calibration_sample( + path: str, example_inputs: Tuple[torch.Tensor, ...] +) -> Tuple[torch.Tensor, ...]: + suffix = Path(path).suffix.lower() + if suffix in {".pt", ".pth"}: + sample = torch.load(path, weights_only=False) # nosec B614 trusted inputs + return as_input_tuple(sample) + raise ValueError(f"Unsupported calibration file type: {path}") + + +def load_calibration_samples( + calibration_data: str | None, + example_inputs: Tuple[torch.Tensor, ...], +) -> Optional[List[Tuple[torch.Tensor, ...]]]: + if calibration_data is None: + return None + + path = Path(calibration_data) + if path.is_file(): + return [load_calibration_sample(str(path), example_inputs)] + + if not path.is_dir(): + raise ValueError( + f"Calibration data path '{calibration_data}' is not a file or directory." + ) + + supported_suffixes = {".pt", ".pth"} + candidates = sorted( + str(p) + for p in path.rglob("*") + if p.is_file() and p.suffix.lower() in supported_suffixes + ) + if not candidates: + raise ValueError( + f"No supported calibration files found in directory '{calibration_data}'." + ) + + samples: List[Tuple[torch.Tensor, ...]] = [] + for candidate in candidates[:CALIBRATION_MAX_SAMPLES]: + samples.append(load_calibration_sample(candidate, example_inputs)) + + return samples + + +def _validate_calibration_sample( + calibration_sample: Tuple[torch.Tensor, ...], + example_inputs: Tuple[torch.Tensor, ...], +) -> None: + expected_len = len(example_inputs) + + if len(calibration_sample) != expected_len: + raise ValueError( + "Calibration sample has %d inputs, expected %d." + % (len(calibration_sample), expected_len) + ) + for input_idx, (expected, actual) in enumerate( + zip(example_inputs, calibration_sample) + ): + if isinstance(expected, torch.Tensor) and isinstance(actual, torch.Tensor): + if expected.shape != actual.shape: + raise ValueError( + "Calibration sample input %d shape %s does not match " + "expected shape %s." + % (input_idx, list(actual.shape), list(expected.shape)) + ) + elif type(expected) is not type(actual): + raise ValueError( + "Calibration sample input %d type %s does not match " + "expected type %s." + % (input_idx, type(actual).__name__, type(expected).__name__) + ) + + +def quantize( + model: GraphModule, + model_name: str, + compile_specs: ArmCompileSpec, + example_inputs: Tuple[torch.Tensor], + quant_mode: QuantMode = QuantMode.INT8, + calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]] = None, +) -> GraphModule: + """This is the official recommended flow for quantization in pytorch 2.0 + export. + """ + logger.info("Quantizing Model...") + + quantizer = create_quantizer(compile_specs) + + match quant_mode: + case QuantMode.INT8: + operator_config = get_symmetric_quantization_config(is_per_channel=True) + case QuantMode.A16W8: + if compile_specs.tosa_spec.support_extension("int16"): + operator_config = get_symmetric_a16w8_quantization_config( + is_per_channel=True + ) + else: + raise ValueError( + f"Context TOSA spec {compile_specs.tosa_spec} doesn't support int16" + ) + case _: + raise ValueError(f"Unsupported quantization mode: {quant_mode}") + + quantizer.set_global(operator_config) + m = prepare_pt2e(model, quantizer) + + if calibration_samples is None: + calibration_samples = [example_inputs] + + for sample in calibration_samples: + _validate_calibration_sample(sample, example_inputs) + m(*sample) + + m = convert_pt2e(m) + return m + + +class QuantAddTest(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, a): + return a + a + + example_input = (torch.rand([13, 3], dtype=torch.float32),) # a - normal values + can_delegate = True # when quantized + + +class QuantAddTest2(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, a, b): + p = a + a + q = b + b + r = p + q + return p, q, r + + example_input = ( + torch.randn([13, 7, 3], dtype=torch.float32), + torch.randn([13, 7, 3], dtype=torch.float32), + ) + can_delegate = True # when quantized + + +class QuantOpTest(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, w, x, y, z): + o1 = w - x + o2 = o1 + y + o3 = o2 * z + return o1, o2, o3 + + example_input = ( + torch.randn([3, 1, 2], dtype=torch.float32), # w - normal values + torch.randn([3, 5, 2], dtype=torch.float32), # x - normal values + torch.randn([3, 5, 1], dtype=torch.float32) + * -0.000001, # y - small -ve values, needs to be calibration for tests + torch.randn([3, 5, 2], dtype=torch.float32) * 1000, # z - large values + ) + can_delegate = True # when quantized + + +class QuantLinearTest(torch.nn.Module): + def __init__(self): + super().__init__() + # Define a simple linear layer + self.linear = torch.nn.Linear(61, 37) + + def forward(self, x): + return self.linear(x) + + example_input = (torch.randn([8, 61], dtype=torch.float32),) + can_delegate = True + + +MODELS = { + "qadd": QuantAddTest, + "qadd2": QuantAddTest2, + "qops": QuantOpTest, + # TODO: Remove this from here, once we have dedicated MCU test pipeline ready. This is an interim solution. + # See https://github.com/pytorch/executorch/discussions/13944 + "qlinear": QuantLinearTest, +} + + +def _get_compile_spec(args) -> ArmCompileSpec: + compile_spec: ArmCompileSpec + + if args.target.startswith("TOSA"): + tosa_spec = TosaSpecification.create_from_string(args.target) + compile_spec = TosaCompileSpec(tosa_spec) + elif "ethos-u" in args.target: + extra_flags = ["--verbose-operators", "--verbose-cycle-estimate"] + if args.enable_debug_mode is not None: + extra_flags.append("--enable-debug-db") + if args.direct_drive: + extra_flags.append("--separate-io-regions") + extra_flags.append("--cop-format=COP2") + max_scratch_size = args.max_scratch_size + if ( + max_scratch_size is None + and args.target.startswith("ethos-u55") + and args.system_config in (None, "Ethos_U55_High_End_Embedded") + and args.memory_mode in (None, "Shared_Sram") + and args.config in (None, "Arm/vela.ini") + ): + max_scratch_size = 2 * 1024 * 1024 + compile_spec = EthosUCompileSpec( + args.target, + system_config=args.system_config, + memory_mode=args.memory_mode, + extra_flags=extra_flags, + config_ini=args.config, + max_scratch_size=max_scratch_size, + plan_delegate_scratch=args.plan_delegate_scratch, + ) + elif "vgf" in args.target: + if args.quantize: + tosa_spec = TosaSpecification.create_from_string("TOSA-1.0+INT") + else: + tosa_spec = TosaSpecification.create_from_string("TOSA-1.0+FP") + compile_spec = VgfCompileSpec(tosa_spec) + else: + raise RuntimeError(f"Unkown target {args.target}") + + if args.intermediates is not None: + compile_spec.dump_intermediate_artifacts_to(args.intermediates) + + if args.enable_debug_mode is not None: + mode = ArmCompileSpec.DebugMode[args.enable_debug_mode.upper()] + compile_spec.dump_debug_info(mode) + + return compile_spec + + +def dump_delegation_info( + edge, intermediate_files_folder: Optional[str] = None, *, cortex_m: bool = False +): + graph_module = edge.exported_program().graph_module + delegation_info = get_delegation_info(graph_module) + if cortex_m: + for breakdown in delegation_info.delegation_by_operator.values(): + if breakdown.op_type.startswith("cortex_m"): + delegation_info.num_delegated_nodes += breakdown.non_delegated + delegation_info.num_non_delegated_nodes -= breakdown.non_delegated + breakdown.delegated += breakdown.non_delegated + breakdown.non_delegated = 0 + df = delegation_info.get_operator_delegation_dataframe() + table = tabulate(df, headers="keys", tablefmt="fancy_grid") + delegation_info_string = f"Delegation info:\n{delegation_info.get_summary()}\nDelegation table:\n{table}\n" + logger.debug(delegation_info_string) + if intermediate_files_folder is not None: + delegation_file_path = os.path.join( + intermediate_files_folder, "delegation_info.txt" + ) + with open(delegation_file_path, "w") as file: + file.write(delegation_info_string) + if not logger.isEnabledFor(logging.DEBUG): + print_delegation_summary( + delegation_info, intermediate_files_folder, cortex_m=cortex_m + ) + + +def print_delegation_summary( + delegation_info, + intermediate_files_folder: Optional[str] = None, + *, + cortex_m: bool = False, +) -> None: + non_delegated_ops = sorted( + ( + (breakdown.op_type, breakdown.non_delegated) + for breakdown in delegation_info.delegation_by_operator.values() + if breakdown.non_delegated > 0 + ), + key=lambda item: (-item[1], item[0]), + ) + + summary_lines = ["Delegation summary:"] + if delegation_info.num_delegated_nodes == 0: + summary_lines.append(" Model was not delegated.") + elif delegation_info.num_non_delegated_nodes == 0: + summary_lines.append(" Model was fully delegated.") + else: + summary_lines.append(" Model was partially delegated.") + + if cortex_m: + summary_lines.append( + f" Delegated Cortex-M ops: {delegation_info.num_delegated_nodes}" + ) + else: + summary_lines.append( + f" Delegated partitions for silicon acceleration: {delegation_info.num_delegated_subgraphs}" + ) + summary_lines.append( + f" Non-delegated ops: {delegation_info.num_non_delegated_nodes}" + ) + + if non_delegated_ops: + summary_lines.append(" Non-delegated operators:") + for op_type, count in non_delegated_ops: + summary_lines.append(f" - {op_type}: {count}") + + if intermediate_files_folder is not None: + delegation_file_path = os.path.join( + intermediate_files_folder, "delegation_info.txt" + ) + summary_lines.append("") + summary_lines.append("Full delegation report:") + summary_lines.append(f" {delegation_file_path}") + + print("\n".join(summary_lines)) + + +def _save_bpte_program( + exec_prog, + original_model: torch.nn.Module, + output_name: str, + example_inputs: Tuple[torch.Tensor, ...], + args, +): + # Construct MethodTestSuite for Each Method + + # Generate Test Suites + method_names = [ + method.name for method in exec_prog.executorch_program.execution_plan + ] + + program_inputs = {m_name: [example_inputs] for m_name in method_names} + + method_test_suites: List[MethodTestSuite] = [] + for m_name in method_names: + method_inputs = program_inputs[m_name] + + # To create a bundled program, we first create every test cases from input. We leverage eager model + # to generate expected output for each test input, and use MethodTestCase to hold the information of + # each test case. We gather all MethodTestCase for same method into one MethodTestSuite, and generate + # bundled program by all MethodTestSuites. + method_test_cases: List[MethodTestCase] = [] + + if args.intermediates: + # Save model.pth + intermediates_path = Path(args.intermediates) + model_path = os.path.join(intermediates_path, "model.pth") + try: + torch.save(original_model, model_path) + except: + logger.warning(f"Could not torch.save(model, {model_path})") + + method_index = 0 + for method_input in method_inputs: + output_ref = original_model(*method_input) + + logger.debug(f"input_{method_index}: {method_input}") + logger.debug(f"output_ref_{method_index}: {output_ref}") + + if args.intermediates: + # Save model input and referece output + input_path = os.path.join( + intermediates_path, f"input_{method_index}.pt" # type: ignore[possibly-undefined] + ) + try: + torch.save(method_input, input_path) + except: + logger.warning( + f"Could not torch.save(input_{method_index}, {input_path})" + ) + refoutput_path = os.path.join( + intermediates_path, f"output_ref_{method_index}.pt" + ) + try: + torch.save(output_ref, refoutput_path) + except: + logger.warning( + f"Could not torch.save(output_ref_{method_index}, {refoutput_path})" + ) + + method_test_cases.append( + MethodTestCase( + inputs=method_input, + expected_outputs=output_ref, + ) + ) + + method_index = method_index + 1 + + method_test_suites.append( + MethodTestSuite( + method_name=m_name, + test_cases=method_test_cases, + ) + ) + + # Generate BundledProgram + output_dir = os.path.dirname(output_name) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + _save_bundled_program(exec_prog, method_test_suites, output_name) + + +def quantize_model( + model: GraphModule, + example_inputs: Tuple[torch.Tensor], + compile_spec, + model_name: str, + strict_export: bool, + quant_mode: QuantMode, + calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], +) -> Tuple[GraphModule, ExportedProgram]: + model_quant = quantize( + model, + model_name, + compile_spec, + example_inputs, + quant_mode, + calibration_samples, + ) + # Wrap quantized model back into an exported_program + exported_program = torch.export.export( + model_quant, example_inputs, strict=strict_export + ) + + return model_quant, exported_program + + +def _to_edge_TOSA_delegate( + target: str, + exported_program: ExportedProgram, + compile_spec, + model: GraphModule, + quant_mode: Optional[QuantMode], + example_inputs: Tuple[torch.Tensor], + model_name: str, + strict_export: bool, + calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], + direct_drive: bool, +): + model_quant = None + if quant_mode is not None: + model_quant, exported_program = quantize_model( + model, + example_inputs, + compile_spec, + model_name, + strict_export, + quant_mode, + calibration_samples, + ) + + partitioner = create_partitioner(compile_spec) + + logger.info("Lowering Model to Edge with delegation...") + edge = to_edge_transform_and_lower( + exported_program, + partitioner=[partitioner], + compile_config=EdgeCompileConfig( + _check_ir_validity=False, + ), + ) + + # Replace quantized_decomposed::{quantize,dequantize}_per_tensor nodes + # with cortex_m:: equivalents for int8/int16 QDQ ops remaining outside + # the delegated subgraph. + edge = _apply_replace_quant_nodes(edge, target, direct_drive) + + return model_quant, edge + + +def _to_edge_cortex_m( + exported_program: ExportedProgram, + args, + model: GraphModule, + example_inputs: Tuple[torch.Tensor], + calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], + target_config: CortexMTargetConfig, +): + """Cortex-M/CMSIS-NN compilation path with no delegation.""" + logger.info( + f"Using Cortex-M/CMSIS-NN compilation path for cpu={target_config.cpu.name} " + f"backend={target_config.backend.name} " + f"layout={'explicit' if args.cortex_m_explicit_layout else 'dim-order'}" + ) + + if args.cortex_m_explicit_layout and not args.quantize: + raise RuntimeError( + "--cortex-m-explicit-layout requires --quantize; explicit layout " + "does not fall back to portable float spatial operators." + ) + + def _to_channels_last(x): + if isinstance(x, torch.Tensor): + if x.dim() == 4: + # Singleton channels can satisfy both contiguity checks while + # retaining NCHW strides, so always request the target format. + channels_last = x.to(memory_format=torch.channels_last) + if channels_last.stride() != x.stride(): + logger.warning( + "Converting input tensor with shape %s to channels_last", + list(x.shape), + ) + return channels_last + return x + elif isinstance(x, tuple): + return tuple(_to_channels_last(t) for t in x) + return x + + if not args.quantize: + logger.warning( + "Quantization is DISABLED. Cortex-M typically requires quantization." + ) + model_quant = None + else: + if not args.cortex_m_explicit_layout: + model = model.to(memory_format=torch.channels_last) # type: ignore[call-overload] + example_inputs = tuple(_to_channels_last(x) for x in example_inputs) + # Refresh fake-tensor strides after changing the captured module's + # memory format so the legacy quantizer sees channels-last inputs. + model = torch.export.export( + model, example_inputs, strict=args.strict_export + ).module() + + quantizer = CortexMQuantizer(use_explicit_layout=args.cortex_m_explicit_layout) + + prepared = prepare_pt2e(model, quantizer) + + if calibration_samples is None: + calibration_samples = [example_inputs] + + for sample in calibration_samples: + prepared(*sample) + + model_quant = convert_pt2e(prepared) + + exported_program = torch.export.export( + model_quant, example_inputs, strict=args.strict_export + ) + + logger.info("Lowering Model to Edge for Cortex-M...") + edge = to_edge_transform_and_lower( + exported_program, + compile_config=cortex_m_edge_compile_config(), + transform_passes=CortexMPassManager( + target_config=target_config, + use_explicit_layout=args.cortex_m_explicit_layout, + ), + ) + + return model_quant, edge, example_inputs + + +def _to_edge_no_delegate( + args, + exported_program: ExportedProgram, + compile_spec, + model: GraphModule, + quant_mode: Optional[QuantMode], + example_inputs: Tuple[torch.Tensor], + model_name: str, + strict_export: bool, + calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]], +): + model_quant = None + if quant_mode is not None: + # As we can target multiple output encodings, one must + # be specified. + model, exported_program = quantize_model( + model, + example_inputs, + compile_spec, + model_name, + strict_export, + quant_mode, + calibration_samples, + ) + model_quant = model + + logger.info("Lowering Model to Edge...") + edge = to_edge_transform_and_lower( + exported_program, + compile_config=EdgeCompileConfig( + _check_ir_validity=False, + ), + ) + + # Replace quantized_decomposed::{quantize,dequantize}_per_tensor nodes + # with cortex_m:: equivalents for int8/int16 QDQ ops remaining outside + # the delegated subgraph. + edge = _apply_replace_quant_nodes(edge, args.target, args.direct_drive) + + return model_quant, edge + + +def export(args: argparse.Namespace) -> None: # noqa: C901 + """Export a model using the parsed compiler options. + + Args: + args (argparse.Namespace): Validated options from the AoT CLI. + + """ + # if we have custom ops, register them before processing the model + if args.so_library is not None: + logger.info(f"Loading custom ops from {args.so_library}") + torch.ops.load_library(args.so_library) + + if ( + args.model_name in MODELS.keys() + and args.delegate is True + and MODELS[args.model_name].can_delegate is False # type: ignore[attr-defined] + ): + raise RuntimeError(f"Model {args.model_name} cannot be delegated.") + + # Pick model from one of the supported lists + original_model, example_inputs = get_model_and_inputs_from_name( + args.model_name, args.model_input + ) + calibration_samples = load_calibration_samples( + args.calibration_data, example_inputs + ) + model = original_model.eval() + model.requires_grad_(False) + + # export under the assumption we quantize, the exported form also works + # in to_edge if we don't quantize + exported_program = torch.export.export( + model, example_inputs, strict=args.strict_export + ) + + model = exported_program.module() + + model_name = os.path.basename(os.path.splitext(args.model_name)[0]) + if args.intermediates: + os.makedirs(args.intermediates, exist_ok=True) + + # We only support Python3.10 and above, so use a later pickle protocol + torch.export.save( + exported_program, + f"{args.intermediates}/{model_name}_exported_program.pt2", + pickle_protocol=5, + ) + + # Quantize if required + model_quant = None + if args.quantize: + quant_mode = QuantMode.A16W8 if "int16" in args.target else QuantMode.INT8 + else: + quant_mode = None + + if args.target.startswith("cortex-m"): + # Cortex-M path: CMSIS-NN portable kernels, no delegation + target_config = CortexMTargetConfig.from_target_string(args.target) + if args.delegate: + logger.warning( + f"--delegate is ignored for target {args.target!r} " + "(this target does not use delegated ops)." + ) + args.delegate = False + model_quant, edge, example_inputs = _to_edge_cortex_m( + exported_program, + args, + model, + example_inputs, + calibration_samples, + target_config, + ) + elif args.delegate: + # As we can target multiple output encodings, one must + # be specified. + model_quant, edge = _to_edge_TOSA_delegate( + args.target, + exported_program, + _get_compile_spec(args), + model, + quant_mode, + example_inputs, + args.model_name, + args.strict_export, + calibration_samples, + args.direct_drive, + ) + else: + model_quant, edge = _to_edge_no_delegate( + args, + exported_program, + _get_compile_spec(args), + model, + quant_mode, + example_inputs, + args.model_name, + args.strict_export, + calibration_samples, + ) + + dump_delegation_info( + edge, args.intermediates, cortex_m=args.target.startswith("cortex-m") + ) + + logger.info("Lowering Model to ExecuTorch...") + try: + exec_prog = edge.to_executorch( + config=ExecutorchBackendConfig(extract_delegate_segments=False) + ) + except RuntimeError as e: + if "Missing out variants" in str(e.args[0]): + raise RuntimeError( + e.args[0] + + ".\nThis likely due to an external so library not being loaded. Supply a path to it with the -s flag." + ).with_traceback(e.__traceback__) from None + else: + raise e + + output_name = f"{model_name}" + ( + f"_arm_delegate_{args.target}" + if args.delegate is True + else f"_arm_{args.target}" + ) + + if args.bundleio: + output_file_name = f"{output_name}.bpte" + else: + output_file_name = f"{output_name}.pte" + + if args.output is not None: + if args.output.endswith(".pte") or args.output.endswith(".bpte"): + # --output is a pte or bundle pte filename use it as output name + if args.bundleio and not args.output.endswith(".bpte"): + raise RuntimeError( + f"--bundleio expects a .bpte file ending to --output and not .pte {args.output}" + ) + if not args.bundleio and not args.output.endswith(".pte"): + raise RuntimeError( + f"When not using --bundleio a .bpte file should not be use as --output {args.output}" + ) + output_file_name = args.output + output_dir = os.path.dirname(output_file_name) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + else: + # --output is a folder + os.makedirs(args.output, exist_ok=True) + output_file_name = os.path.join(args.output, output_file_name) + + if args.bundleio: + # Realize the quantization impact on numerics when generating reference output + reference_model = original_model if not model_quant else model_quant + _save_bpte_program( + exec_prog, reference_model, output_file_name, example_inputs, args + ) + print(f"Bundle PTE file saved as {output_file_name}") + else: + save_pte_program(exec_prog, output_file_name) + print(f"PTE file saved as {output_file_name}") + + if args.bundleio or args.etrecord: + etrecord_file_name = os.path.splitext(output_file_name)[0] + "_etrecord.bin" + try: + edge_program_manager_copy = copy.deepcopy(edge) + generate_etrecord(etrecord_file_name, edge_program_manager_copy, exec_prog) + print(f"ETRecord saved as {etrecord_file_name}") + except Exception as e: + # Treat ETRecord failures as non-fatal only when generated as a side-effect + # of --bundleio. When --etrecord is explicitly requested, fail loudly. + if args.bundleio and not args.etrecord: + logger.warning(f"ETRecord generation failed (non-fatal): {e}") + else: + raise diff --git a/backends/arm/scripts/evaluate_model.py b/backends/arm/scripts/evaluate_model.py index eadbba1a4a1..eca73effd08 100644 --- a/backends/arm/scripts/evaluate_model.py +++ b/backends/arm/scripts/evaluate_model.py @@ -27,7 +27,7 @@ import torch -from executorch.backends.arm.scripts.aot_arm_compiler import ( +from executorch.backends.arm.scripts.aot_arm_compiler_backend import ( CALIBRATION_MAX_SAMPLES, dump_delegation_info, get_model_and_inputs_from_name, diff --git a/backends/arm/test/misc/test_ethosu_scratch.py b/backends/arm/test/misc/test_ethosu_scratch.py index d587b701fc3..d5b3d85bf83 100644 --- a/backends/arm/test/misc/test_ethosu_scratch.py +++ b/backends/arm/test/misc/test_ethosu_scratch.py @@ -13,10 +13,8 @@ from executorch.backends.arm import arm_vela from executorch.backends.arm.ethosu import EthosUCompileSpec from executorch.backends.arm.ethosu.backend import EthosUBackend -from executorch.backends.arm.scripts.aot_arm_compiler import ( - _get_args, - _get_compile_spec, -) +from executorch.backends.arm.scripts.aot_arm_compiler import _get_args +from executorch.backends.arm.scripts.aot_arm_compiler_backend import _get_compile_spec from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import EthosUPipelineINTBase from executorch.backends.arm.tosa.backend import TOSABackend diff --git a/backends/cortex_m/__init__.py b/backends/cortex_m/__init__.py index 98de203f0ac..63f62750941 100644 --- a/backends/cortex_m/__init__.py +++ b/backends/cortex_m/__init__.py @@ -8,3 +8,10 @@ logger = logging.getLogger(__name__) if logger.level == logging.NOTSET: logger.setLevel(min(logging.INFO, logging.root.getEffectiveLevel())) + logger.info( + "Using default logging level %s for %s. Override with " + "logging.getLogger(%r).setLevel(logging.DEBUG).", + logging.getLevelName(logger.level), + __name__, + __name__, + ) diff --git a/backends/cortex_m/test/test_explicit_layout.py b/backends/cortex_m/test/test_explicit_layout.py index c4503cc4872..2be1cc6345f 100644 --- a/backends/cortex_m/test/test_explicit_layout.py +++ b/backends/cortex_m/test/test_explicit_layout.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -44,7 +45,9 @@ def forward(self, x): def _compile(module, inputs, *, explicit_layout: bool, quantize: bool = True): - from executorch.backends.arm.scripts.aot_arm_compiler import _to_edge_cortex_m + from executorch.backends.arm.scripts.aot_arm_compiler_backend import ( + _to_edge_cortex_m, + ) exported_program = torch.export.export(module, inputs, strict=True) return _to_edge_cortex_m( diff --git a/exir/backend/backend_api.py b/exir/backend/backend_api.py index bc59cb1434a..49a60a582f8 100644 --- a/exir/backend/backend_api.py +++ b/exir/backend/backend_api.py @@ -39,6 +39,8 @@ from torch._export.utils import is_buffer, is_lifted_tensor_constant, is_param from torch.export.exported_program import ExportedProgram, InputSpec, OutputSpec +logger = logging.getLogger(__name__) + @singledispatch def to_backend(args): @@ -281,10 +283,10 @@ def _partition_and_lower_one_graph_module( ) if len(node_list) == 0: - logging.debug(f"Did not find any nodes for tag {tag}") + logger.debug(f"Did not find any nodes for tag {tag}") continue - logging.debug(f"For tag {tag}, found nodes {node_list}") + logger.debug(f"For tag {tag}, found nodes {node_list}") # Tag the nodes that are params as buffers, so we can order the submodule as (Parms + Buffers) (User Inputs) replace_ctx = ( @@ -304,7 +306,7 @@ def _partition_and_lower_one_graph_module( # Copy the output node meta from the original output node, because # create_submodule_from_nodes doesn't cover the meta field submodule_output_node.meta = tagged_graph_module_output_node.meta - logging.debug("Partitioned graph module: %s", tagged_graph_module.graph) + logger.debug("Partitioned graph module: %s", tagged_graph_module.graph) ( submodule_program, @@ -401,7 +403,7 @@ def to_backend( try: fake_edge_program = get_fake_program(edge_program) except Exception as e: - logging.warning( + logger.warning( f"Error in get_fake_program for graph {edge_program.graph_module}, fallback to deepcopy: {e}" ) fake_edge_program = copy.deepcopy(edge_program) @@ -415,7 +417,7 @@ def to_backend( edge_program.graph_module, ), f"The partitioner {partitioner_instance} should not modify the graph module" else: - logging.warning("Disabled validating the partitioner.") + logger.warning("Disabled validating the partitioner.") assert ( partitioner_result.partition_tags is not None @@ -466,10 +468,10 @@ def _create_partitions_in_graph_module( ) if len(node_list) == 0: - logging.debug(f"Did not find any nodes for tag {tag}") + logger.debug(f"Did not find any nodes for tag {tag}") continue - logging.debug(f"For tag {tag}, found nodes {node_list}") + logger.debug(f"For tag {tag}, found nodes {node_list}") # Tag the nodes that are params as buffers, so we can order the submodule as (Parms + Buffers) (User Inputs) replace_ctx = ( @@ -487,7 +489,7 @@ def _create_partitions_in_graph_module( submodule_output_node = submodule.graph.output_node() # Copy the output node meta from the original output node, because # create_submodule_from_nodes doesn't cover the meta field - logging.debug("Partitioned graph module: %s", tagged_graph_module.graph) + logger.debug("Partitioned graph module: %s", tagged_graph_module.graph) ( submodule_program, toplevel_input_specs_to_delete, @@ -728,7 +730,7 @@ def to_backend( try: fake_edge_program = get_fake_program(edge_program) except Exception as e: - logging.warning( + logger.warning( f"Error in get_fake_program for graph {edge_program.graph_module}, fallback to deepcopy: {e}" ) fake_edge_program = copy.deepcopy(edge_program) @@ -743,7 +745,7 @@ def to_backend( edge_program.graph_module, ), f"The partitioner {partitioner_instance} should not modify the graph module" else: - logging.warning("Disabled validating the partitioner.") + logger.warning("Disabled validating the partitioner.") assert ( partitioner_result.partition_tags is not None