From 735aedb08f55e3062c6d50bf673e1dc124381208 Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Fri, 20 Nov 2026 16:37:13 -0800 Subject: [PATCH] [Espressif] Lower fused-quant spatial mean to ESP-NN Accelerate static int8 means over the spatial dimensions of 4-D tensors with independent input and output quantization parameters. Convert the input to NHWC, run the vendor mean kernel per batch, and restore singleton dimensions for keepdim. Existing permutation cleanup removes redundant conversions after spatial producers. Unsupported reductions and unsafe arithmetic retain portable execution. Cover lowering, serialization, layout cleanup, arithmetic bounds, strict export, and shared ESP32-S3 cases. Refresh the MLPerf Tiny expectations to accelerate the means in ResNet-8 and MobileNetV1 0.25. Validation: 176 Python checks, four model coverage/serialization tests, 19 ESP32-S3 emulator cases, and changed-file lint passed. Authored with OpenAI Codex. --- backends/espressif/CMakeLists.txt | 1 + backends/espressif/ops/op_quantized_mean.cpp | 68 +++++++ backends/espressif/ops/operators.py | 25 +++ backends/espressif/ops/operators.yaml | 6 + backends/espressif/passes/lower_to_esp.py | 104 ++++++++-- .../espressif/test/models/test_mlperf_tiny.py | 6 +- backends/espressif/test/ops/test_mean.py | 190 ++++++++++++++++++ 7 files changed, 384 insertions(+), 16 deletions(-) create mode 100644 backends/espressif/ops/op_quantized_mean.cpp create mode 100644 backends/espressif/test/ops/test_mean.py diff --git a/backends/espressif/CMakeLists.txt b/backends/espressif/CMakeLists.txt index 59c999b54e3..a134a7f026c 100644 --- a/backends/espressif/CMakeLists.txt +++ b/backends/espressif/CMakeLists.txt @@ -12,6 +12,7 @@ idf_component_register( "ops/op_quantized_add.cpp" "ops/op_quantized_max_pool2d.cpp" "ops/op_quantized_avg_pool2d.cpp" + "ops/op_quantized_mean.cpp" "ops/op_quantized_linear.cpp" "ops/op_quantized_conv2d.cpp" "ops/op_quantized_depthwise_conv2d.cpp" diff --git a/backends/espressif/ops/op_quantized_mean.cpp b/backends/espressif/ops/op_quantized_mean.cpp new file mode 100644 index 00000000000..681e162a3fd --- /dev/null +++ b/backends/espressif/ops/op_quantized_mean.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include + +#include +#include + +namespace espressif::native { + +using executorch::aten::Tensor; +using executorch::runtime::KernelRuntimeContext; + +// cppcheck-suppress unusedFunction +Tensor& quantized_mean_int8_out( + KernelRuntimeContext& ctx, + const Tensor& input, + int64_t input_zero_point, + int64_t output_zero_point, + int64_t output_multiplier, + int64_t output_shift, + Tensor& out) { + ET_KERNEL_CHECK(ctx, input.dim() == 4, InvalidArgument, out); + const auto batches = input.size(0); + const auto height = input.size(1); + const auto width = input.size(2); + const auto channels = input.size(3); + ET_KERNEL_CHECK( + ctx, + height > 0 && width > 0 && channels > 0 && + height <= std::numeric_limits::max() / width && + height * width <= std::numeric_limits::max() / channels, + InvalidArgument, + out); + const auto count = height * width; + const auto input_bytes = count * channels; + ET_KERNEL_CHECK( + ctx, + count <= std::numeric_limits::max() / 255 && batches >= 0 && + static_cast(batches) <= input.nbytes() / input_bytes && + static_cast(batches) <= out.nbytes() / channels, + InvalidArgument, + out); + for (int64_t batch = 0; batch < batches; ++batch) { + esp_nn_mean_nhwc_s8( + input.const_data_ptr() + batch * input_bytes, + out.mutable_data_ptr() + batch * channels, + static_cast(height), + static_cast(width), + static_cast(channels), + static_cast(input_zero_point), + static_cast(output_zero_point), + static_cast(output_multiplier), + static_cast(output_shift)); + } + return out; +} + +} // namespace espressif::native diff --git a/backends/espressif/ops/operators.py b/backends/espressif/ops/operators.py index 65f0dccb01c..c82be94ee0b 100644 --- a/backends/espressif/ops/operators.py +++ b/backends/espressif/ops/operators.py @@ -66,6 +66,14 @@ "quantized_avg_pool2d_int8.out(Tensor input, int[2] kernel_size, int[2] stride, " "int[2] padding, *, Tensor(a!) out) -> Tensor(a!)" ) +lib.define( + "quantized_mean_int8(Tensor input, int input_zero_point, int output_zero_point, " + "int output_multiplier, int output_shift) -> Tensor" +) +lib.define( + "quantized_mean_int8.out(Tensor input, int input_zero_point, int output_zero_point, " + "int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)" +) lib.define( "quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, " "Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, " @@ -154,6 +162,23 @@ def quantized_avg_pool2d_int8(input, kernel_size, stride, padding): return result +@register_fake("esp_nn::quantized_mean_int8") +def quantized_mean_int8_meta( + input, input_zero_point, output_zero_point, output_multiplier, output_shift +): + batch, _, _, channels = input.shape + return input.new_empty((batch, channels)) + + +@impl(lib, "quantized_mean_int8", "CompositeExplicitAutograd") +def quantized_mean_int8( + input, input_zero_point, output_zero_point, output_multiplier, output_shift +): + total = (input.to(torch.int64) - input_zero_point).sum((1, 2)) + result = requantize(total, output_multiplier, output_shift) + output_zero_point + return result.clamp(-128, 127).to(torch.int8) + + @register_fake("esp_nn::quantized_mul_int8") def quantized_mul_int8_meta( input1, diff --git a/backends/espressif/ops/operators.yaml b/backends/espressif/ops/operators.yaml index 2be14d16d8b..74941a91c36 100644 --- a/backends/espressif/ops/operators.yaml +++ b/backends/espressif/ops/operators.yaml @@ -40,6 +40,12 @@ - arg_meta: null kernel_name: espressif::quantized_avg_pool2d_int8_out +- func: esp_nn::quantized_mean_int8.out(Tensor input, int input_zero_point, int output_zero_point, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!) + variants: function + kernels: + - arg_meta: null + kernel_name: espressif::quantized_mean_int8_out + - func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!) variants: function kernels: diff --git a/backends/espressif/passes/lower_to_esp.py b/backends/espressif/passes/lower_to_esp.py index e55468d86e2..c05ecb1ba08 100644 --- a/backends/espressif/passes/lower_to_esp.py +++ b/backends/espressif/passes/lower_to_esp.py @@ -342,6 +342,85 @@ def _lower_avg_pool2d(program, node) -> bool: return True +def _mean_multiplier(input_params, output_params, count): + try: + multiplier, shift = quantize_multiplier( + input_params.scale / output_params.scale / count + ) + except ValueError: + return None + max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp)) + if ( + shift < -30 + or count > torch.iinfo(torch.int32).max // 255 + or (count * max_input) << max(shift, 0) + > torch.iinfo(torch.int32).max - abs(output_params.zp) + ): + return None + max_float_input = c_float(max_input * c_float(input_params.scale).value).value + rounding_bound = 1 - count * torch.finfo(torch.float32).eps + if ( + rounding_bound <= 0 + or max_float_input * count / rounding_bound > torch.finfo(torch.float32).max + or not math.isfinite(c_float(1.0 / output_params.scale).value) + ): + return None + return multiplier, shift + + +def _lower_mean(program, node) -> bool: + inp = get_arg(node, "inp", torch.fx.Node) + value = inp.meta.get("val") + if ( + not isinstance(value, torch.Tensor) + or value.dim() != 4 + or value.dtype != torch.int8 + or any(type(dim) is not int or dim <= 0 for dim in value.shape) + or value.numel() > torch.iinfo(torch.int32).max + or not value.is_contiguous() + or get_arg(node, "dtype") not in (None, torch.float32) + ): + return False + dims = get_arg(node, "dim") + if ( + not isinstance(dims, (tuple, list)) + or len(dims) != 2 + or any(type(dim) is not int or not -4 <= dim < 4 for dim in dims) + or sorted(dim % 4 for dim in dims) != [2, 3] + ): + return False + input_params, output_params = ( + _qparams(program, node, prefix) for prefix in ("inp", "out") + ) + if input_params is None or output_params is None: + return False + batch, channels, height, width = value.shape + multiplier = _mean_multiplier(input_params, output_params, height * width) + if multiplier is None: + return False + graph = node.graph + with graph.inserting_before(node): + nhwc = graph.call_function( + exir_ops.edge.aten.permute_copy.default, (inp, [0, 2, 3, 1]) + ) + nhwc.meta["val"] = value.permute(0, 2, 3, 1).contiguous() + replacement = graph.call_function( + exir_ops.edge.esp_nn.quantized_mean_int8.default, + (nhwc, input_params.zp, output_params.zp, *multiplier), + ) + replacement.meta = node.meta.copy() + replacement.meta["val"] = value.new_empty((batch, channels)) + if get_arg(node, "keepdim"): + replacement = graph.call_function( + exir_ops.edge.aten.view_copy.default, + (replacement, [batch, channels, 1, 1]), + ) + replacement.meta = node.meta.copy() + node.replace_all_uses_with(replacement) + graph.erase_node(node) + return True + + def _weighted_multipliers(input_params, output_params, weight, scales, bias): rescales = [ quantize_multiplier(input_params.scale * scale / output_params.scale) @@ -682,6 +761,14 @@ def _lower_conv2d(program, node, constants, target, kernel_mode): return True +_UNARY_LOWERINGS = { + exir_ops.edge.fused_quant.mean.dim: _lower_mean, + exir_ops.edge.fused_quant.avg_pool2d_channels_last.default: _lower_avg_pool2d, + exir_ops.edge.fused_quant.max_pool2d_with_indices_channels_last.default: _lower_max_pool2d, + **dict.fromkeys(_LUT_OPS, _lower_lut), +} + + class LowerToEsp(ExportedProgramPassBase): """Lower supported fused-quant operators to ESP-NN.""" @@ -715,8 +802,9 @@ def call(self, exported_program): modified = False for node in list(graph.nodes): - if node.target in _LUT_OPS: - modified |= _lower_lut(exported_program, node) + unary_lowering = _UNARY_LOWERINGS.get(node.target) + if unary_lowering is not None: + modified |= unary_lowering(exported_program, node) continue if node.target == exir_ops.edge.fused_quant.linear.default: modified |= _lower_linear(exported_program, node, constants) @@ -729,18 +817,6 @@ def call(self, exported_program): exported_program, node, constants, self.target, self.kernel_mode ) continue - if ( - node.target - == exir_ops.edge.fused_quant.avg_pool2d_channels_last.default - ): - modified |= _lower_avg_pool2d(exported_program, node) - continue - if ( - node.target - == exir_ops.edge.fused_quant.max_pool2d_with_indices_channels_last.default - ): - modified |= _lower_max_pool2d(exported_program, node) - continue lowering = _BINARY_LOWERINGS.get(node.target) if lowering is None: continue diff --git a/backends/espressif/test/models/test_mlperf_tiny.py b/backends/espressif/test/models/test_mlperf_tiny.py index 383ba5a69e5..1c760576ea0 100644 --- a/backends/espressif/test/models/test_mlperf_tiny.py +++ b/backends/espressif/test/models/test_mlperf_tiny.py @@ -69,15 +69,16 @@ class ModelCase: _FQ.relu.default: 7, _FQ.add.default: 3, _FQ.linear.default: 1, + _FQ.mean.dim: 1, }, { _ESP.quantized_conv2d_int8.default: 9, _ESP.quantized_add_int8.default: 3, _ESP.quantized_linear_int8.default: 1, + _ESP.quantized_mean_int8.default: 1, }, { "aten::constant_pad_nd": 2, - "aten::mean.dim": 1, "aten::_softmax": 1, }, ), @@ -87,15 +88,16 @@ class ModelCase: _FQ.convolution.default: 27, _FQ.relu.default: 27, _FQ.linear.default: 1, + _FQ.mean.dim: 1, }, { _ESP.quantized_conv2d_int8.default: 14, _ESP.quantized_depthwise_conv2d_int8.default: 13, _ESP.quantized_linear_int8.default: 1, + _ESP.quantized_mean_int8.default: 1, }, { "aten::constant_pad_nd": 5, - "aten::mean.dim": 1, "aten::_softmax": 1, }, ), diff --git a/backends/espressif/test/ops/test_mean.py b/backends/espressif/test/ops/test_mean.py new file mode 100644 index 00000000000..bce98e60ed9 --- /dev/null +++ b/backends/espressif/test/ops/test_mean.py @@ -0,0 +1,190 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import math + +import pytest +import torch +from executorch.backends.espressif.lowering import get_esp_fused_quant_recipe +from executorch.backends.espressif.passes.lower_to_esp import _mean_multiplier, _QParams +from executorch.backends.espressif.test.tester import EspTestCase, EspTester +from executorch.backends.test.suite.operators.test_mean import MeanModel +from executorch.backends.transforms.fused_quant.graph_utils import ( + get_qparams_from_node, + get_scale, +) +from executorch.exir._serialize._program import deserialize_pte_binary +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.export import export + +_MEAN = exir_ops.edge.aten.mean.dim +_FQ_MEAN = exir_ops.edge.fused_quant.mean.dim +_ESP_MEAN = exir_ops.edge.esp_nn.quantized_mean_int8.default +_PERMUTE = exir_ops.edge.aten.permute_copy.default + + +class EspMean(MeanModel): + def __init__(self, dim=(2, 3), *, supported=True, **kwargs): + super().__init__(dim=dim, **kwargs) + self.ops_before_transforms = {_FQ_MEAN: 1} + self.ops_after_transforms = { + _FQ_MEAN: 0, + _MEAN: 0 if supported else 1, + _ESP_MEAN: 1 if supported else 0, + } + + +class EspMeanAfterPool(EspMean): + def __init__(self): + super().__init__() + self.ops_before_transforms[ + exir_ops.edge.fused_quant.max_pool2d_with_indices.default + ] = 1 + self.ops_after_transforms[ + exir_ops.edge.esp_nn.quantized_max_pool2d_int8.default + ] = 1 + self.ops_after_transforms[_PERMUTE] = 1 + + def forward(self, x): + return super().forward(torch.nn.functional.max_pool2d(x, 2)) + + +class EspMeanWithFloatUsers(EspMean): + def forward(self, x): + result = super().forward(x) + return result, torch.sin(result), torch.sin(x) + + +def _input(shape): + values = torch.arange(math.prod(shape)).reshape(shape) + return (values * 0.7).sin() * 3 + values.remainder(5) * 0.2 + + +test_cases = { + "channels_1": EspTestCase(EspMean(), (_input((1, 1, 5, 7)),)), + "channels_3": EspTestCase(EspMean(), (_input((1, 3, 5, 7)),)), + "channels_4": EspTestCase(EspMean(), (_input((1, 4, 5, 7)),)), + "channels_17": EspTestCase(EspMean(), (_input((1, 17, 3, 5)),)), + "batches": EspTestCase(EspMean(), (_input((3, 7, 5, 9)),)), + "keepdim": EspTestCase(EspMean(keepdim=True), (_input((2, 5, 7, 3)),)), + "negative_dims": EspTestCase(EspMean(dim=(-2, -1)), (_input((1, 4, 5, 7)),)), + "reversed_dims": EspTestCase(EspMean(dim=(3, 2)), (_input((1, 4, 5, 7)),)), + "single_row": EspTestCase(EspMean(), (_input((1, 4, 1, 11)),)), + "single_column": EspTestCase(EspMean(), (_input((1, 4, 11, 1)),)), + "single_pixel": EspTestCase(EspMean(), (_input((2, 4, 1, 1)),)), + "negative": EspTestCase(EspMean(), (_input((1, 7, 5, 9)) - 5,)), + "zeros": EspTestCase(EspMean(), (torch.zeros(1, 4, 3, 5),)), + "pool_producer": EspTestCase(EspMeanAfterPool(), (_input((2, 4, 8, 10)),)), + "float_users": EspTestCase(EspMeanWithFloatUsers(), (_input((1, 4, 5, 7)),)), + "all_dims_fallback": EspTestCase( + EspMean(dim=None, supported=False), (_input((1, 4, 5, 7)),) + ), + "channel_fallback": EspTestCase( + EspMean(dim=(1,), supported=False), (_input((1, 4, 5, 7)),) + ), + "batch_fallback": EspTestCase( + EspMean(dim=(0, 2), supported=False), (_input((2, 4, 5, 7)),) + ), + "unbatched_fallback": EspTestCase( + EspMean(dim=(1, 2), supported=False), (_input((4, 5, 7)),) + ), +} + + +@pytest.fixture(params=test_cases.values(), ids=test_cases.keys()) +def mean_case(request): + case = request.param + tester = EspTester(case.model, case.example_inputs).test_dialect( + case.model.ops_before_transforms, + case.model.ops_after_transforms, + qtol=case.model.ops_after_transforms[_ESP_MEAN], + atol=0, + rtol=torch.finfo(torch.float32).eps, + ) + return case, tester + + +def test_dialect_mean(mean_case): + case, tester = mean_case + for node in tester.get_artifact().exported_program().graph.nodes: + if node.target == _ESP_MEAN: + assert node.args[0].meta["val"].is_contiguous() + assert node.args[0].meta["val"].dtype == torch.int8 + assert node.args[0].meta["val"].shape[-1] == case.example_inputs[0].shape[1] + if node.target == _PERMUTE: + assert node.meta["val"].dtype == torch.int8 + tester.to_executorch().serialize() + plan = deserialize_pte_binary(tester.get_artifact()).program.execution_plan[0] + assert not plan.delegates + operators = {(op.name, op.overload) for op in plan.operators} + assert (("esp_nn::quantized_mean_int8", "out") in operators) == bool( + case.model.ops_after_transforms[_ESP_MEAN] + ) + + +def test_implementation_mean(mean_case, pytestconfig): + case, tester = mean_case + tester.test_implementation( + pytestconfig.rootpath, + rtol=( + torch.finfo(torch.float32).eps + if isinstance(case.model, EspMeanWithFloatUsers) + else 0 + ), + num_inferences=2, + ) + + +def test_mean_uses_independent_qparams(): + case = test_cases["channels_4"] + tester = EspTester(case.model, case.example_inputs).export_fused_quant().to_edge() + program = tester.get_artifact().exported_program() + (mean,) = program.graph.find_nodes(op="call_function", target=_FQ_MEAN) + output_scale = get_scale(program, get_qparams_from_node(mean, "out")) + assert get_scale(program, get_qparams_from_node(mean, "inp")) != output_scale + expected = program.module()(*case.example_inputs) + tester.run_passes().check_node_count({_ESP_MEAN: 1}) + actual = tester.get_artifact().exported_program().module()(*case.example_inputs) + torch.testing.assert_close( + actual, expected, atol=output_scale, rtol=torch.finfo(torch.float32).eps + ) + + +@pytest.mark.parametrize( + "input_scale,output_scale,count,supported", + [ + (0.125, 0.03125, 35, True), + (1, 1, torch.iinfo(torch.int32).max // 255 + 1, False), + (1, 1e-12, 35, False), + (1e-12, 1, 35, False), + (1e-38, 1e-39, 35, False), + (1e38, 1e38, 2, False), + ], +) +def test_mean_arithmetic_bounds(input_scale, output_scale, count, supported): + assert ( + _mean_multiplier( + _QParams(input_scale, -128), _QParams(output_scale, 127), count + ) + is not None + ) == supported + + +def test_mean_strict_recipe(): + case = test_cases["keepdim"] + session = export( + case.model, + example_inputs=[case.example_inputs], + export_recipe=get_esp_fused_quant_recipe(allow_cpu_fallback=False), + ) + assert session.get_executorch_program_manager().buffer + case = test_cases["channel_fallback"] + with pytest.raises(RuntimeError, match="aten::mean"): + export( + case.model, + example_inputs=[case.example_inputs], + export_recipe=get_esp_fused_quant_recipe(allow_cpu_fallback=False), + )