From 2df27d7a6e49a5bb040b660470a02dc29f87696b Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Wed, 18 Nov 2026 11:13:57 -0800 Subject: [PATCH 1/2] [Espressif] Add MLPerf Tiny operator-library coverage tests Exercise deep autoencoder, DS-CNN, ResNet-8 and MobileNetV1 0.25 through EspTester using deterministic weights, inputs and calibration. Assert accelerated and portable operator inventories, compare lowering with the Fused Quant reference, check serialization, and print Q/DQ and layout counts. Run the same cases on the ESP32-S3 emulator in the optimized CI job. Include the portable padding, mean and softmax kernels needed by the models. Compare classifier probabilities with an absolute tolerance of 1e-3 instead of using intermediate activation scales. Retain the autoencoder quantization tolerance. Authored with OpenAI Codex. --- .ci/scripts/test_espressif.sh | 6 +- backends/espressif/test/models/__init__.py | 5 + .../espressif/test/models/test_mlperf_tiny.py | 164 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 backends/espressif/test/models/__init__.py create mode 100644 backends/espressif/test/models/test_mlperf_tiny.py diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index 85505f65a9b..5a533e77472 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -80,6 +80,7 @@ espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \ | tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -) if [[ "${espnn_kernels}" == optimized ]]; then espnn_ops="${espnn_ops},aten::sub.out,aten::sin.out,aten::relu.out,aten::max_pool2d_with_indices.out,aten::avg_pool2d.out,aten::permute_copy.out,aten::convolution.out,aten::addmm.out,aten::mm.out,aten::mul.Scalar_out" + espnn_ops="${espnn_ops},aten::constant_pad_nd.out,aten::_softmax.out,aten::mean.out" fi toolchain_file="${IDF_PATH}/tools/cmake/toolchain-${target}.cmake" @@ -194,9 +195,12 @@ if [[ "${espnn_kernels}" == optimized ]]; then logged "operator-library-host-tests.log" python -m pytest -q \ backends/espressif/test/test_pass_manager.py \ backends/espressif/test/ops -k 'not implementation' + logged "model-coverage-tests.log" python -m pytest -q -s \ + backends/espressif/test/models -k 'not implementation' # Stop on failure so later cases cannot overwrite its firmware and logs. ESPRESSIF_BUILD_DIR="${build_dir}" python -m pytest \ - -x -v --tb=short backends/espressif/test/ops -k implementation \ + -x -v --tb=short backends/espressif/test/ops \ + backends/espressif/test/models -k implementation \ 2>&1 | tee "${build_dir}/operator-library-tests.log" for sdkconfig in "${build_dir}"/operator-library-*/sdkconfig; do expected_config=CONFIG_NN_OPTIMIZED diff --git a/backends/espressif/test/models/__init__.py b/backends/espressif/test/models/__init__.py new file mode 100644 index 00000000000..2e41cd717f6 --- /dev/null +++ b/backends/espressif/test/models/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/backends/espressif/test/models/test_mlperf_tiny.py b/backends/espressif/test/models/test_mlperf_tiny.py new file mode 100644 index 00000000000..383ba5a69e5 --- /dev/null +++ b/backends/espressif/test/models/test_mlperf_tiny.py @@ -0,0 +1,164 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +from collections import Counter +from dataclasses import dataclass + +import pytest +import torch +from executorch.backends.espressif.lowering import get_operator_library_report +from executorch.backends.espressif.scratch import get_scratch_queries +from executorch.backends.espressif.target import EspKernelMode, EspTarget +from executorch.backends.espressif.test.tester import EspTester +from executorch.examples.models.mlperf_tiny.deep_autoencoder import DeepAutoEncoderModel +from executorch.examples.models.mlperf_tiny.ds_cnn import DSCNNKWSModel +from executorch.examples.models.mlperf_tiny.mobilenet_v1_025 import MobileNetV1025Model +from executorch.examples.models.mlperf_tiny.resnet8 import ResNet8Model +from executorch.examples.models.model_base import EagerModelBase +from executorch.exir._serialize._program import deserialize_pte_binary +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.dialects.edge._ops import EdgeOpOverload + +_FQ = exir_ops.edge.fused_quant +_ESP = exir_ops.edge.esp_nn + + +@dataclass +class ModelCase: + model_factory: type[EagerModelBase] + ops_before_transforms: dict[EdgeOpOverload, int] + ops_after_transforms: dict[EdgeOpOverload, int] + portable_ops: dict[str, int] + + +test_cases = { + "deep_autoencoder": ModelCase( + DeepAutoEncoderModel, + {_FQ.linear.default: 10, _FQ.relu.default: 9}, + {_ESP.quantized_linear_int8.default: 10}, + {}, + ), + "ds_cnn": ModelCase( + DSCNNKWSModel, + { + _FQ.convolution.default: 9, + _FQ.relu.default: 9, + _FQ.avg_pool2d.default: 1, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 5, + _ESP.quantized_depthwise_conv2d_int8.default: 4, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::avg_pool2d": 1, + "aten::constant_pad_nd": 1, + "aten::_softmax": 1, + }, + ), + "resnet8": ModelCase( + ResNet8Model, + { + _FQ.convolution.default: 9, + _FQ.relu.default: 7, + _FQ.add.default: 3, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 9, + _ESP.quantized_add_int8.default: 3, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::constant_pad_nd": 2, + "aten::mean.dim": 1, + "aten::_softmax": 1, + }, + ), + "mobilenet_v1_025": ModelCase( + MobileNetV1025Model, + { + _FQ.convolution.default: 27, + _FQ.relu.default: 27, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 14, + _ESP.quantized_depthwise_conv2d_int8.default: 13, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::constant_pad_nd": 5, + "aten::mean.dim": 1, + "aten::_softmax": 1, + }, + ), +} + + +@pytest.fixture(params=test_cases) +@torch.no_grad() +def model_case(request): + name = request.param + case = test_cases[name] + if ( + _ESP.quantized_conv2d_int8.default in case.ops_after_transforms + and get_scratch_queries(EspTarget.ESP32S3, EspKernelMode.OPTIMIZED) is None + ): + pytest.skip("Build with EXECUTORCH_BUILD_ESP_NN_PYBINDS=ON") + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + wrapper = case.model_factory() + model = wrapper.get_eager_model() + inputs = wrapper.get_example_inputs() + calibration = [wrapper.get_example_inputs() for _ in range(4)] + tester = EspTester(model, inputs, calibration).test_dialect( + case.ops_before_transforms, + case.ops_after_transforms, + qtol=1 if name == "deep_autoencoder" else 0, + atol=0 if name == "deep_autoencoder" else 1e-3, + rtol=torch.finfo(torch.float32).eps, + ) + return name, case, tester + + +def test_dialect_mlperf_tiny(model_case): + name, case, tester = model_case + edge = tester.get_artifact() + report = get_operator_library_report(edge) + counts = { + group: Counter(op["target"] for op in report[group]) + for group in ("cpu_ops", "quantization_ops", "layout_ops") + } + esp_ops = Counter( + node.target.name() + for node in edge.exported_program().graph.nodes + if node.op == "call_function" + and isinstance(node.target, EdgeOpOverload) + and node.target.namespace == "esp_nn" + ) + assert esp_ops == { + op.name(): count for op, count in case.ops_after_transforms.items() + } + assert counts["cpu_ops"] == case.portable_ops + tester.to_executorch().serialize() + program = deserialize_pte_binary(tester.get_artifact()).program + plan = program.execution_plan[0] + assert not plan.delegates + assert all(not op.name.startswith("fused_quant::") for op in plan.operators) + print(json.dumps({"model": name, "esp_nn_ops": esp_ops, **counts}, indent=2)) + + +@pytest.mark.skipif( + not os.environ.get("ESPRESSIF_BUILD_DIR"), + reason="Set ESPRESSIF_BUILD_DIR to the ESP32-S3 build directory", +) +def test_implementation_mlperf_tiny(model_case, pytestconfig): + _, _, tester = model_case + tester.test_implementation(pytestconfig.rootpath, atol=1e-6, rtol=1e-5) From dbc6f542581124d1b998933eeaca4efe9826c24d Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Fri, 20 Nov 2026 10:48:03 -0800 Subject: [PATCH 2/2] [Espressif] Restore clone in the emulator runtime selection Keep aten::clone.out available in the optimized ESP32-S3 test runtime. The existing Conv2d float_users case returns a constant bias, which requires the clone inserted by constant folding when the program is serialized. Removing the registration in the model coverage change prevented that method from loading. The manual list remains until the deferred PTE tooling can derive the union of required operators from the test programs. Shell syntax and scoped lintrunner pass. Authored with OpenAI Codex. --- .ci/scripts/test_espressif.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index 5a533e77472..6e2002c437f 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -80,7 +80,7 @@ espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \ | tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -) if [[ "${espnn_kernels}" == optimized ]]; then espnn_ops="${espnn_ops},aten::sub.out,aten::sin.out,aten::relu.out,aten::max_pool2d_with_indices.out,aten::avg_pool2d.out,aten::permute_copy.out,aten::convolution.out,aten::addmm.out,aten::mm.out,aten::mul.Scalar_out" - espnn_ops="${espnn_ops},aten::constant_pad_nd.out,aten::_softmax.out,aten::mean.out" + espnn_ops="${espnn_ops},aten::constant_pad_nd.out,aten::_softmax.out,aten::mean.out,aten::clone.out" fi toolchain_file="${IDF_PATH}/tools/cmake/toolchain-${target}.cmake"