diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index 85505f65a9b..6e2002c437f 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -80,6 +80,7 @@ espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \ | tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -) if [[ "${espnn_kernels}" == optimized ]]; then espnn_ops="${espnn_ops},aten::sub.out,aten::sin.out,aten::relu.out,aten::max_pool2d_with_indices.out,aten::avg_pool2d.out,aten::permute_copy.out,aten::convolution.out,aten::addmm.out,aten::mm.out,aten::mul.Scalar_out" + espnn_ops="${espnn_ops},aten::constant_pad_nd.out,aten::_softmax.out,aten::mean.out,aten::clone.out" fi toolchain_file="${IDF_PATH}/tools/cmake/toolchain-${target}.cmake" @@ -194,9 +195,12 @@ if [[ "${espnn_kernels}" == optimized ]]; then logged "operator-library-host-tests.log" python -m pytest -q \ backends/espressif/test/test_pass_manager.py \ backends/espressif/test/ops -k 'not implementation' + logged "model-coverage-tests.log" python -m pytest -q -s \ + backends/espressif/test/models -k 'not implementation' # Stop on failure so later cases cannot overwrite its firmware and logs. ESPRESSIF_BUILD_DIR="${build_dir}" python -m pytest \ - -x -v --tb=short backends/espressif/test/ops -k implementation \ + -x -v --tb=short backends/espressif/test/ops \ + backends/espressif/test/models -k implementation \ 2>&1 | tee "${build_dir}/operator-library-tests.log" for sdkconfig in "${build_dir}"/operator-library-*/sdkconfig; do expected_config=CONFIG_NN_OPTIMIZED diff --git a/backends/espressif/test/models/__init__.py b/backends/espressif/test/models/__init__.py new file mode 100644 index 00000000000..2e41cd717f6 --- /dev/null +++ b/backends/espressif/test/models/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/backends/espressif/test/models/test_mlperf_tiny.py b/backends/espressif/test/models/test_mlperf_tiny.py new file mode 100644 index 00000000000..383ba5a69e5 --- /dev/null +++ b/backends/espressif/test/models/test_mlperf_tiny.py @@ -0,0 +1,164 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +from collections import Counter +from dataclasses import dataclass + +import pytest +import torch +from executorch.backends.espressif.lowering import get_operator_library_report +from executorch.backends.espressif.scratch import get_scratch_queries +from executorch.backends.espressif.target import EspKernelMode, EspTarget +from executorch.backends.espressif.test.tester import EspTester +from executorch.examples.models.mlperf_tiny.deep_autoencoder import DeepAutoEncoderModel +from executorch.examples.models.mlperf_tiny.ds_cnn import DSCNNKWSModel +from executorch.examples.models.mlperf_tiny.mobilenet_v1_025 import MobileNetV1025Model +from executorch.examples.models.mlperf_tiny.resnet8 import ResNet8Model +from executorch.examples.models.model_base import EagerModelBase +from executorch.exir._serialize._program import deserialize_pte_binary +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.dialects.edge._ops import EdgeOpOverload + +_FQ = exir_ops.edge.fused_quant +_ESP = exir_ops.edge.esp_nn + + +@dataclass +class ModelCase: + model_factory: type[EagerModelBase] + ops_before_transforms: dict[EdgeOpOverload, int] + ops_after_transforms: dict[EdgeOpOverload, int] + portable_ops: dict[str, int] + + +test_cases = { + "deep_autoencoder": ModelCase( + DeepAutoEncoderModel, + {_FQ.linear.default: 10, _FQ.relu.default: 9}, + {_ESP.quantized_linear_int8.default: 10}, + {}, + ), + "ds_cnn": ModelCase( + DSCNNKWSModel, + { + _FQ.convolution.default: 9, + _FQ.relu.default: 9, + _FQ.avg_pool2d.default: 1, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 5, + _ESP.quantized_depthwise_conv2d_int8.default: 4, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::avg_pool2d": 1, + "aten::constant_pad_nd": 1, + "aten::_softmax": 1, + }, + ), + "resnet8": ModelCase( + ResNet8Model, + { + _FQ.convolution.default: 9, + _FQ.relu.default: 7, + _FQ.add.default: 3, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 9, + _ESP.quantized_add_int8.default: 3, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::constant_pad_nd": 2, + "aten::mean.dim": 1, + "aten::_softmax": 1, + }, + ), + "mobilenet_v1_025": ModelCase( + MobileNetV1025Model, + { + _FQ.convolution.default: 27, + _FQ.relu.default: 27, + _FQ.linear.default: 1, + }, + { + _ESP.quantized_conv2d_int8.default: 14, + _ESP.quantized_depthwise_conv2d_int8.default: 13, + _ESP.quantized_linear_int8.default: 1, + }, + { + "aten::constant_pad_nd": 5, + "aten::mean.dim": 1, + "aten::_softmax": 1, + }, + ), +} + + +@pytest.fixture(params=test_cases) +@torch.no_grad() +def model_case(request): + name = request.param + case = test_cases[name] + if ( + _ESP.quantized_conv2d_int8.default in case.ops_after_transforms + and get_scratch_queries(EspTarget.ESP32S3, EspKernelMode.OPTIMIZED) is None + ): + pytest.skip("Build with EXECUTORCH_BUILD_ESP_NN_PYBINDS=ON") + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + wrapper = case.model_factory() + model = wrapper.get_eager_model() + inputs = wrapper.get_example_inputs() + calibration = [wrapper.get_example_inputs() for _ in range(4)] + tester = EspTester(model, inputs, calibration).test_dialect( + case.ops_before_transforms, + case.ops_after_transforms, + qtol=1 if name == "deep_autoencoder" else 0, + atol=0 if name == "deep_autoencoder" else 1e-3, + rtol=torch.finfo(torch.float32).eps, + ) + return name, case, tester + + +def test_dialect_mlperf_tiny(model_case): + name, case, tester = model_case + edge = tester.get_artifact() + report = get_operator_library_report(edge) + counts = { + group: Counter(op["target"] for op in report[group]) + for group in ("cpu_ops", "quantization_ops", "layout_ops") + } + esp_ops = Counter( + node.target.name() + for node in edge.exported_program().graph.nodes + if node.op == "call_function" + and isinstance(node.target, EdgeOpOverload) + and node.target.namespace == "esp_nn" + ) + assert esp_ops == { + op.name(): count for op, count in case.ops_after_transforms.items() + } + assert counts["cpu_ops"] == case.portable_ops + tester.to_executorch().serialize() + program = deserialize_pte_binary(tester.get_artifact()).program + plan = program.execution_plan[0] + assert not plan.delegates + assert all(not op.name.startswith("fused_quant::") for op in plan.operators) + print(json.dumps({"model": name, "esp_nn_ops": esp_ops, **counts}, indent=2)) + + +@pytest.mark.skipif( + not os.environ.get("ESPRESSIF_BUILD_DIR"), + reason="Set ESPRESSIF_BUILD_DIR to the ESP32-S3 build directory", +) +def test_implementation_mlperf_tiny(model_case, pytestconfig): + _, _, tester = model_case + tester.test_implementation(pytestconfig.rootpath, atol=1e-6, rtol=1e-5)