From 71be8b9ea82d9a8cb8c19687e21a1435b1625c69 Mon Sep 17 00:00:00 2001 From: Martin Pavella Date: Thu, 2 Jul 2026 15:44:54 +0200 Subject: [PATCH] NXP backend: Use a quantized dataset for testing operators where a single-bit error is expected. --- .../test_add_tensor_converter.py | 20 +++++++++--- .../test_mul_tensor_converter.py | 5 ++- .../node_converter/test_sigmoid_converter.py | 27 +++++----------- .../test_sub_tensor_converter.py | 20 +++++++++--- .../test_upsample_bilinear2d.py | 32 +++++++------------ .../test_view_copy_converter.py | 4 +++ 6 files changed, 59 insertions(+), 49 deletions(-) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py index ce588192471..2ec1c50cc01 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py @@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape): mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) def test__basic_nsys_inference_qat(self, mocker, request): @@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request): mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -84,7 +90,7 @@ def test__basic_nsys_inference_qat(self, mocker, request): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, use_qat=True, ) @@ -117,6 +123,9 @@ def test__broadcast(self, mocker, request, input_spec): mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -126,7 +135,7 @@ def test__broadcast(self, mocker, request, input_spec): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) @pytest.mark.parametrize( @@ -230,6 +239,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec): expected_non_delegated_ops={}, ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -239,5 +251,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py index bafe110119b..5e807a7731f 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py @@ -204,6 +204,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec): expected_non_delegated_ops={}, ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -213,5 +216,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py index bdd41d1eab0..5c4e4f4f007 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py @@ -10,14 +10,13 @@ import pytest import torch -from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.model_output_comparator import ( AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import DequantizePerTensor, Sigmoid +from executorch.backends.nxp.tests.ops_aliases import Sigmoid from torch import nn from executorch.backends.nxp.tests.use_qat import * # noqa F403 @@ -30,9 +29,7 @@ def reseed_model_per_test_run(): class TestSigmoid: # noinspection PyMethodMayBeStatic - def assert_delegated( - self, model, input_shape, mocker, request, use_qat=False, atol=None - ): + def assert_delegated(self, model, input_shape, mocker, request, use_qat=False): graph_verifier = DetailedGraphVerifier( mocker, expected_delegated_ops={Sigmoid: 1}, @@ -42,8 +39,9 @@ def assert_delegated( # Create a RandomDatasetCreator that covers also negative numbers to properly test the operator. dataset_creator = RandomDatasetCreator(low=-2, high=2) - kwargs = {"atol": atol} if atol is not None else {} - output_comparator = AllCloseOutputComparator(**kwargs) + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True + comparator = AllCloseOutputComparator(atol=1) lower_run_compare( model, @@ -51,8 +49,9 @@ def assert_delegated( graph_verifier, request, dataset_creator, - output_comparator, + comparator, use_qat=use_qat, + remove_quant_io_ops=remove_quant_io_ops, ) def test__basic_nsys_inference__qat(self, mocker, request, use_qat): @@ -74,14 +73,4 @@ def test__basic_nsys_inference__qat(self, mocker, request, use_qat): def test__input_shapes(self, mocker, request, input_shape): model = nn.Sigmoid() - output_scale = 1.0 / 256.0 - lowering_spy = mocker.spy(NeutronPartitioner, "partition") - self.assert_delegated( - model, input_shape, mocker, request, atol=output_scale - ) # Allow single bit error. - - # Verify that the `atol` is indeed equal to the output scale. - # In the near future, we would like to add support for testing with int8 IO, where this check will be trivial. - nodes = list(lowering_spy.spy_return.tagged_exported_program.graph.nodes) - assert nodes[-2].target == DequantizePerTensor - assert nodes[-2].args[1] == output_scale + self.assert_delegated(model, input_shape, mocker, request) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py index 17de2f190e1..7a8c1a33d6f 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py @@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape): mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) def test__basic_nsys_inference_qat(self, mocker, request): @@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request): mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -85,7 +91,7 @@ def test__basic_nsys_inference_qat(self, mocker, request): dataset_creator, comparator, use_qat=True, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) @pytest.mark.parametrize( @@ -113,6 +119,9 @@ def test__broadcast(self, mocker, request, input_spec): mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={} ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -122,7 +131,7 @@ def test__broadcast(self, mocker, request, input_spec): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) @pytest.mark.parametrize( @@ -225,6 +234,9 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec): expected_non_delegated_ops={}, ) dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -234,5 +246,5 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec): request, dataset_creator, comparator, - remove_quant_io_ops=True, + remove_quant_io_ops=remove_quant_io_ops, ) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py index f9b2269751f..949f193b267 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py @@ -51,7 +51,6 @@ def forward(self, x): class TestUpsampleBilinear2D: - # TODO Use quantized dataset and `atol=1` in the tests. # noinspection PyMethodMayBeStatic def assert_delegated( @@ -61,7 +60,6 @@ def assert_delegated( mocker, request, use_qat=False, - atol=None, expected_delegated_ops=None, ): if expected_delegated_ops is None: @@ -76,8 +74,9 @@ def assert_delegated( # Cover also negative values to thoroughly test the operator. dataset_creator = RandomDatasetCreator(low=-2, high=2) - kwargs = {"atol": atol} if atol is not None else {} - output_comparator = AllCloseOutputComparator(**kwargs) + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True + comparator = AllCloseOutputComparator(atol=1) lower_run_compare( model, @@ -85,8 +84,9 @@ def assert_delegated( graph_verifier, request, dataset_creator, - output_comparator, + comparator, use_qat=use_qat, + remove_quant_io_ops=remove_quant_io_ops, ) # noinspection PyMethodMayBeStatic @@ -103,20 +103,14 @@ def test__qat__align_corners(self, mocker, request, use_qat): input_shape = (1, 2, 3, 4) output_size = (5, 7) model = UpsampleBilinearModule(size=output_size, align_corners=align_corners) - atol = 0.015 # ~= output scale -> single bit error. - self.assert_delegated( - model, input_shape, mocker, request, use_qat=use_qat, atol=atol - ) + self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat) def test__qat__not_align_corners(self, mocker, request, use_qat): align_corners = False input_shape = (1, 2, 3, 4) output_size = (6, 8) model = UpsampleBilinearModule(size=output_size, align_corners=align_corners) - atol = 0.015 # ~= output scale -> single bit error. - self.assert_delegated( - model, input_shape, mocker, request, use_qat=use_qat, atol=atol - ) + self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat) @pytest.mark.parametrize( "input_shape, output_size", @@ -136,8 +130,7 @@ def test__not_align_corners__output_size( ): align_corners = False model = UpsampleBilinearModule(size=output_size, align_corners=align_corners) - atol = 0.016 # ~= output scale -> single bit error. - self.assert_delegated(model, input_shape, mocker, request, atol=atol) + self.assert_delegated(model, input_shape, mocker, request) def test__not_align_corners__output_size__unsupported(self): align_corners = False @@ -162,8 +155,7 @@ def test__not_align_corners__output_size__unsupported(self): def test__not_align_corners__scales(self, mocker, request, input_shape, scale): align_corners = False model = UpsampleBilinearModule(scale=scale, align_corners=align_corners) - atol = 0.016 # ~= output scale -> single bit error. - self.assert_delegated(model, input_shape, mocker, request, atol=atol) + self.assert_delegated(model, input_shape, mocker, request) def test__not_align_corners__scales__unsupported(self): align_corners = False @@ -196,8 +188,7 @@ def test__align_corners__output_size( ): align_corners = True model = UpsampleBilinearModule(size=output_size, align_corners=align_corners) - atol = 0.016 # ~= output scale -> single bit error. - self.assert_delegated(model, input_shape, mocker, request, atol=atol) + self.assert_delegated(model, input_shape, mocker, request) def test__align_corners__output_size__unsupported(self): align_corners = True @@ -253,8 +244,7 @@ def test__align_corners__output_size__input_size_equal_to_one(self): def test__align_corners__scales(self, mocker, request, input_shape, scale): align_corners = True model = UpsampleBilinearModule(scale=scale, align_corners=align_corners) - atol = 0.016 # ~= output scale -> single bit error. - self.assert_delegated(model, input_shape, mocker, request, atol=atol) + self.assert_delegated(model, input_shape, mocker, request) def test__align_corners__scales__unsupported(self): align_corners = True diff --git a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py index 2cb8ad05006..2a2d270e30a 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py @@ -177,6 +177,9 @@ def assert_delegated_and_correct( ) dataset = RandomDatasetCreator(low=-128, high=128) + + # Quantize the dataset and allow a single bit error. + remove_quant_io_ops = True comparator = AllCloseOutputComparator(atol=1) lower_run_compare( @@ -188,6 +191,7 @@ def assert_delegated_and_correct( comparator, mocker=mocker, use_qat=use_qat, + remove_quant_io_ops=remove_quant_io_ops, ) @staticmethod