NXP backend: Use a quantized dataset for testing operators where a single-bit error is expected.

backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py

@@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
             mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     def test__basic_nsys_inference_qat(self, mocker, request):
@@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request):
             mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -84,7 +90,7 @@ def test__basic_nsys_inference_qat(self, mocker, request):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
             use_qat=True,
         )
 
@@ -117,6 +123,9 @@ def test__broadcast(self, mocker, request, input_spec):
             mocker, expected_delegated_ops={AddTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -126,7 +135,7 @@ def test__broadcast(self, mocker, request, input_spec):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     @pytest.mark.parametrize(
@@ -230,6 +239,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
             expected_non_delegated_ops={},
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -239,5 +251,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )

backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py

@@ -204,6 +204,9 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
             expected_non_delegated_ops={},
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -213,5 +216,5 @@ def test__broadcast__channels_first_input(self, mocker, request, input_spec):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )

backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py

@@ -10,14 +10,13 @@
 import pytest
 import torch
 
-from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner
 from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator
 from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier
 from executorch.backends.nxp.tests.model_output_comparator import (
     AllCloseOutputComparator,
 )
 from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
-from executorch.backends.nxp.tests.ops_aliases import DequantizePerTensor, Sigmoid
+from executorch.backends.nxp.tests.ops_aliases import Sigmoid
 from torch import nn
 from executorch.backends.nxp.tests.use_qat import *  # noqa F403
 
@@ -30,9 +29,7 @@ def reseed_model_per_test_run():
 
 class TestSigmoid:
     # noinspection PyMethodMayBeStatic
-    def assert_delegated(
-        self, model, input_shape, mocker, request, use_qat=False, atol=None
-    ):
+    def assert_delegated(self, model, input_shape, mocker, request, use_qat=False):
         graph_verifier = DetailedGraphVerifier(
             mocker,
             expected_delegated_ops={Sigmoid: 1},
@@ -42,17 +39,19 @@ def assert_delegated(
         # Create a RandomDatasetCreator that covers also negative numbers to properly test the operator.
         dataset_creator = RandomDatasetCreator(low=-2, high=2)
 
-        kwargs = {"atol": atol} if atol is not None else {}
-        output_comparator = AllCloseOutputComparator(**kwargs)
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
+        comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
             model,
             input_shape,
             graph_verifier,
             request,
             dataset_creator,
-            output_comparator,
+            comparator,
             use_qat=use_qat,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     def test__basic_nsys_inference__qat(self, mocker, request, use_qat):
@@ -74,14 +73,4 @@ def test__basic_nsys_inference__qat(self, mocker, request, use_qat):
     def test__input_shapes(self, mocker, request, input_shape):
         model = nn.Sigmoid()
 
-        output_scale = 1.0 / 256.0
-        lowering_spy = mocker.spy(NeutronPartitioner, "partition")
-        self.assert_delegated(
-            model, input_shape, mocker, request, atol=output_scale
-        )  # Allow single bit error.
-
-        # Verify that the `atol` is indeed equal to the output scale.
-        # In the near future, we would like to add support for testing with int8 IO, where this check will be trivial.
-        nodes = list(lowering_spy.spy_return.tagged_exported_program.graph.nodes)
-        assert nodes[-2].target == DequantizePerTensor
-        assert nodes[-2].args[1] == output_scale
+        self.assert_delegated(model, input_shape, mocker, request)

backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py

@@ -56,6 +56,9 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
             mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -65,7 +68,7 @@ def test__basic_nsys_inference(self, mocker, request, x_input_shape):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     def test__basic_nsys_inference_qat(self, mocker, request):
@@ -75,6 +78,9 @@ def test__basic_nsys_inference_qat(self, mocker, request):
             mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -85,7 +91,7 @@ def test__basic_nsys_inference_qat(self, mocker, request):
             dataset_creator,
             comparator,
             use_qat=True,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     @pytest.mark.parametrize(
@@ -113,6 +119,9 @@ def test__broadcast(self, mocker, request, input_spec):
             mocker, expected_delegated_ops={SubTensor: 1}, expected_non_delegated_ops={}
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -122,7 +131,7 @@ def test__broadcast(self, mocker, request, input_spec):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     @pytest.mark.parametrize(
@@ -225,6 +234,9 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec):
             expected_non_delegated_ops={},
         )
         dataset_creator = RandomDatasetCreator(low=-1.0, high=1.0)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -234,5 +246,5 @@ def test__broadcast_channels_first_input(self, mocker, request, input_spec):
             request,
             dataset_creator,
             comparator,
-            remove_quant_io_ops=True,
+            remove_quant_io_ops=remove_quant_io_ops,
         )

backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py

@@ -51,7 +51,6 @@ def forward(self, x):
 
 
 class TestUpsampleBilinear2D:
-    # TODO Use quantized dataset and `atol=1` in the tests.
 
     # noinspection PyMethodMayBeStatic
     def assert_delegated(
@@ -61,7 +60,6 @@ def assert_delegated(
         mocker,
         request,
         use_qat=False,
-        atol=None,
         expected_delegated_ops=None,
     ):
         if expected_delegated_ops is None:
@@ -76,17 +74,19 @@ def assert_delegated(
         # Cover also negative values to thoroughly test the operator.
         dataset_creator = RandomDatasetCreator(low=-2, high=2)
 
-        kwargs = {"atol": atol} if atol is not None else {}
-        output_comparator = AllCloseOutputComparator(**kwargs)
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
+        comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
             model,
             input_shape,
             graph_verifier,
             request,
             dataset_creator,
-            output_comparator,
+            comparator,
             use_qat=use_qat,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     # noinspection PyMethodMayBeStatic
@@ -103,20 +103,14 @@ def test__qat__align_corners(self, mocker, request, use_qat):
         input_shape = (1, 2, 3, 4)
         output_size = (5, 7)
         model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
-        atol = 0.015  # ~= output scale -> single bit error.
-        self.assert_delegated(
-            model, input_shape, mocker, request, use_qat=use_qat, atol=atol
-        )
+        self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat)
 
     def test__qat__not_align_corners(self, mocker, request, use_qat):
         align_corners = False
         input_shape = (1, 2, 3, 4)
         output_size = (6, 8)
         model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
-        atol = 0.015  # ~= output scale -> single bit error.
-        self.assert_delegated(
-            model, input_shape, mocker, request, use_qat=use_qat, atol=atol
-        )
+        self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat)
 
     @pytest.mark.parametrize(
         "input_shape, output_size",
@@ -136,8 +130,7 @@ def test__not_align_corners__output_size(
     ):
         align_corners = False
         model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
-        atol = 0.016  # ~= output scale -> single bit error.
-        self.assert_delegated(model, input_shape, mocker, request, atol=atol)
+        self.assert_delegated(model, input_shape, mocker, request)
 
     def test__not_align_corners__output_size__unsupported(self):
         align_corners = False
@@ -162,8 +155,7 @@ def test__not_align_corners__output_size__unsupported(self):
     def test__not_align_corners__scales(self, mocker, request, input_shape, scale):
         align_corners = False
         model = UpsampleBilinearModule(scale=scale, align_corners=align_corners)
-        atol = 0.016  # ~= output scale -> single bit error.
-        self.assert_delegated(model, input_shape, mocker, request, atol=atol)
+        self.assert_delegated(model, input_shape, mocker, request)
 
     def test__not_align_corners__scales__unsupported(self):
         align_corners = False
@@ -196,8 +188,7 @@ def test__align_corners__output_size(
     ):
         align_corners = True
         model = UpsampleBilinearModule(size=output_size, align_corners=align_corners)
-        atol = 0.016  # ~= output scale -> single bit error.
-        self.assert_delegated(model, input_shape, mocker, request, atol=atol)
+        self.assert_delegated(model, input_shape, mocker, request)
 
     def test__align_corners__output_size__unsupported(self):
         align_corners = True
@@ -253,8 +244,7 @@ def test__align_corners__output_size__input_size_equal_to_one(self):
     def test__align_corners__scales(self, mocker, request, input_shape, scale):
         align_corners = True
         model = UpsampleBilinearModule(scale=scale, align_corners=align_corners)
-        atol = 0.016  # ~= output scale -> single bit error.
-        self.assert_delegated(model, input_shape, mocker, request, atol=atol)
+        self.assert_delegated(model, input_shape, mocker, request)
 
     def test__align_corners__scales__unsupported(self):
         align_corners = True

backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py

@@ -177,6 +177,9 @@ def assert_delegated_and_correct(
         )
 
         dataset = RandomDatasetCreator(low=-128, high=128)
+
+        # Quantize the dataset and allow a single bit error.
+        remove_quant_io_ops = True
         comparator = AllCloseOutputComparator(atol=1)
 
         lower_run_compare(
@@ -188,6 +191,7 @@ def assert_delegated_and_correct(
             comparator,
             mocker=mocker,
             use_qat=use_qat,
+            remove_quant_io_ops=remove_quant_io_ops,
         )
 
     @staticmethod