Skip to content

Commit 20c0651

Browse files
authored
Arm backend: Generalize MatchArgDtypePass (#24227)
The pass is responsible for matching dtypes of operators inserted after executorch's type-promotion. It currently just handles sub and where. Additionally, it does not handle all args of sub correctly. - Expand targets of MatchArgDtypePass - Use torch.result_type to cast both operands of supported binary operations. This hanldes cases like uint8+int8 which previously incorrectly cast to int8 (uint8 value of 200 can't be represented by int8) - Move the pass later to make sure constants are folded. - Allow promotable mixed inputs in the combined TOSA profile. - Add mixed-dtype tests and update VGF matmul expectations. The removed strict xfails covered vector matmul cases that decompose to mul and sum; casting mul's integer operand now lets those cases lower successfully. Keep xfails for the backends and A16W8 cases that still need them. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Erik Lundell <erik.lundell@arm.com>
1 parent 149bba5 commit 20c0651

10 files changed

Lines changed: 122 additions & 60 deletions

File tree

‎backends/arm/_passes/arm_pass_manager.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -652,7 +652,7 @@ def _tosa_pipeline(
652652
]
653653
)
654654

655-
# Scalars -> tensors, match tensor dtypes and ranks.
655+
# Scalars -> tensors, match comparison dtypes and tensor ranks.
656656
self.add_passes(
657657
[
658658
ReplaceScalarWithTensorByProfilePass(),
@@ -661,7 +661,6 @@ def _tosa_pipeline(
661661
DecomposeLeakyReLUPass(), # Emits full_like so before ConvertFullLikeToFullPass
662662
DecomposePReLUPass(),
663663
ConvertFullLikeToFullPass(),
664-
MatchArgDtypePass(),
665664
UnsqueezeScalarPlaceholdersPass(exported_program),
666665
MatchArgRanksPass(exported_program),
667666
]
@@ -700,6 +699,7 @@ def _tosa_pipeline(
700699
RewriteAvgPool2dPass(),
701700
ComputeConstantOpsAOTPass(exported_program),
702701
FuseConstantArgsPass(exported_program),
702+
MatchArgDtypePass(),
703703
CastInt64BuffersToInt32Pass(exported_program),
704704
DecomposeSelectPass(),
705705
ConvertSqueezesToViewPass(),

‎backends/arm/_passes/fuse_constant_ops_pass.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from executorch.backends.arm._passes.fuse_equal_placeholders_pass import (
2121
FuseEqualPlaceholdersPass,
2222
)
23+
from executorch.backends.arm._passes.match_arg_dtype_pass import MatchArgDtypePass
2324
from executorch.backends.arm.tosa.dialect.shape import meta_has_shape_mark
2425
from executorch.backends.arm.tosa.mapping import TosaSpecialDtype
2526
from executorch.backends.transforms.utils import (
@@ -50,7 +51,7 @@ def f():
5051
5152
"""
5253

53-
_passes_required_after: Set[Type[ExportPass]] = set()
54+
_passes_required_after: Set[Type[ExportPass]] = {MatchArgDtypePass}
5455

5556
def __init__(self, exported_program: ExportedProgram, *args, **kwargs) -> None:
5657
super().__init__(*args, **kwargs)

‎backends/arm/_passes/match_arg_dtype_pass.py‎

Lines changed: 24 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,10 @@
77

88
import torch
99
from executorch.backends.arm._passes import ArmOpTargetedPass
10-
from executorch.backends.arm._passes.arm_pass_utils import create_node, get_node_arg
10+
from executorch.backends.arm._passes.arm_pass_utils import create_node
1111
from executorch.exir.dialects._ops import ops as exir_ops
1212
from executorch.exir.pass_base import ExportPass, PassResult
1313

14-
DTYPE_RANK = {
15-
torch.bool: 0,
16-
torch.uint8: 1,
17-
torch.int8: 2,
18-
torch.int16: 3,
19-
torch.int32: 4,
20-
torch.int64: 5,
21-
torch.float16: 6,
22-
torch.bfloat16: 7,
23-
torch.float32: 8,
24-
torch.float64: 9,
25-
}
26-
27-
28-
def get_largest_dtype(dtype_1, dtype_2):
29-
"""Find the largest dtype."""
30-
return dtype_1 if DTYPE_RANK[dtype_1] > DTYPE_RANK[dtype_2] else dtype_2
31-
3214

3315
class MatchArgDtypePass(ArmOpTargetedPass):
3416
"""Pass to match data types of non-condition input tensors.
@@ -44,26 +26,32 @@ class MatchArgDtypePass(ArmOpTargetedPass):
4426

4527
_passes_required_after: Set[Type[ExportPass]] = set()
4628

47-
target_ops = {exir_ops.edge.aten.sub.Tensor, exir_ops.edge.aten.where.self}
29+
target_ops = {
30+
exir_ops.edge.aten.maximum.default,
31+
exir_ops.edge.aten.pow.Tensor_Tensor,
32+
exir_ops.edge.aten.sub.Tensor,
33+
exir_ops.edge.aten.add.Tensor,
34+
exir_ops.edge.aten.mul.Tensor,
35+
exir_ops.edge.aten.where.self,
36+
}
4837

4938
def call(self, graph_module: torch.fx.GraphModule):
5039
modified_graph = False
5140
graph = graph_module.graph
5241

53-
for node in list(graph.nodes):
42+
for node in graph.nodes:
5443
if node.op != "call_function" or node.target not in self.target_ops:
5544
continue
56-
57-
input_ = get_node_arg(node.args, 0)
58-
other_ = get_node_arg(node.args, 1)
59-
60-
input_dtype = input_.meta["val"].dtype
61-
other_dtype = other_.meta["val"].dtype
62-
target_dtype = input_dtype
63-
if input_dtype != other_dtype:
64-
target_dtype = get_largest_dtype(input_dtype, other_dtype)
65-
66-
for arg in node.args[1:]:
45+
args_to_cast = (
46+
node.args[1:]
47+
if node.target == exir_ops.edge.aten.where.self
48+
else node.args
49+
)
50+
input_ = args_to_cast[0]
51+
other_ = args_to_cast[1]
52+
target_dtype = torch.result_type(input_.meta["val"], other_.meta["val"])
53+
54+
for arg in args_to_cast:
6755
arg_dtype = arg.meta["val"].dtype
6856

6957
if arg_dtype != target_dtype:
@@ -79,13 +67,13 @@ def call(self, graph_module: torch.fx.GraphModule):
7967
replace_node_int8.kwargs = {"dtype": torch.int8}
8068

8169
with graph.inserting_after(replace_node_int8):
82-
replace_node_fp32 = create_node(
70+
replace_node_target = create_node(
8371
graph,
8472
exir_ops.edge.dim_order_ops._to_dim_order_copy.default,
8573
)
86-
replace_node_fp32.args = (replace_node_int8,)
87-
replace_node_fp32.kwargs = {"dtype": target_dtype}
88-
node.replace_input_with(arg, replace_node_fp32)
74+
replace_node_target.args = (replace_node_int8,)
75+
replace_node_target.kwargs = {"dtype": target_dtype}
76+
node.replace_input_with(arg, replace_node_target)
8977
else:
9078
with graph.inserting_after(arg):
9179
replace_node = create_node(

‎backends/arm/operator_support/where_support.py‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -43,28 +43,35 @@ def is_node_tosa_supported(
4343
return False
4444

4545
x_dtype, y_dtype = x.meta["val"].dtype, y.meta["val"].dtype
46-
if tosa_spec.support_float():
47-
supported_float = [
48-
torch.bool,
49-
torch.float16,
50-
torch.float32,
51-
]
52-
if tosa_spec.support_extension("bf16"):
53-
supported_float.append(torch.bfloat16)
46+
supported_float = [torch.bool, torch.float16, torch.float32]
47+
if tosa_spec.support_extension("bf16"):
48+
supported_float.append(torch.bfloat16)
49+
supported_integer = (torch.bool, torch.int8, torch.int16, torch.int32)
5450

51+
if tosa_spec.support_float():
5552
if x_dtype in supported_float and y_dtype in supported_float:
5653
return True
5754

5855
if tosa_spec.support_integer():
5956
if (
60-
x_dtype in (torch.bool, torch.int8, torch.int16, torch.int32)
57+
x_dtype in supported_integer
6158
or (x_dtype == torch.float32 and x.target in DQ_OPS)
6259
) and (
63-
y_dtype in (torch.bool, torch.int8, torch.int16, torch.int32)
60+
y_dtype in supported_integer
6461
or (y_dtype == torch.float32 and y.target in DQ_OPS)
6562
):
6663
return True
6764

65+
if (
66+
tosa_spec.support_integer()
67+
and tosa_spec.support_float()
68+
and (
69+
(x_dtype in supported_integer and y_dtype in supported_float)
70+
or (x_dtype in supported_float and y_dtype in supported_integer)
71+
)
72+
):
73+
return True
74+
6875
self.reporter.report_reject(
6976
node,
7077
(

‎backends/arm/test/misc/test_tosa_operator_support.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
CheckFPComparisonInputs,
1717
CheckKnownUnsupportedTOSASemantics,
1818
)
19+
from executorch.backends.arm.operator_support.where_support import WhereSupported
1920
from executorch.backends.arm.tosa import TosaSpecification
2021
from executorch.exir.backend.utils import WhyNoPartitionReporter
2122
from executorch.exir.dialects._ops import ops as exir_ops
@@ -50,6 +51,25 @@ def _topk_node(shape=(2, 8), dtype=torch.float32, args=(2,), kwargs=None):
5051
)
5152

5253

54+
@pytest.mark.parametrize(
55+
"x_dtype,y_dtype",
56+
((torch.bfloat16, torch.bfloat16), (torch.int32, torch.bfloat16)),
57+
)
58+
@pytest.mark.parametrize("bf16_enabled", (False, True))
59+
def test_where_bf16_requires_extension(x_dtype, y_dtype, bf16_enabled) -> None:
60+
graph = torch.fx.Graph()
61+
condition = _placeholder(graph, "condition", (3,), torch.bool)
62+
x = _placeholder(graph, "x", (3,), x_dtype)
63+
y = _placeholder(graph, "y", (3,), y_dtype)
64+
node = graph.call_function(exir_ops.edge.aten.where.self, (condition, x, y))
65+
spec = "TOSA-1.0+FP+INT+bf16" if bf16_enabled else "TOSA-1.0+FP+INT"
66+
checker = WhereSupported(
67+
TosaSpecification.create_from_string(spec), WhyNoPartitionReporter()
68+
)
69+
70+
assert checker.is_node_supported({}, node) == bf16_enabled
71+
72+
5373
@pytest.mark.parametrize(
5474
"target",
5575
(

‎backends/arm/test/ops/test_matmul.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -385,14 +385,13 @@ def forward(self, x1: torch.Tensor, x2: torch.Tensor, x3: torch.Tensor):
385385
"double_input_randn_rand_4d_1d": "aten.mv.default is not supported",
386386
"single_input_randn_1d": "aten.dot.default is not supported",
387387
}
388-
vgf_quant_xfails = {
388+
vgf_a16w8_xfails = {
389389
name: reason
390390
for name, reason in xfails.items()
391391
if name
392392
not in {
393-
"double_input_randn_rand_1d_1d",
394-
"double_input_randn_rand_2d_1d",
395-
"single_input_randn_1d",
393+
"double_input_randn_rand_3d_1d",
394+
"double_input_randn_rand_4d_1d",
396395
}
397396
}
398397

@@ -496,7 +495,7 @@ def test_matmul_vgf_no_quant_fp8(test_case: test_case_t):
496495
pipeline.run()
497496

498497

499-
@common.parametrize("test_case", test_suite, xfails=vgf_quant_xfails)
498+
@common.parametrize("test_case", test_suite)
500499
@common.SkipIfNoModelConverter
501500
def test_matmul_vgf_quant(test_case: test_case_t):
502501
test_data = test_case()
@@ -511,7 +510,7 @@ def test_matmul_vgf_quant(test_case: test_case_t):
511510
pipeline.run()
512511

513512

514-
@common.parametrize("test_case", test_suite, xfails=xfails)
513+
@common.parametrize("test_case", test_suite, xfails=vgf_a16w8_xfails)
515514
@common.SkipIfNoModelConverter
516515
def test_matmul_vgf_quant_a16w8(test_case: test_case_t):
517516
test_data = test_case()

‎backends/arm/test/ops/test_maximum.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,17 @@ class Maximum(torch.nn.Module):
5454
),
5555
}
5656

57+
test_parameters_mixed = {
58+
"int32_float32": lambda: (
59+
torch.tensor([1, 2, 3], dtype=torch.int32),
60+
torch.tensor([1.5, 1.5, 3.5]),
61+
),
62+
"float32_int32": lambda: (
63+
torch.tensor([1.5, 1.5, 3.5]),
64+
torch.tensor([1, 2, 3], dtype=torch.int32),
65+
),
66+
}
67+
5768
def __init__(self):
5869
super().__init__()
5970

@@ -65,7 +76,8 @@ def forward(self, x, y):
6576
"test_data",
6677
Maximum.test_parameters
6778
| Maximum.test_parameters_fp16
68-
| Maximum.test_parameters_bf16,
79+
| Maximum.test_parameters_bf16
80+
| Maximum.test_parameters_mixed,
6981
)
7082
def test_maximum_tosa_FP(test_data: Tuple):
7183
TosaPipelineFP[test_t](

‎backends/arm/test/ops/test_pow.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,17 @@ class Pow_TensorTensor(torch.nn.Module):
7474
),
7575
}
7676

77+
test_data_mixed = {
78+
"int32_float32": lambda: (
79+
torch.tensor([1, 2, 3], dtype=torch.int32),
80+
torch.tensor([1.5, 2.0, 0.5]),
81+
),
82+
"float32_int32": lambda: (
83+
torch.tensor([1.5, 2.0, 3.5]),
84+
torch.tensor([1, 2, 3], dtype=torch.int32),
85+
),
86+
}
87+
7788
test_data_quant = {
7889
"positive_base_positive_exp": lambda: (
7990
torch.rand(2, 3, 4) * 1.5 + 0.25,
@@ -179,7 +190,9 @@ def forward(self, x: torch.Tensor):
179190

180191
@common.parametrize(
181192
"test_data",
182-
Pow_TensorTensor.test_data | Pow_TensorTensor.test_data_bf16,
193+
Pow_TensorTensor.test_data
194+
| Pow_TensorTensor.test_data_bf16
195+
| Pow_TensorTensor.test_data_mixed,
183196
x_fail,
184197
strict=False,
185198
)

‎backends/arm/test/ops/test_sub.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,16 @@
8282
torch.rand(2, 3, dtype=torch.bfloat16),
8383
),
8484
}
85+
sub2_test_data_mixed = {
86+
"int32_float32": lambda: (
87+
torch.tensor([1, 2], dtype=torch.int32),
88+
torch.tensor([0.5, 1.5]),
89+
),
90+
"float32_int32": lambda: (
91+
torch.tensor([0.5, 1.5]),
92+
torch.tensor([1, 2], dtype=torch.int32),
93+
),
94+
}
8595

8696
# Sub and tan - the tan has a really steep curve just before Pi/2 and a point of discontinuity at Pi/2
8797
# so if the sub result is inaccurate, the error will be amplified by the tan
@@ -142,7 +152,8 @@ def test_sub_tensor_tosa_FP(test_data):
142152

143153

144154
@common.parametrize(
145-
"test_data", sub2_test_data | sub2_test_data_fp16 | sub2_test_data_bf16
155+
"test_data",
156+
sub2_test_data | sub2_test_data_fp16 | sub2_test_data_bf16 | sub2_test_data_mixed,
146157
)
147158
def test_sub_tensor_tosa_FP_2(test_data: Tuple[torch.Tensor, torch.Tensor]):
148159
"""Test Two-Operand Subtraction (TOSA FP)"""
@@ -286,7 +297,8 @@ def test_sub_tensor_vgf_no_quant(test_data: Tuple[torch.Tensor]):
286297

287298

288299
@common.parametrize(
289-
"test_data", sub2_test_data | sub2_test_data_bf16 | sub2_test_data_fp16
300+
"test_data",
301+
sub2_test_data | sub2_test_data_bf16 | sub2_test_data_fp16 | sub2_test_data_mixed,
290302
)
291303
@common.SkipIfNoModelConverter
292304
def test_sub_tensor_vgf_no_quant_2(test_data: Tuple[torch.Tensor, torch.Tensor]):

‎backends/arm/test/ops/test_where.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ def scalar_condition(input: torch.Tensor):
131131
(torch.float32, torch.bool),
132132
tensor_condition,
133133
)
134+
int32_float32_tensor_cond = Where(
135+
1,
136+
(torch.int32, torch.float32),
137+
lambda input: input.to(torch.float32) > 0,
138+
)
134139

135140
# Scalar tests
136141
two_dim_scalar_cond = Where(
@@ -183,6 +188,9 @@ def scalar_condition(input: torch.Tensor):
183188
"float16_tensor_cond": lambda: float16_tensor_cond,
184189
"const_float32": lambda: const_float32,
185190
}
191+
test_modules_mixed = {
192+
"int32_float32_tensor_cond": lambda: int32_float32_tensor_cond,
193+
}
186194

187195
test_modules_FP_bf16 = {
188196
"bf16_tensor_cond": lambda: bf16_tensor_cond,
@@ -292,7 +300,9 @@ def test_where_self_u85_INT(test_module):
292300
pipeline.run()
293301

294302

295-
@common.parametrize("test_module", test_modules_FP | test_modules_FP_bf16)
303+
@common.parametrize(
304+
"test_module", test_modules_FP | test_modules_FP_bf16 | test_modules_mixed
305+
)
296306
@common.SkipIfNoModelConverter
297307
def test_where_self_vgf_no_quant(test_module):
298308
module = test_module()

0 commit comments

Comments
 (0)