Skip to content

Commit 9e3ec43

Browse files
authored
[XNNPACK] Lower fp32 <-> fp16 casts as XNNConvert (#24277)
### Summary Split out of #24112 as asked in review. In the fp16 MI-GAN inpainting model the fp32 I/O casts were the only ops left outside the XNNPACK delegate. `ToDimOrderCopyConfig` now accepts a `_to_dim_order_copy` that only changes dtype between fp32 and fp16 and keeps the input's dim order. Other dtype conversions, and a dtype cast combined with a dim order change, are still rejected. The `_to_copy` visitor (`ToCopyVisitor`, previously `ConvertMemoryFormat`) serializes such a cast as `XNNConvert`. `ChannelsLastTaggedReshapePass` treats the cast as layout-agnostic, so it inherits its input's layout. `RemoveRedundantCopyPass` no longer removes the cast as a "same-layout copy" and never pairs it with a layout copy as the first half of an NCHW/NHWC round trip. It still removes a same-layout copy that follows the cast. The shared check lives in `backends/xnnpack/utils/utils.py` as `is_dtype_cast`. No runtime change is needed: `XNNConvert` already exists in the runtime. On a Galaxy S23 the casts as `Convert` nodes are within noise against the unpartitioned portable casts. The benefit is that 100% of the forward now runs inside the delegate, with no partition boundary at the graph input and output. Review order: `generic_node_configs.py`, `op_to_copy.py`, then `channels_last_tagged_reshape_pass.py`, `remove_redundant_copy_pass.py` and `utils.py`, then the tests. ### Test plan - `backends/xnnpack/test/ops/test_to_copy_dtype.py`: fp32 I/O around an fp16 conv and a cast-only graph are fully delegated with no cast left outside and match eager. An int cast and a dtype plus dim order cast (`.to(torch.float16, memory_format=torch.contiguous_format)` on a channels-last input) are not partitioned, while the layout-preserving `.to(torch.float16)` on the same input is lowered. - `backends/xnnpack/test/passes/test_remove_redundant_copy_pass.py`: fp32 I/O around an fp16 conv and a bare fp16/fp32 round trip keep their casts. A channels-last copy after an fp16 cast is still removed. ```bash python -m pytest backends/xnnpack/test/ops/test_to_copy_dtype.py \ backends/xnnpack/test/passes/test_remove_redundant_copy_pass.py # 12 passed (6 to_copy dtype, 6 remove-redundant-copy incl. 3 existing) python -m pytest backends/xnnpack/test/ops/test_to_copy_dtype.py \ backends/xnnpack/test/passes/test_remove_redundant_copy_pass.py \ backends/xnnpack/test/passes/test_channels_last_tagged_reshape.py \ backends/xnnpack/test/ops/test_to_copy.py backends/xnnpack/test/ops/test_conv2d.py # 78 passed ``` Run on macOS (Apple Silicon, Python 3.13) against the prebuilt pybindings. `lintrunner` on the changed files reports no issues. cc @nil-is-all @GregoryComer @digantdesai @cbilgin @JakeStevens
1 parent a0a365b commit 9e3ec43

7 files changed

Lines changed: 232 additions & 9 deletions

File tree

‎backends/xnnpack/_passes/channels_last_tagged_reshape_pass.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
is_tagged_as_implicit_q_dq,
2020
tag_as_implicit_q_dq,
2121
)
22-
from executorch.backends.xnnpack.utils.utils import is_param_node
22+
from executorch.backends.xnnpack.utils.utils import is_dtype_cast, is_param_node
2323
from executorch.exir.dialects._ops import ops as exir_ops
2424
from executorch.exir.pass_base import PassResult
2525

@@ -562,7 +562,10 @@ def call(self, graph_module: torch.fx.GraphModule): # noqa: C901
562562
# The node requires nchw inputs
563563
for input_node in node.all_input_nodes:
564564
self.input_to_nchw(graph_module, input_node, node)
565-
elif node.target == exir_ops.edge.aten._to_copy.default:
565+
elif (
566+
node.target == exir_ops.edge.aten._to_copy.default
567+
and not is_dtype_cast(node)
568+
):
566569
self.tag_node(node)
567570
else:
568571
# The node can have inputs in any format (but all must be the

‎backends/xnnpack/_passes/remove_redundant_copy_pass.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
)
1111
from executorch.backends.xnnpack._passes.xnnpack_pass import XNNPACKPass
1212
from executorch.backends.xnnpack.utils.quant_utils import is_dequant, is_quant
13+
from executorch.backends.xnnpack.utils.utils import is_dtype_cast
1314
from executorch.exir.dialects._ops import ops as exir_ops
1415
from executorch.exir.pass_base import PassResult
1516

@@ -25,9 +26,15 @@ def _try_remove_regular_redundant_to_copy(self, node, graph):
2526
"""
2627
input_node = node.args[0]
2728

29+
# A dtype cast is not a layout copy: it is serialized as a convert node
30+
# and must stay, whatever the tags around it say.
31+
if is_dtype_cast(node):
32+
return False
33+
2834
# Check if input is a to_copy with opposite memory format
2935
if (
3036
input_node.target == exir_ops.edge.aten._to_copy.default
37+
and not is_dtype_cast(input_node)
3138
and ChannelsLastTaggedReshapePass.is_nchw_node(input_node)
3239
!= ChannelsLastTaggedReshapePass.is_nchw_node(node)
3340
and len(input_node.users) == 1

‎backends/xnnpack/operators/op_to_copy.py‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,25 @@
1616
from executorch.backends.xnnpack.operators.quant_params import QuantParams
1717

1818
from executorch.backends.xnnpack.serialization.xnnpack_graph_schema import (
19+
XNNConvert,
1920
XNNGraph,
2021
XNNStaticTranspose,
2122
XNode,
2223
)
2324
from executorch.backends.xnnpack.utils.utils import (
2425
check_or_raise,
2526
get_input_node,
27+
is_dtype_cast,
2628
PERM_NCHW_TO_NHWC,
2729
PERM_NHWC_TO_NCHW,
2830
)
2931

3032

3133
@register_node_visitor
32-
class ConvertMemoryFormat(NodeVisitor):
34+
class ToCopyVisitor(NodeVisitor):
35+
"""Serializes _to_copy as a static transpose when it changes the layout
36+
and as a convert node when it changes the dtype."""
37+
3338
target = "aten._to_copy.default"
3439

3540
def __init__(self, *args) -> None:
@@ -42,6 +47,23 @@ def define_node(
4247
vals_to_ids: Dict[torch.fx.Node, int],
4348
debug_handle: int,
4449
) -> None:
50+
input_node = get_input_node(node, 0)
51+
if is_dtype_cast(node):
52+
# The layout follows the tags on the two tensors.
53+
self.define_tensor(input_node, xnn_graph, vals_to_ids)
54+
self.define_tensor(node, xnn_graph, vals_to_ids)
55+
xnn_graph.xnodes.append(
56+
XNode(
57+
xnode_union=XNNConvert(
58+
input_id=vals_to_ids[input_node],
59+
output_id=vals_to_ids[node],
60+
flags=0,
61+
),
62+
debug_handle=debug_handle,
63+
)
64+
)
65+
return
66+
4567
memory_format_target = node.kwargs.get("memory_format", torch.contiguous_format)
4668
to_channels_last = bool(memory_format_target == torch.channels_last)
4769
to_contiguous = bool(memory_format_target == torch.contiguous_format)
@@ -50,7 +72,6 @@ def define_node(
5072
"Unsupported Memory Format for XNNPACK",
5173
)
5274

53-
input_node = get_input_node(node, 0)
5475
input_quant_params = QuantParams.from_inputs(input_node, self._exported_program)
5576
output_quant_params = QuantParams.from_outputs(node)
5677

‎backends/xnnpack/partition/config/generic_node_configs.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -645,13 +645,22 @@ def check_constraints(self, node: torch.fx.Node, ep: ExportedProgram) -> bool:
645645
input_dtype = input_node.meta["val"].dtype
646646
output_dtype = node.meta["val"].dtype
647647

648-
# Return False if doing dtype conversion
649648
if input_dtype != output_dtype:
650-
why(
651-
node,
652-
reason=f"dtype conversion from {input_dtype} to {output_dtype} is not supported",
649+
# A dtype cast is lowered to an XNNPACK convert node, as long as
650+
# the op does not change the dim order at the same time.
651+
supported_cast = {input_dtype, output_dtype} <= {
652+
torch.float32,
653+
torch.float16,
654+
}
655+
same_dim_order = list(input_node.meta["val"].dim_order()) == list(
656+
node.meta["val"].dim_order()
653657
)
654-
return False
658+
if not (supported_cast and same_dim_order):
659+
why(
660+
node,
661+
reason=f"dtype conversion from {input_dtype} to {output_dtype} is not supported",
662+
)
663+
return False
655664

656665
return True
657666

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# Copyright (c) 2026 Software Mansion
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import unittest
8+
9+
import torch
10+
from executorch.backends.xnnpack.test.tester import Tester
11+
12+
13+
class TestToCopyDtype(unittest.TestCase):
14+
"""fp32 <-> fp16 casts are lowered to XNNPACK convert nodes."""
15+
16+
def setUp(self):
17+
torch._dynamo.reset()
18+
19+
class Fp16Inner(torch.nn.Module):
20+
"""fp32 I/O contract around an fp16 model: the common deployment shape."""
21+
22+
def __init__(self):
23+
super().__init__()
24+
self.conv = torch.nn.Conv2d(3, 8, 3, padding=1).half()
25+
26+
def forward(self, x):
27+
return self.conv(x.to(torch.float16)).to(torch.float32)
28+
29+
class CastOnly(torch.nn.Module):
30+
def forward(self, x):
31+
return (x.to(torch.float16) * 2.0).to(torch.float32)
32+
33+
cast_ops = [
34+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default",
35+
"executorch_exir_dialects_edge__ops_aten__to_copy_default",
36+
]
37+
38+
def _run(self, module, inputs):
39+
(
40+
Tester(module.eval(), inputs)
41+
.export()
42+
.to_edge_transform_and_lower()
43+
.check_not(self.cast_ops)
44+
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
45+
.to_executorch()
46+
.serialize()
47+
.run_method_and_compare_outputs(atol=2e-2, rtol=2e-2)
48+
)
49+
50+
def test_fp32_io_around_fp16_conv(self):
51+
self._run(self.Fp16Inner(), (torch.randn(1, 3, 8, 8),))
52+
53+
def test_cast_only(self):
54+
self._run(self.CastOnly(), (torch.randn(2, 5),))
55+
56+
def test_int_cast_not_partitioned(self):
57+
class ToInt(torch.nn.Module):
58+
def forward(self, x):
59+
return x.to(torch.int32)
60+
61+
(
62+
Tester(ToInt().eval(), (torch.randn(2, 5),))
63+
.export()
64+
.to_edge_transform_and_lower()
65+
.check_not(["torch.ops.higher_order.executorch_call_delegate"])
66+
)
67+
68+
def test_dtype_cast_with_dim_order_change_not_partitioned(self):
69+
class ToHalfChannelsLast(torch.nn.Module):
70+
def forward(self, x):
71+
return x.to(torch.float16, memory_format=torch.channels_last)
72+
73+
(
74+
Tester(ToHalfChannelsLast().eval(), (torch.randn(1, 3, 4, 4),))
75+
.export()
76+
.to_edge_transform_and_lower()
77+
.check_not(["torch.ops.higher_order.executorch_call_delegate"])
78+
.check_count({self.cast_ops[0]: 1})
79+
)
80+
81+
def test_dtype_cast_to_contiguous_from_channels_last_not_partitioned(self):
82+
class ChannelsLastToHalfContiguous(torch.nn.Module):
83+
def forward(self, x):
84+
x = x.to(memory_format=torch.channels_last)
85+
return x.to(torch.float16, memory_format=torch.contiguous_format)
86+
87+
(
88+
Tester(ChannelsLastToHalfContiguous().eval(), (torch.randn(1, 3, 4, 4),))
89+
.export()
90+
.to_edge_transform_and_lower()
91+
.check_count({self.cast_ops[0]: 1})
92+
.to_executorch()
93+
.serialize()
94+
.run_method_and_compare_outputs(atol=2e-2, rtol=2e-2)
95+
)
96+
97+
def test_dtype_cast_preserving_channels_last_partitioned(self):
98+
class ChannelsLastToHalf(torch.nn.Module):
99+
def forward(self, x):
100+
return x.to(memory_format=torch.channels_last).to(torch.float16)
101+
102+
self._run(ChannelsLastToHalf(), (torch.randn(1, 3, 4, 4),))

‎backends/xnnpack/test/passes/test_remove_redundant_copy_pass.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,3 +180,72 @@ def test_complex_quantized_redundant_copy_removal(self):
180180
)
181181
.run_method_and_compare_outputs(qtol=1)
182182
)
183+
184+
class FloatCastAroundConv(torch.nn.Module):
185+
"""fp32 I/O around an fp16 conv: the casts share the layout tags of the
186+
copies next to them and must not be removed as same-layout copies."""
187+
188+
def __init__(self):
189+
super().__init__()
190+
self.conv = torch.nn.Conv2d(3, 3, 3).half()
191+
192+
def forward(self, x):
193+
return self.conv(x.to(torch.float16)).to(torch.float32)
194+
195+
class FloatCastRoundTrip(torch.nn.Module):
196+
def forward(self, x):
197+
return x.to(torch.float16).to(torch.float32)
198+
199+
class ChannelsLastAfterFloatCast(torch.nn.Module):
200+
"""conv -> fp16 cast -> channels_last: the layout copy behind the cast
201+
is NHWC -> NHWC and must still be removed, or the delegate transposes
202+
the already-NHWC data a second time."""
203+
204+
def __init__(self):
205+
super().__init__()
206+
self.conv = torch.nn.Conv2d(3, 8, 3, padding=1)
207+
208+
def forward(self, x):
209+
return self.conv(x).to(torch.float16).to(memory_format=torch.channels_last)
210+
211+
def test_float_cast_around_conv_kept(self):
212+
(
213+
Tester(self.FloatCastAroundConv(), (torch.randn(1, 3, 8, 8),))
214+
.export()
215+
.to_edge()
216+
.run_passes(self.PassStage)
217+
.check_count(
218+
{
219+
"executorch_exir_dialects_edge__ops_aten__to_copy_default": 4,
220+
}
221+
)
222+
.run_method_and_compare_outputs(atol=2e-2, rtol=2e-2)
223+
)
224+
225+
def test_channels_last_copy_after_float_cast_removed(self):
226+
(
227+
Tester(self.ChannelsLastAfterFloatCast(), (torch.randn(1, 3, 8, 8),))
228+
.export()
229+
.to_edge()
230+
.run_passes(self.PassStage)
231+
.check_count(
232+
{
233+
"executorch_exir_dialects_edge__ops_aten__to_copy_default": 2,
234+
}
235+
)
236+
.run_method_and_compare_outputs(atol=2e-2, rtol=2e-2)
237+
)
238+
239+
def test_float_cast_round_trip_kept(self):
240+
(
241+
Tester(self.FloatCastRoundTrip(), (torch.randn(1, 3, 8, 8),))
242+
.export()
243+
.to_edge()
244+
.run_passes(self.PassStage)
245+
.check_count(
246+
{
247+
"executorch_exir_dialects_edge__ops_aten__to_copy_default": 2,
248+
}
249+
)
250+
.run_method_and_compare_outputs(atol=2e-2, rtol=2e-2)
251+
)

‎backends/xnnpack/utils/utils.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,18 @@ def get_input_node(node: torch.fx.Node, input_index: int) -> torch.fx.Node:
5858
return cast(torch.fx.Node, node.args[input_index])
5959

6060

61+
def is_dtype_cast(node: torch.fx.Node) -> bool:
62+
"""A _to_copy that changes dtype keeps the layout of its input and is
63+
serialized as a convert node, not a transpose."""
64+
src = node.args[0]
65+
return (
66+
isinstance(src, torch.fx.Node)
67+
and "val" in src.meta
68+
and "val" in node.meta
69+
and node.meta["val"].dtype != src.meta["val"].dtype
70+
)
71+
72+
6173
def normalize_mean_dims(mean_dims: Sequence[int] | int | None, rank: int) -> List[int]:
6274
"""Return mean dims as non-negative indices for the given rank."""
6375
if rank <= 0:

0 commit comments

Comments
 (0)