Skip to content

Commit e14f332

Browse files
committed
Remove the copy of the nonzero result again in to_executorch
Export copies the result of `nonzero` into the default dim order before partitioning, so every op and delegate is compiled for the layout it reads. But in the final program the copy needs its own planned buffer, as large as the upper bound of the result: about 12 MiB more for a returned `nonzero` of a 1024 x 1024 input. It also needs the `_clone_dim_order` kernel. And a runtime built before the kernel fix writes the result row by row, so the copy reads it in the wrong places, with no error. The new `FoldNonzeroCopyPass` removes the copy in `to_executorch` when it is the only reader of the result and neither runs in a delegate, which is the usual case. The copy node then runs `nonzero` itself and keeps its own name, which the graph signature can list as an output. So `nonzero` declares the default dim order and its kernel writes that order. Such a program plans one buffer for the result, as before, needs no extra kernel, and gives the right indices on old and new runtimes. The pass runs right after `SpecPropPass`, because every retrace gives `nonzero` the layout of ATen again, and it changes the graph in place. When the copy stays, for example because a delegate takes the copy but not `nonzero`, `nonzero` keeps the `(1, 0)` layout, which its kernel follows. `test_optional_tensor_list` expects the program of main again, without the copy. The new test checks that the program has no copy, that `nonzero` declares the default dim order, that every output in the graph signature names a node of the graph, and that it runs right. It fails without the fold, and also if the fold removes the copy node instead. On a runtime built before the kernel fix, the end-to-end tests of the earlier changes also fail without it.
1 parent 2fb72bf commit e14f332

5 files changed

Lines changed: 81 additions & 4 deletions

File tree

‎exir/emit/test/test_emit.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -669,11 +669,10 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
669669
.executorch_program
670670
)
671671
self.assertTrue(
672-
isinstance(program.execution_plan[0].values[8].val, OptionalTensorList)
672+
isinstance(program.execution_plan[0].values[3].val, OptionalTensorList)
673673
)
674674
self._assertCallLength(program, 0, 3)
675-
# The nonzero result is copied into the default dim order before index.
676-
self._assertCallLength(program, 2, 4)
675+
self._assertCallLength(program, 1, 4)
677676

678677
def test_optional_float_list(self) -> None:
679678
class M(torch.nn.Module):

‎exir/passes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
LegalizePortableDimOrderPass,
4848
)
4949
from executorch.exir.passes.memory_format_ops_pass import (
50+
FoldNonzeroCopyPass,
5051
MemoryFormatOpsPass,
5152
NonzeroDimOrderPass,
5253
ReplaceEmptyLikePass,
@@ -86,6 +87,7 @@
8687
"OpReplacePass",
8788
"ToDevicePass",
8889
"EdgeToBackendOpsPass",
90+
"FoldNonzeroCopyPass",
8991
"LegalizePortableDimOrderPass",
9092
"MemoryFormatOpsPass",
9193
"MemoryPlanningPass",

‎exir/passes/memory_format_ops_pass.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
)
2222
from executorch.exir.tensor import dim_order_from_stride
2323
from torch.fx.experimental.symbolic_shapes import guard_or_false
24+
from torch.fx.passes.infra.pass_base import PassBase
2425

2526
logger = logging.getLogger(__file__)
2627
logger.setLevel(logging.INFO)
@@ -80,7 +81,8 @@ class NonzeroDimOrderPass(ExportPass):
8081
ATen returns that result column by column, in dim order (1, 0), which most
8182
portable kernels cannot read and the dim order ops cannot express. The pass
8283
runs before partitioning, so every op and every delegate is compiled for the
83-
layout it reads at run time.
84+
layout it reads at run time. FoldNonzeroCopyPass removes the copy again
85+
where nonzero can write the default dim order itself.
8486
"""
8587

8688
def __init__(self) -> None:
@@ -190,6 +192,52 @@ def _layout_changed(result, meta) -> bool:
190192
return _layout(result.to_tensor()) != _layout(original)
191193

192194

195+
class FoldNonzeroCopyPass(PassBase):
196+
"""
197+
Removes the copy that NonzeroDimOrderPass adds after nonzero when the copy
198+
is the only reader of the result and both run outside any delegate. The
199+
nonzero kernel then writes its result in the default dim order itself. So
200+
the program needs one buffer for it, and runtimes whose nonzero kernel
201+
always writes in that order return the right indices too.
202+
203+
It runs after SpecPropPass, because a retrace gives nonzero the layout of
204+
ATen again, and it changes the graph in place.
205+
"""
206+
207+
def call(self, graph_module: torch.fx.GraphModule) -> PassResult:
208+
modified = False
209+
for module in graph_module.modules():
210+
if not isinstance(module, torch.fx.GraphModule):
211+
continue
212+
folded = False
213+
for node in list(module.graph.nodes):
214+
if (
215+
node.target != exir_ops.edge.aten.nonzero.default
216+
or len(node.users) != 1
217+
):
218+
continue
219+
(copy,) = node.users
220+
if copy.target != exir_ops.edge.dim_order_ops._clone_dim_order.default:
221+
continue
222+
# The copy node becomes the nonzero, since the graph signature
223+
# can name it as an output.
224+
layout = {
225+
key: copy.meta[key]
226+
for key in ("val", "tensor_meta", "spec")
227+
if key in copy.meta
228+
}
229+
copy.target = node.target
230+
copy.args = node.args
231+
copy.kwargs = node.kwargs
232+
copy.meta = {**node.meta, **layout}
233+
module.graph.erase_node(node)
234+
folded = True
235+
if folded:
236+
module.recompile()
237+
modified = True
238+
return PassResult(graph_module, modified)
239+
240+
193241
def _layout(tensor: torch.Tensor) -> List[int]:
194242
# A dim of size 1 moves no element, whatever its place in the dim order.
195243
return [

‎exir/program/_program.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
convert_constant_dim_order_pass,
4545
dead_code_elimination_pass,
4646
EdgeToBackendOpsPass,
47+
FoldNonzeroCopyPass,
4748
LegalizePortableDimOrderPass,
4849
MemoryFormatOpsPass,
4950
NonzeroDimOrderPass,
@@ -792,6 +793,7 @@ def edge_to_executorch_passes(
792793
# there exists an unbacked symint operation.
793794
*config.passes,
794795
SpecPropPass(),
796+
FoldNonzeroCopyPass(),
795797
PropagateDevicePass(
796798
skip_h2d_for_method_inputs=device_cfg.skip_h2d_for_method_inputs,
797799
skip_d2h_for_method_outputs=device_cfg.skip_d2h_for_method_outputs,

‎exir/tests/test_memory_format_ops_pass.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,32 @@ def forward(self, x):
523523
self.assertTrue(output.is_contiguous())
524524
torch.testing.assert_close(output, torch.nonzero(x))
525525

526+
def test_nonzero_writes_default_dim_order_without_copy(self) -> None:
527+
class Model(torch.nn.Module):
528+
def forward(self, x):
529+
indices = torch.nonzero(x)
530+
return indices, indices + 1
531+
532+
model = Model()
533+
x = torch.tensor([[0.0, 1.0, 2.0], [3.0, 0.0, 0.0]])
534+
program = to_edge(export(model, (x,), strict=True)).to_executorch()
535+
exported_program = program.exported_program()
536+
nonzero, add = [
537+
node
538+
for node in exported_program.graph.nodes
539+
if isinstance(node.target, torch._ops.OpOverload)
540+
]
541+
self.assertEqual(nonzero.target, torch.ops.aten.nonzero.out)
542+
self.assertEqual(add.target, torch.ops.aten.add.out)
543+
self.assertEqual(list(nonzero.meta["spec"].dim_order), [0, 1])
544+
names = {node.name for node in exported_program.graph.nodes}
545+
for spec in exported_program.graph_signature.output_specs:
546+
self.assertIn(spec.arg.name, names)
547+
module = _load_for_executorch_from_buffer(program.buffer)
548+
outputs = module.run_method("forward", (x,))
549+
for output, expected in zip(outputs, model(x), strict=True):
550+
torch.testing.assert_close(output, expected)
551+
526552
def test_nonzero_dim_order_pass_skips_graph_without_nonzero(self) -> None:
527553
graph_module = (
528554
to_edge(export(torch.nn.ReLU(), (torch.randn(2, 3),), strict=True))

0 commit comments

Comments
 (0)