|
21 | 21 | ) |
22 | 22 | from executorch.exir.tensor import dim_order_from_stride |
23 | 23 | from torch.fx.experimental.symbolic_shapes import guard_or_false |
| 24 | +from torch.fx.passes.infra.pass_base import PassBase |
24 | 25 |
|
25 | 26 | logger = logging.getLogger(__file__) |
26 | 27 | logger.setLevel(logging.INFO) |
@@ -80,7 +81,8 @@ class NonzeroDimOrderPass(ExportPass): |
80 | 81 | ATen returns that result column by column, in dim order (1, 0), which most |
81 | 82 | portable kernels cannot read and the dim order ops cannot express. The pass |
82 | 83 | runs before partitioning, so every op and every delegate is compiled for the |
83 | | - layout it reads at run time. |
| 84 | + layout it reads at run time. FoldNonzeroCopyPass removes the copy again |
| 85 | + where nonzero can write the default dim order itself. |
84 | 86 | """ |
85 | 87 |
|
86 | 88 | def __init__(self) -> None: |
@@ -190,6 +192,52 @@ def _layout_changed(result, meta) -> bool: |
190 | 192 | return _layout(result.to_tensor()) != _layout(original) |
191 | 193 |
|
192 | 194 |
|
| 195 | +class FoldNonzeroCopyPass(PassBase): |
| 196 | + """ |
| 197 | + Removes the copy that NonzeroDimOrderPass adds after nonzero when the copy |
| 198 | + is the only reader of the result and both run outside any delegate. The |
| 199 | + nonzero kernel then writes its result in the default dim order itself. So |
| 200 | + the program needs one buffer for it, and runtimes whose nonzero kernel |
| 201 | + always writes in that order return the right indices too. |
| 202 | +
|
| 203 | + It runs after SpecPropPass, because a retrace gives nonzero the layout of |
| 204 | + ATen again, and it changes the graph in place. |
| 205 | + """ |
| 206 | + |
| 207 | + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: |
| 208 | + modified = False |
| 209 | + for module in graph_module.modules(): |
| 210 | + if not isinstance(module, torch.fx.GraphModule): |
| 211 | + continue |
| 212 | + folded = False |
| 213 | + for node in list(module.graph.nodes): |
| 214 | + if ( |
| 215 | + node.target != exir_ops.edge.aten.nonzero.default |
| 216 | + or len(node.users) != 1 |
| 217 | + ): |
| 218 | + continue |
| 219 | + (copy,) = node.users |
| 220 | + if copy.target != exir_ops.edge.dim_order_ops._clone_dim_order.default: |
| 221 | + continue |
| 222 | + # The copy node becomes the nonzero, since the graph signature |
| 223 | + # can name it as an output. |
| 224 | + layout = { |
| 225 | + key: copy.meta[key] |
| 226 | + for key in ("val", "tensor_meta", "spec") |
| 227 | + if key in copy.meta |
| 228 | + } |
| 229 | + copy.target = node.target |
| 230 | + copy.args = node.args |
| 231 | + copy.kwargs = node.kwargs |
| 232 | + copy.meta = {**node.meta, **layout} |
| 233 | + module.graph.erase_node(node) |
| 234 | + folded = True |
| 235 | + if folded: |
| 236 | + module.recompile() |
| 237 | + modified = True |
| 238 | + return PassResult(graph_module, modified) |
| 239 | + |
| 240 | + |
193 | 241 | def _layout(tensor: torch.Tensor) -> List[int]: |
194 | 242 | # A dim of size 1 moves no element, whatever its place in the dim order. |
195 | 243 | return [ |
|
0 commit comments