From e3c44dc9eaec062496a5ac3f4f3bd15f3db2cade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Tue, 17 Nov 2026 13:11:50 +0100 Subject: [PATCH 1/3] Replace in-place writes before XNNPACK quantizer annotation prepare_pt2e observes a tensor before a later in-place write through a view, so the quantized model ignores what was written (torchvision's batch_images fills new_full with copy_). Add a replace_in_place_ops transform that rewrites such writes to their functional variants, and call it from XNNPACKQuantizer.transform_for_annotation. Writes to graph inputs and buffers are kept. --- backends/transforms/replace_in_place_ops.py | 111 ++++++++++++++++++ backends/transforms/targets.bzl | 20 ++++ .../test/test_replace_in_place_ops.py | 75 ++++++++++++ backends/xnnpack/quantizer/BUCK | 1 + .../xnnpack/quantizer/xnnpack_quantizer.py | 2 + .../test/quantizer/test_pt2e_quantization.py | 25 ++++ 6 files changed, 234 insertions(+) create mode 100644 backends/transforms/replace_in_place_ops.py create mode 100644 backends/transforms/test/test_replace_in_place_ops.py diff --git a/backends/transforms/replace_in_place_ops.py b/backends/transforms/replace_in_place_ops.py new file mode 100644 index 00000000000..e3af17d7148 --- /dev/null +++ b/backends/transforms/replace_in_place_ops.py @@ -0,0 +1,111 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict + +from typing import Any, List, Optional, Tuple + +import torch +from torch.fx import GraphModule, Node + +# Views that an in-place write can go through, mapped to the op that writes the +# result back into the viewed tensor. +_VIEW_TO_SCATTER = { + torch.ops.aten.select.int: torch.ops.aten.select_scatter.default, + torch.ops.aten.slice.Tensor: torch.ops.aten.slice_scatter.default, + torch.ops.aten.diagonal.default: torch.ops.aten.diagonal_scatter.default, + torch.ops.aten.as_strided.default: torch.ops.aten.as_strided_scatter.default, +} + + +def _get_functional_op(op: torch._ops.OpOverload) -> Optional[torch._ops.OpOverload]: + name = op._schema.name.split("::")[-1] + if not name.endswith("_"): + return None + packet = getattr(getattr(torch.ops, op.namespace), name[:-1], None) + if packet is None or op._overloadname not in packet.overloads(): + return None + return getattr(packet, op._overloadname) + + +def _get_arg0_alias_info(node: Node) -> Optional[Any]: + if not isinstance(node.target, torch._ops.OpOverload) or not node.args: + return None + return node.target._schema.arguments[0].alias_info + + +def replace_in_place_ops(graph_module: GraphModule) -> GraphModule: + """Replace in-place ops on tensors created in the graph with their functional + variants, e.g. `buf.select(0, 0).copy_(x)` with + `select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`. + + Meant for a quantizer's `transform_for_annotation`: an observer reads its + input where it is inserted, so it misses a later in-place write to that + input, and the quantized model ignores whatever was written (for example + torchvision's `batch_images`, which fills `new_full` with `copy_`). Writes + to graph inputs and buffers are kept, as they have to stay visible outside + the graph. + """ + graph = graph_module.graph + for node in list(graph.nodes): + alias_info = _get_arg0_alias_info(node) + if alias_info is None or not alias_info.is_write: + continue + if not isinstance(node.args[0], Node): + continue + functional_op = _get_functional_op(node.target) + views: List[Node] = [] + base = node.args[0] + while base.target in _VIEW_TO_SCATTER: + views.append(base) + base = base.args[0] + if ( + functional_op is None + or not isinstance(base.target, torch._ops.OpOverload) + or any(r.alias_info for r in base.target._schema.returns) + ): + continue + # Aliases of `base` created before the write and used after it would + # need to be recomputed from the new value; leave those writes in place. + aliases = [] + to_visit = [base] + while to_visit: + n = to_visit.pop() + for user in n.users: + if _get_arg0_alias_info(user) is not None and user.args[0] is n: + aliases.append(user) + to_visit.append(user) + if any( + user > node + for alias in aliases + if alias < node and alias not in views + for user in alias.users + ): + continue + + with graph.inserting_before(node): + new_value = graph.call_function(functional_op, node.args, node.kwargs) + new_value.meta = dict(node.meta) + replacements: List[Tuple[Node, Node]] = [ + (node, new_value), + (node.args[0], new_value), + ] + for view in views: + parent = view.args[0] + new_value = graph.call_function( + _VIEW_TO_SCATTER[view.target], + (parent, new_value, *view.args[1:]), + view.kwargs, + ) + new_value.meta = {**node.meta, "val": parent.meta.get("val")} + replacements.append((parent, new_value)) + for old, new in replacements: + old.replace_all_uses_with( + new, delete_user_cb=lambda user, write=node: user > write + ) + graph.erase_node(node) + graph_module.recompile() + return graph_module diff --git a/backends/transforms/targets.bzl b/backends/transforms/targets.bzl index 79ac4684c5e..7ea7147819c 100644 --- a/backends/transforms/targets.bzl +++ b/backends/transforms/targets.bzl @@ -239,6 +239,15 @@ def define_common_targets(): ], ) + runtime.python_library( + name = "replace_in_place_ops", + srcs = ["replace_in_place_ops.py"], + visibility = ["PUBLIC"], + deps = [ + "//caffe2:torch", + ], + ) + runtime.python_library( name = "remove_clone_ops", srcs = ["remove_clone_ops.py"], @@ -692,6 +701,17 @@ def define_common_targets(): ], ) + runtime.python_test( + name = "test_replace_in_place_ops", + srcs = [ + "test/test_replace_in_place_ops.py", + ], + deps = [ + "//caffe2:torch", + ":replace_in_place_ops", + ], + ) + runtime.python_test( name = "test_remove_clone_ops", srcs = [ diff --git a/backends/transforms/test/test_replace_in_place_ops.py b/backends/transforms/test/test_replace_in_place_ops.py new file mode 100644 index 00000000000..a45fe17e6d4 --- /dev/null +++ b/backends/transforms/test/test_replace_in_place_ops.py @@ -0,0 +1,75 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +import torch +from executorch.backends.transforms.replace_in_place_ops import replace_in_place_ops + + +def _targets(gm: torch.fx.GraphModule): + return [n.target for n in gm.graph.nodes if n.op == "call_function"] + + +class TestReplaceInPlaceOps(unittest.TestCase): + def _export(self, module, *inputs): + return torch.export.export(module.eval(), inputs, strict=True).module() + + def test_write_through_view_becomes_scatter(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3, 4) + buf.select(0, 1)[:, 1:3].copy_(x) + return buf * 2 + + x = torch.randn(3, 2) + gm = replace_in_place_ops(self._export(M(), x)) + targets = _targets(gm) + self.assertNotIn(torch.ops.aten.copy_.default, targets) + self.assertIn(torch.ops.aten.select_scatter.default, targets) + self.assertIn(torch.ops.aten.slice_scatter.default, targets) + torch.testing.assert_close(gm(x), M()(x)) + + def test_direct_in_place_op_becomes_functional(self): + class M(torch.nn.Module): + def forward(self, x): + y = x + 1 + y.relu_() + return y * 2 + + x = torch.randn(4) + gm = replace_in_place_ops(self._export(M(), x)) + self.assertNotIn(torch.ops.aten.relu_.default, _targets(gm)) + torch.testing.assert_close(gm(x), M()(x)) + + def test_write_to_input_is_kept(self): + class M(torch.nn.Module): + def forward(self, x, y): + x.select(0, 0).copy_(y) + return x + 1 + + x, y = torch.randn(2, 3), torch.randn(3) + gm = replace_in_place_ops(self._export(M(), x.clone(), y)) + self.assertIn(torch.ops.aten.copy_.default, _targets(gm)) + + def test_alias_read_after_write_is_kept(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3) + row = buf.select(0, 1) + buf.select(0, 0).copy_(x) + return row + buf.sum() + + x = torch.randn(3) + gm = replace_in_place_ops(self._export(M(), x)) + # `row` was taken before the write and is read after it, so rewriting + # would leave it pointing at the old value. + self.assertIn(torch.ops.aten.copy_.default, _targets(gm)) + torch.testing.assert_close(gm(x), M()(x)) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/xnnpack/quantizer/BUCK b/backends/xnnpack/quantizer/BUCK index 8257d561197..f07661ebbb9 100644 --- a/backends/xnnpack/quantizer/BUCK +++ b/backends/xnnpack/quantizer/BUCK @@ -9,6 +9,7 @@ fbcode_target( srcs = ["xnnpack_quantizer.py"], deps = [ ":xnnpack_quantizer_utils", + "//executorch/backends/transforms:replace_in_place_ops", "//executorch/backends/xnnpack/utils:xnnpack_utils", "//caffe2:torch", "//executorch/exir:lib", diff --git a/backends/xnnpack/quantizer/xnnpack_quantizer.py b/backends/xnnpack/quantizer/xnnpack_quantizer.py index db4f4796942..d57f3870d37 100644 --- a/backends/xnnpack/quantizer/xnnpack_quantizer.py +++ b/backends/xnnpack/quantizer/xnnpack_quantizer.py @@ -9,6 +9,7 @@ import torch import torch._dynamo as torchdynamo import torch.nn.functional as F +from executorch.backends.transforms.replace_in_place_ops import replace_in_place_ops from executorch.backends.xnnpack.quantizer.xnnpack_quantizer_utils import ( _convert_scalars_to_attrs, OP_TO_ANNOTATOR, @@ -382,6 +383,7 @@ def transform_for_annotation( self, model: torch.fx.GraphModule ) -> torch.fx.GraphModule: """Transforms scalar values to tensor attributes""" + model = replace_in_place_ops(model) return _convert_scalars_to_attrs(model) def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: diff --git a/backends/xnnpack/test/quantizer/test_pt2e_quantization.py b/backends/xnnpack/test/quantizer/test_pt2e_quantization.py index f0456a3604e..6133de37149 100644 --- a/backends/xnnpack/test/quantizer/test_pt2e_quantization.py +++ b/backends/xnnpack/test/quantizer/test_pt2e_quantization.py @@ -47,6 +47,7 @@ ) from torchao.quantization.pt2e.quantizer import ComposableQuantizer, Quantizer from torchao.quantization.pt2e.quantizer.embedding_quantizer import EmbeddingQuantizer +from torchao.quantization.utils import compute_error from torchao.testing.pt2e.utils import ( PT2ENumericDebuggerTestCase, PT2EQuantizationTestCase, @@ -719,6 +720,30 @@ def test_save_load(self) -> None: res = loaded_quantized_model(*example_inputs) self.assertEqual(ref_res, res) + def test_buffer_filled_in_place_through_view(self): + # torchvision's batch_images fills new_full with copy_. Observing the + # buffer before that write made the quantized model ignore its input. + class M(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + buf = x.new_zeros(1, 3, 8, 8) + buf.select(0, 0).copy_(x) + return self.conv(buf) + + torch.manual_seed(0) + eager = M().eval() + x = torch.randn(3, 8, 8) + m = export(eager, (x,), strict=True).module() + quantizer = XNNPACKQuantizer().set_global(get_symmetric_quantization_config()) + m = prepare_pt2e(m, quantizer) + for _ in range(4): + m(torch.randn(3, 8, 8)) + m = convert_pt2e(m) + self.assertGreater(compute_error(eager(x), m(x)).item(), 20) + instantiate_parametrized_tests(TestQuantizePT2E) From 3bea937e60f5a79926faf0472040d6539c3b44e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 18 Nov 2026 09:54:19 +0100 Subject: [PATCH 2/3] Rewrite only in-place writes through views that stay equivalent A write was rewritten even when the written view, or the in-place op's result, was used again afterwards, so a second write or read through it saw the old value. The alias check also missed unbind/split getitems. Keep those writes, and leave direct in-place ops alone, since later reads already use the in-place node. Rewrite only copy_, fill_, zero_ and arithmetic writes through select, slice and diagonal, compute val for the new nodes under the fake mode, and copy only provenance meta. Make the transform a PassBase, ReplaceInplaceOpsPass, and log the writes it keeps. --- backends/transforms/replace_in_place_ops.py | 111 ---------- backends/transforms/replace_inplace_ops.py | 186 ++++++++++++++++ backends/transforms/targets.bzl | 10 +- .../test/test_replace_in_place_ops.py | 75 ------- .../test/test_replace_inplace_ops.py | 203 ++++++++++++++++++ backends/xnnpack/quantizer/BUCK | 2 +- .../xnnpack/quantizer/xnnpack_quantizer.py | 7 +- .../test/quantizer/test_pt2e_quantization.py | 48 ++++- 8 files changed, 445 insertions(+), 197 deletions(-) delete mode 100644 backends/transforms/replace_in_place_ops.py create mode 100644 backends/transforms/replace_inplace_ops.py delete mode 100644 backends/transforms/test/test_replace_in_place_ops.py create mode 100644 backends/transforms/test/test_replace_inplace_ops.py diff --git a/backends/transforms/replace_in_place_ops.py b/backends/transforms/replace_in_place_ops.py deleted file mode 100644 index e3af17d7148..00000000000 --- a/backends/transforms/replace_in_place_ops.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-strict - -from typing import Any, List, Optional, Tuple - -import torch -from torch.fx import GraphModule, Node - -# Views that an in-place write can go through, mapped to the op that writes the -# result back into the viewed tensor. -_VIEW_TO_SCATTER = { - torch.ops.aten.select.int: torch.ops.aten.select_scatter.default, - torch.ops.aten.slice.Tensor: torch.ops.aten.slice_scatter.default, - torch.ops.aten.diagonal.default: torch.ops.aten.diagonal_scatter.default, - torch.ops.aten.as_strided.default: torch.ops.aten.as_strided_scatter.default, -} - - -def _get_functional_op(op: torch._ops.OpOverload) -> Optional[torch._ops.OpOverload]: - name = op._schema.name.split("::")[-1] - if not name.endswith("_"): - return None - packet = getattr(getattr(torch.ops, op.namespace), name[:-1], None) - if packet is None or op._overloadname not in packet.overloads(): - return None - return getattr(packet, op._overloadname) - - -def _get_arg0_alias_info(node: Node) -> Optional[Any]: - if not isinstance(node.target, torch._ops.OpOverload) or not node.args: - return None - return node.target._schema.arguments[0].alias_info - - -def replace_in_place_ops(graph_module: GraphModule) -> GraphModule: - """Replace in-place ops on tensors created in the graph with their functional - variants, e.g. `buf.select(0, 0).copy_(x)` with - `select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`. - - Meant for a quantizer's `transform_for_annotation`: an observer reads its - input where it is inserted, so it misses a later in-place write to that - input, and the quantized model ignores whatever was written (for example - torchvision's `batch_images`, which fills `new_full` with `copy_`). Writes - to graph inputs and buffers are kept, as they have to stay visible outside - the graph. - """ - graph = graph_module.graph - for node in list(graph.nodes): - alias_info = _get_arg0_alias_info(node) - if alias_info is None or not alias_info.is_write: - continue - if not isinstance(node.args[0], Node): - continue - functional_op = _get_functional_op(node.target) - views: List[Node] = [] - base = node.args[0] - while base.target in _VIEW_TO_SCATTER: - views.append(base) - base = base.args[0] - if ( - functional_op is None - or not isinstance(base.target, torch._ops.OpOverload) - or any(r.alias_info for r in base.target._schema.returns) - ): - continue - # Aliases of `base` created before the write and used after it would - # need to be recomputed from the new value; leave those writes in place. - aliases = [] - to_visit = [base] - while to_visit: - n = to_visit.pop() - for user in n.users: - if _get_arg0_alias_info(user) is not None and user.args[0] is n: - aliases.append(user) - to_visit.append(user) - if any( - user > node - for alias in aliases - if alias < node and alias not in views - for user in alias.users - ): - continue - - with graph.inserting_before(node): - new_value = graph.call_function(functional_op, node.args, node.kwargs) - new_value.meta = dict(node.meta) - replacements: List[Tuple[Node, Node]] = [ - (node, new_value), - (node.args[0], new_value), - ] - for view in views: - parent = view.args[0] - new_value = graph.call_function( - _VIEW_TO_SCATTER[view.target], - (parent, new_value, *view.args[1:]), - view.kwargs, - ) - new_value.meta = {**node.meta, "val": parent.meta.get("val")} - replacements.append((parent, new_value)) - for old, new in replacements: - old.replace_all_uses_with( - new, delete_user_cb=lambda user, write=node: user > write - ) - graph.erase_node(node) - graph_module.recompile() - return graph_module diff --git a/backends/transforms/replace_inplace_ops.py b/backends/transforms/replace_inplace_ops.py new file mode 100644 index 00000000000..bca3a3890e6 --- /dev/null +++ b/backends/transforms/replace_inplace_ops.py @@ -0,0 +1,186 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict + +import copy +import logging +import operator +from typing import Any, Dict, List, Optional, Set + +import torch +from torch.fx import Graph, GraphModule, Node +from torch.fx.passes.infra.pass_base import PassBase, PassResult +from torch.utils import _pytree as pytree + +logger: logging.Logger = logging.getLogger(__name__) + +aten = torch.ops.aten + +# Views a write can go through, mapped to the op that writes the result back +# into the viewed tensor. +_VIEW_TO_SCATTER: Dict[torch._ops.OpOverload, torch._ops.OpOverload] = { + aten.select.int: aten.select_scatter.default, + aten.slice.Tensor: aten.slice_scatter.default, + aten.diagonal.default: aten.diagonal_scatter.default, +} + +# In-place ops that are rewritten, mapped to their functional variants. +_INPLACE_TO_FUNCTIONAL: Dict[torch._ops.OpOverload, torch._ops.OpOverload] = { + aten.copy_.default: aten.copy.default, + aten.fill_.Scalar: aten.fill.Scalar, + aten.fill_.Tensor: aten.fill.Tensor, + aten.zero_.default: aten.zero.default, + aten.add_.Tensor: aten.add.Tensor, + aten.sub_.Tensor: aten.sub.Tensor, + aten.mul_.Tensor: aten.mul.Tensor, + aten.div_.Tensor: aten.div.Tensor, +} + +_PROVENANCE_KEYS = ("stack_trace", "nn_module_stack", "source_fn_stack") + + +def _returns_alias(node: Any) -> bool: + return ( + isinstance(node, Node) + and isinstance(node.target, torch._ops.OpOverload) + and any(r.alias_info is not None for r in node.target._schema.returns) + ) + + +def _is_alias(node: Any) -> bool: + if isinstance(node, Node) and node.target is operator.getitem: + return _returns_alias(node.args[0]) + return _returns_alias(node) + + +def _is_write(node: Node) -> bool: + if not isinstance(node.target, torch._ops.OpOverload) or not node.args: + return False + alias_info = node.target._schema.arguments[0].alias_info + return alias_info is not None and alias_info.is_write + + +def _aliases(base: Node) -> Set[Node]: + """Views of `base`, in-place ops on it, and getitems of its multi-output + views such as unbind or split.""" + aliases: Set[Node] = set() + to_visit = [base] + while to_visit: + n = to_visit.pop() + for user in n.users: + if user not in aliases and ( + (_returns_alias(user) and user.args[0] is n) + or (user.target is operator.getitem and _returns_alias(n)) + ): + aliases.add(user) + to_visit.append(user) + return aliases + + +def _fake_run(op: torch._ops.OpOverload, args: Any, kwargs: Any) -> Any: + args, kwargs = pytree.tree_map_only( + Node, lambda n: n.meta.get("val"), (args, kwargs) + ) + fake_mode = torch._guards.detect_fake_mode(pytree.tree_leaves((args, kwargs))) + if fake_mode is None: + return None + with fake_mode: + return op(*args, **kwargs) + + +def _add_node( + graph: Graph, op: torch._ops.OpOverload, args: Any, kwargs: Any, like: Node +) -> Node: + node = graph.call_function(op, args, kwargs) + node.meta = {k: copy.copy(like.meta[k]) for k in _PROVENANCE_KEYS if k in like.meta} + node.meta["val"] = _fake_run(op, args, kwargs) + return node + + +def _rewrite(graph: Graph, node: Node) -> Optional[str]: + """Rewrite `node`, an in-place write through a view, or return why it is + kept.""" + views: List[Node] = [] + base = node.args[0] + while isinstance(base, Node) and base.target in _VIEW_TO_SCATTER: + views.append(base) + base = base.args[0] + if not views: + return "the write does not go through select, slice or diagonal" + if base.op != "call_function" or _is_alias(base): + return "the written tensor is an input, get_attr or view" + if node.target not in _INPLACE_TO_FUNCTIONAL: + return "the op has no functional replacement here" + # Only `base` is redirected to the new value below, so every later access + # must go through it. A view, or the in-place op's own result, that is read + # after the write would still see the old value. + if node.users or any( + user > node for alias in _aliases(base) if alias < node for user in alias.users + ): + return "an alias of the written tensor is read after the write" + functional_op = _INPLACE_TO_FUNCTIONAL[node.target] + value = _fake_run(functional_op, node.args, node.kwargs) + written = node.args[0].meta.get("val") + if not ( + isinstance(value, torch.Tensor) + and isinstance(written, torch.Tensor) + and value.dtype == written.dtype + and value.shape == written.shape + ): + return "the functional op changes the dtype or shape" + + with graph.inserting_before(node): + new_value = _add_node(graph, functional_op, node.args, node.kwargs, node) + for view in views: + scatter = _VIEW_TO_SCATTER[view.target] + scatter_args = (view.args[0], new_value, *view.args[1:]) + new_value = _add_node(graph, scatter, scatter_args, view.kwargs, node) + base.replace_all_uses_with(new_value, delete_user_cb=lambda user: user > node) + graph.erase_node(node) + return None + + +class ReplaceInplaceOpsPass(PassBase): + """Replace in-place writes through views of tensors created in the graph + with functional ops, e.g. `buf.select(0, 0).copy_(x)` with + `select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`. + + Meant for a quantizer's `transform_for_annotation`: an observer reads its + input where it is inserted, so it misses a later in-place write through a + view of that input, and the quantized model ignores whatever was written + (pytorch/ao#5040, e.g. torchvision's `batch_images`). + + Only `copy_`, `fill_`, `zero_`, `add_`, `sub_`, `mul_` and `div_` through + chains of `select`, `slice` and `diagonal` are rewritten. These writes are + kept, logged at debug level, and still hit the issue above: + + - writes through other views (`view`, `permute`, `unbind`, ...); + - writes to graph inputs and to `get_attr` parameters, buffers and + constants; + - writes after which a view of the same tensor, or the in-place op's own + result, is read; + - writes whose functional op would change the dtype. + + Direct in-place ops such as `relu_` are left alone: later reads already use + the in-place node, so an observer sees the written value. + """ + + def call(self, graph_module: GraphModule) -> PassResult: + modified = False + for node in list(graph_module.graph.nodes): + if node.op != "call_function" or not _is_write(node): + continue + if not _is_alias(node.args[0]): + continue + reason = _rewrite(graph_module.graph, node) + if reason is None: + modified = True + else: + logger.debug("Keeping %s: %s", node.format_node(), reason) + if modified: + graph_module.recompile() + return PassResult(graph_module, modified) diff --git a/backends/transforms/targets.bzl b/backends/transforms/targets.bzl index 7ea7147819c..7f0bcc29221 100644 --- a/backends/transforms/targets.bzl +++ b/backends/transforms/targets.bzl @@ -240,8 +240,8 @@ def define_common_targets(): ) runtime.python_library( - name = "replace_in_place_ops", - srcs = ["replace_in_place_ops.py"], + name = "replace_inplace_ops", + srcs = ["replace_inplace_ops.py"], visibility = ["PUBLIC"], deps = [ "//caffe2:torch", @@ -702,13 +702,13 @@ def define_common_targets(): ) runtime.python_test( - name = "test_replace_in_place_ops", + name = "test_replace_inplace_ops", srcs = [ - "test/test_replace_in_place_ops.py", + "test/test_replace_inplace_ops.py", ], deps = [ "//caffe2:torch", - ":replace_in_place_ops", + ":replace_inplace_ops", ], ) diff --git a/backends/transforms/test/test_replace_in_place_ops.py b/backends/transforms/test/test_replace_in_place_ops.py deleted file mode 100644 index a45fe17e6d4..00000000000 --- a/backends/transforms/test/test_replace_in_place_ops.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import unittest - -import torch -from executorch.backends.transforms.replace_in_place_ops import replace_in_place_ops - - -def _targets(gm: torch.fx.GraphModule): - return [n.target for n in gm.graph.nodes if n.op == "call_function"] - - -class TestReplaceInPlaceOps(unittest.TestCase): - def _export(self, module, *inputs): - return torch.export.export(module.eval(), inputs, strict=True).module() - - def test_write_through_view_becomes_scatter(self): - class M(torch.nn.Module): - def forward(self, x): - buf = x.new_zeros(2, 3, 4) - buf.select(0, 1)[:, 1:3].copy_(x) - return buf * 2 - - x = torch.randn(3, 2) - gm = replace_in_place_ops(self._export(M(), x)) - targets = _targets(gm) - self.assertNotIn(torch.ops.aten.copy_.default, targets) - self.assertIn(torch.ops.aten.select_scatter.default, targets) - self.assertIn(torch.ops.aten.slice_scatter.default, targets) - torch.testing.assert_close(gm(x), M()(x)) - - def test_direct_in_place_op_becomes_functional(self): - class M(torch.nn.Module): - def forward(self, x): - y = x + 1 - y.relu_() - return y * 2 - - x = torch.randn(4) - gm = replace_in_place_ops(self._export(M(), x)) - self.assertNotIn(torch.ops.aten.relu_.default, _targets(gm)) - torch.testing.assert_close(gm(x), M()(x)) - - def test_write_to_input_is_kept(self): - class M(torch.nn.Module): - def forward(self, x, y): - x.select(0, 0).copy_(y) - return x + 1 - - x, y = torch.randn(2, 3), torch.randn(3) - gm = replace_in_place_ops(self._export(M(), x.clone(), y)) - self.assertIn(torch.ops.aten.copy_.default, _targets(gm)) - - def test_alias_read_after_write_is_kept(self): - class M(torch.nn.Module): - def forward(self, x): - buf = x.new_zeros(2, 3) - row = buf.select(0, 1) - buf.select(0, 0).copy_(x) - return row + buf.sum() - - x = torch.randn(3) - gm = replace_in_place_ops(self._export(M(), x)) - # `row` was taken before the write and is read after it, so rewriting - # would leave it pointing at the old value. - self.assertIn(torch.ops.aten.copy_.default, _targets(gm)) - torch.testing.assert_close(gm(x), M()(x)) - - -if __name__ == "__main__": - unittest.main() diff --git a/backends/transforms/test/test_replace_inplace_ops.py b/backends/transforms/test/test_replace_inplace_ops.py new file mode 100644 index 00000000000..5ff0860453f --- /dev/null +++ b/backends/transforms/test/test_replace_inplace_ops.py @@ -0,0 +1,203 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +import torch +from executorch.backends.transforms.replace_inplace_ops import ReplaceInplaceOpsPass +from torch.testing._internal.common_utils import ( + instantiate_parametrized_tests, + parametrize, +) + + +def _targets(gm: torch.fx.GraphModule): + return [n.target for n in gm.graph.nodes if n.op == "call_function"] + + +class TestReplaceInplaceOps(unittest.TestCase): + def _run(self, module, *inputs, strict=True): + """Export `module`, run the pass, and check it against eager.""" + module = module.eval() + gm = torch.export.export( + module, tuple(i.clone() for i in inputs), strict=strict + ).module() + result = ReplaceInplaceOpsPass()(gm) + torch.testing.assert_close( + result.graph_module(*[i.clone() for i in inputs]), + module(*[i.clone() for i in inputs]), + ) + return result + + @parametrize("strict", [True, False]) + def test_write_through_view_becomes_scatter(self, strict): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3, 4) + buf.select(0, 1)[:, 1:3].copy_(x) + return buf * 2 + + result = self._run(M(), torch.randn(3, 2), strict=strict) + self.assertTrue(result.modified) + targets = _targets(result.graph_module) + self.assertNotIn(torch.ops.aten.copy_.default, targets) + self.assertIn(torch.ops.aten.select_scatter.default, targets) + self.assertIn(torch.ops.aten.slice_scatter.default, targets) + + @parametrize("strict", [True, False]) + def test_writes_in_a_loop(self, strict): + # Like torchvision's batch_images. + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(3, 2, 5) + for i in range(3): + buf[i, :, : 2 + i].copy_(x[:, : 2 + i] * i) + return buf + 1 + + result = self._run(M(), torch.randn(2, 4), strict=strict) + self.assertNotIn(torch.ops.aten.copy_.default, _targets(result.graph_module)) + + def test_write_through_diagonal(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(3, 3) + buf.diagonal().add_(x) + return buf * 2 + + result = self._run(M(), torch.randn(3)) + self.assertIn( + torch.ops.aten.diagonal_scatter.default, _targets(result.graph_module) + ) + + def test_new_nodes_get_their_own_meta(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3, 4) + buf.select(0, 1)[:, 1:3].copy_(x) + return buf * 2 + + gm = self._run(M(), torch.randn(3, 2)).graph_module + nodes = {n.target: n for n in gm.graph.nodes} + for target, shape in [ + (torch.ops.aten.copy.default, (3, 2)), + (torch.ops.aten.slice_scatter.default, (3, 4)), + (torch.ops.aten.select_scatter.default, (2, 3, 4)), + ]: + self.assertEqual(tuple(nodes[target].meta["val"].shape), shape) + self.assertIsNot( + nodes[torch.ops.aten.copy.default].meta["nn_module_stack"], + nodes[torch.ops.aten.select_scatter.default].meta["nn_module_stack"], + ) + + def test_second_write_through_the_same_view_is_kept(self): + def first(x): + buf = x.new_zeros(2, 3) + row = buf.select(0, 0) + row.copy_(x) + row.mul_(2) + return buf + 0 + + def augmented(x): + buf = x.new_zeros(2, 3) + row = buf.select(0, 0) + row += x + row *= 2 + return buf + 0 + + def chained(x): + buf = x.new_zeros(2, 3) + buf[0].copy_(x).add_(1) + return buf + 0 + + def base_written_later(x): + buf = x.new_zeros(2, 3) + row = buf.select(0, 0) + row.copy_(x) + buf.add_(1) + return row * 2 + + for fn in (first, augmented, chained, base_written_later): + with self.subTest(fn.__name__): + M = type(fn.__name__, (torch.nn.Module,), {"forward": staticmethod(fn)}) + self._run(M(), torch.randn(3)) + + def test_alias_read_after_write_is_kept(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3) + row = buf.select(0, 0) + buf.select(0, 0).copy_(x) + return row + buf.sum() + + result = self._run(M(), torch.randn(3)) + self.assertFalse(result.modified) + + def test_unbind_alias_read_after_write_is_kept(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3) + first, _ = buf.unbind(0) + buf.select(0, 0).copy_(x) + return first * 2 + + result = self._run(M(), torch.randn(3)) + self.assertFalse(result.modified) + + def test_inplace_view_op_is_kept(self): + class M(torch.nn.Module): + def forward(self, x): + buf = x.new_zeros(2, 3) + x + buf.select(0, 0).unsqueeze_(0) + return buf * 2 + + self.assertFalse(self._run(M(), torch.randn(3)).modified) + + def test_direct_in_place_ops_are_kept(self): + class M(torch.nn.Module): + def forward(self, x): + y = x + 1 + y.relu_() + y.gt_(0.5) + return y * 2 + + result = self._run(M(), torch.randn(4)) + self.assertFalse(result.modified) + self.assertIn(torch.ops.aten.relu_.default, _targets(result.graph_module)) + + def test_write_to_input_is_kept(self): + class M(torch.nn.Module): + def forward(self, x, y): + x.select(0, 0).copy_(y) + return x + 1 + + x, y = torch.randn(2, 3), torch.randn(3) + gm = torch.export.export(M(), (x.clone(), y), strict=True).module() + self.assertFalse(ReplaceInplaceOpsPass()(gm).modified) + gm(x, y) + torch.testing.assert_close(x[0], y) + + def test_write_to_buffer_is_kept(self): + class M(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("buf", torch.zeros(2, 3)) + + def forward(self, x): + self.buf.select(0, 0).copy_(x) + return self.buf * 2 + + x = torch.randn(3) + gm = torch.export.export(M(), (x,), strict=True).module() + self.assertFalse(ReplaceInplaceOpsPass()(gm).modified) + gm(x) + torch.testing.assert_close(gm.buf[0], x) + + +instantiate_parametrized_tests(TestReplaceInplaceOps) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/xnnpack/quantizer/BUCK b/backends/xnnpack/quantizer/BUCK index f07661ebbb9..66f2817f974 100644 --- a/backends/xnnpack/quantizer/BUCK +++ b/backends/xnnpack/quantizer/BUCK @@ -9,7 +9,7 @@ fbcode_target( srcs = ["xnnpack_quantizer.py"], deps = [ ":xnnpack_quantizer_utils", - "//executorch/backends/transforms:replace_in_place_ops", + "//executorch/backends/transforms:replace_inplace_ops", "//executorch/backends/xnnpack/utils:xnnpack_utils", "//caffe2:torch", "//executorch/exir:lib", diff --git a/backends/xnnpack/quantizer/xnnpack_quantizer.py b/backends/xnnpack/quantizer/xnnpack_quantizer.py index d57f3870d37..2a50626c7d9 100644 --- a/backends/xnnpack/quantizer/xnnpack_quantizer.py +++ b/backends/xnnpack/quantizer/xnnpack_quantizer.py @@ -9,7 +9,7 @@ import torch import torch._dynamo as torchdynamo import torch.nn.functional as F -from executorch.backends.transforms.replace_in_place_ops import replace_in_place_ops +from executorch.backends.transforms.replace_inplace_ops import ReplaceInplaceOpsPass from executorch.backends.xnnpack.quantizer.xnnpack_quantizer_utils import ( _convert_scalars_to_attrs, OP_TO_ANNOTATOR, @@ -382,8 +382,9 @@ def set_filter_function(self, filter_fn: Callable[[Node], bool]): def transform_for_annotation( self, model: torch.fx.GraphModule ) -> torch.fx.GraphModule: - """Transforms scalar values to tensor attributes""" - model = replace_in_place_ops(model) + """Replaces in-place writes through views and transforms scalar values + to tensor attributes""" + model = ReplaceInplaceOpsPass()(model).graph_module return _convert_scalars_to_attrs(model) def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: diff --git a/backends/xnnpack/test/quantizer/test_pt2e_quantization.py b/backends/xnnpack/test/quantizer/test_pt2e_quantization.py index 6133de37149..e411a9e1e3a 100644 --- a/backends/xnnpack/test/quantizer/test_pt2e_quantization.py +++ b/backends/xnnpack/test/quantizer/test_pt2e_quantization.py @@ -720,9 +720,9 @@ def test_save_load(self) -> None: res = loaded_quantized_model(*example_inputs) self.assertEqual(ref_res, res) - def test_buffer_filled_in_place_through_view(self): + def test_new_tensor_filled_in_place_through_view(self): # torchvision's batch_images fills new_full with copy_. Observing the - # buffer before that write made the quantized model ignore its input. + # tensor before that write made the quantized model ignore its input. class M(torch.nn.Module): def __init__(self) -> None: super().__init__() @@ -739,11 +739,55 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: m = export(eager, (x,), strict=True).module() quantizer = XNNPACKQuantizer().set_global(get_symmetric_quantization_config()) m = prepare_pt2e(m, quantizer) + self.checkGraphModuleNodes( + m, + expected_node_occurrence={ + ns.call_function(torch.ops.aten.copy_.default): 0, + ns.call_function(torch.ops.aten.select_scatter.default): 1, + }, + ) for _ in range(4): m(torch.randn(3, 8, 8)) m = convert_pt2e(m) + self.checkGraphModuleNodes( + m, + expected_node_list=[ + ns.call_function(torch.ops.aten.select_scatter.default), + ns.call_function( + torch.ops.quantized_decomposed.quantize_per_tensor.default + ), + ns.call_function( + torch.ops.quantized_decomposed.dequantize_per_tensor.default + ), + ns.call_function(torch.ops.aten.conv2d.default), + ], + ) self.assertGreater(compute_error(eager(x), m(x)).item(), 20) + def test_direct_in_place_scalar_op_is_kept(self): + # Rewriting mul_ to mul.Tensor would make transform_for_annotation lift + # the scalar into a quantized constant. + class M(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = self.conv(x) + y.mul_(0.5) + return y + + m = export(M().eval(), (torch.randn(1, 3, 8, 8),), strict=True).module() + quantizer = XNNPACKQuantizer().set_global(get_symmetric_quantization_config()) + m = prepare_pt2e(m, quantizer) + self.checkGraphModuleNodes( + m, + expected_node_occurrence={ + ns.call_function(torch.ops.aten.mul_.Tensor): 1, + ns.call_function(torch.ops.aten.mul.Tensor): 0, + }, + ) + instantiate_parametrized_tests(TestQuantizePT2E) From 7396ca2941f4210e58baa0d4c78409afc7567e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Tue, 17 Nov 2026 13:12:04 +0100 Subject: [PATCH 3/3] Replace in-place writes before QNN quantizer annotation prepare_pt2e observes a tensor before a later in-place write through a view, so the quantized model ignores what was written. QnnQuantizer also annotates select, so its observers split the view chain and copy_ wrote into a dequantized copy of the buffer. Run ReplaceInplaceOpsPass in QnnQuantizer.transform_for_annotation. --- backends/qualcomm/quantizer/quantizer.py | 2 ++ backends/qualcomm/quantizer/targets.bzl | 1 + backends/qualcomm/tests/test_passes.py | 31 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/backends/qualcomm/quantizer/quantizer.py b/backends/qualcomm/quantizer/quantizer.py index e8c3fd5b9af..138a8e83c28 100644 --- a/backends/qualcomm/quantizer/quantizer.py +++ b/backends/qualcomm/quantizer/quantizer.py @@ -45,6 +45,7 @@ QCOM_QUANT_ANNOTATION_KEY, ) from executorch.backends.qualcomm.utils.qnn_sdk_setup import disable_mkldnn_on_amd +from executorch.backends.transforms.replace_inplace_ops import ReplaceInplaceOpsPass from torch._ops import OpOverload from torch.fx import GraphModule @@ -572,6 +573,7 @@ def transform_for_annotation(self, model: GraphModule) -> GraphModule: Returns: GraphModule: The transformed model. """ + model = ReplaceInplaceOpsPass()(model).graph_module per_block_linears = self._get_per_block_linears(model) # HTP FullyConnected reads BLOCKWISE_EXPANSION block scales past their # documented size, which can corrupt the output (pytorch/executorch#23746), diff --git a/backends/qualcomm/quantizer/targets.bzl b/backends/qualcomm/quantizer/targets.bzl index 4247da7b969..b4a0aff41cf 100644 --- a/backends/qualcomm/quantizer/targets.bzl +++ b/backends/qualcomm/quantizer/targets.bzl @@ -15,5 +15,6 @@ def define_common_targets(): visibility = ["PUBLIC"], deps = [ "//executorch/backends/transforms:decompose_sdpa", + "//executorch/backends/transforms:replace_inplace_ops", ], ) diff --git a/backends/qualcomm/tests/test_passes.py b/backends/qualcomm/tests/test_passes.py index 44e827ea312..d2ec726d8cd 100644 --- a/backends/qualcomm/tests/test_passes.py +++ b/backends/qualcomm/tests/test_passes.py @@ -90,7 +90,9 @@ from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner, Partition from torch.library import Library from torchao.quantization import IntxWeightOnlyConfig, PerAxis, quantize_ +from torchao.quantization.pt2e import MinMaxObserver from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e +from torchao.quantization.utils import compute_error class ConstantBranches(torch.nn.Module): @@ -2504,6 +2506,35 @@ def forward(self, x): "indices has no consumer and must stay internal", ) + def test_quantizer_observes_buffer_after_in_place_write(self): + # The quantizer also annotates select, so observers used to split the + # view chain and copy_ wrote into a dequantized copy of the buffer. + class M(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + + def forward(self, x): + buf = x.new_zeros(1, 3, 8, 8) + buf.select(0, 0).copy_(x) + return self.conv(buf) + + torch.manual_seed(0) + eager = M().eval() + x = torch.randn(3, 8, 8) + m = torch.export.export(eager, (x,), strict=True).module() + quantizer = QnnQuantizer() + # MinMax so that a few calibration runs cover the range the buffer is + # filled with; the default moving average starts from its zeros. + quantizer.set_default_quant_config( + quant_dtype=QuantDtype.use_8a8w, act_observer=MinMaxObserver + ) + m = prepare_pt2e(m, quantizer) + for _ in range(4): + m(torch.randn(3, 8, 8)) + m = convert_pt2e(m) + self.assertGreater(compute_error(eager(x), m(x)).item(), 20) + # Attributes PT2E compares in _union_input_edge_with when deciding whether an # input edge and its producer's output can share one observer.