From f5c3610be86248e142febe501f2fe47b39de576c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Sat, 31 Oct 2026 12:27:11 +0100 Subject: [PATCH 1/2] Place pt2e input observers after in-place writes through views prepare_pt2e inserted an input observer right after the producer, so a tensor filled later through a view (e.g. buf.select(0, 0).copy_(x)) was observed, and after convert read, before the write. Observers on such a view chain also made the write land on a dequantized copy. Fixes #5040 --- test/quantization/pt2e/test_quantize_pt2e.py | 56 ++++++++++++++ torchao/quantization/pt2e/prepare.py | 78 ++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/test/quantization/pt2e/test_quantize_pt2e.py b/test/quantization/pt2e/test_quantize_pt2e.py index f0cffccd3f1..ab34afc39d3 100644 --- a/test/quantization/pt2e/test_quantize_pt2e.py +++ b/test/quantization/pt2e/test_quantize_pt2e.py @@ -3523,6 +3523,62 @@ def forward(self, x: torch.Tensor, idx: torch.Tensor) -> torch.Tensor: ViewM(), example_inputs, {"select_scatter": "buf"} ) + def test_quantize_buffer_filled_in_place_through_view(self): + class M(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + buf = x.new_zeros(1, 3, 8, 8) + buf.select(0, 0).copy_(x) + return self.conv(buf) + + class ViewQuantizer(Quantizer): + # Also quantizes the view, so its observers sit on the written chain. + def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: + act_qspec = QuantizationSpec( + dtype=torch.int8, + quant_min=-128, + quant_max=127, + qscheme=torch.per_tensor_affine, + is_dynamic=False, + observer_or_fake_quant_ctr=observer.default_observer, + ) + for node in model.graph.nodes: + if node.target in ( + torch.ops.aten.select.int, + torch.ops.aten.conv2d.default, + ): + node.meta["quantization_annotation"] = QuantizationAnnotation( + input_qspec_map={node.args[0]: act_qspec}, + output_qspec=act_qspec, + _annotated=True, + ) + return model + + def validate(self, model: torch.fx.GraphModule) -> None: + pass + + quantizers = { + "xnnpack": XNNPACKQuantizer().set_global( + get_symmetric_quantization_config() + ), + "view": ViewQuantizer(), + } + for name, quantizer in quantizers.items(): + with self.subTest(quantizer=name): + torch.manual_seed(0) + eager = M().eval() + x = torch.randn(3, 8, 8) + m = torch.export.export(eager, (x,), strict=True).module() + m = prepare_pt2e(m, quantizer) + for _ in range(4): + m(torch.randn(3, 8, 8)) + m = convert_pt2e(m) + # Without the image in `buf`, the output would not depend on x. + self.assertGreater(compute_error(eager(x), m(x)).item(), 20) + def _prepare_mutable_buffer_quantization( self, module: torch.nn.Module, diff --git a/torchao/quantization/pt2e/prepare.py b/torchao/quantization/pt2e/prepare.py index d6098a75a82..00e0f4ca75e 100644 --- a/torchao/quantization/pt2e/prepare.py +++ b/torchao/quantization/pt2e/prepare.py @@ -452,6 +452,59 @@ def _is_mutable_buffer_arg( ) +def _arg0_alias_info(node: Node): + if ( + node.op != "call_function" + or not isinstance(node.target, torch._ops.OpOverload) + or not node.args + or not node.target._schema.arguments + ): + return None + return node.target._schema.arguments[0].alias_info + + +class _InPlaceWrites: + """Tensors written in place through a chain of views, e.g. `buf.select(0, 0).copy_(x)`. + + An observer on an edge of such a chain becomes a quantize/dequantize copy after + convert, so the write would land on the copy instead of `buf`. And an observer + reading `buf` must run after the write, or it sees `buf` before it is filled. + """ + + def __init__(self, graph: Graph): + self.chain_edges: set[tuple[Node, Node]] = set() + self.base: dict[Node, Node] = {} + self.writes: dict[Node, list[Node]] = {} + for node in graph.nodes: + alias_info = _arg0_alias_info(node) + if alias_info is None or not alias_info.is_write: + continue + chain = [node] + dest = node.args[0] + while isinstance(dest, Node): + chain.append(dest) + dest_alias_info = _arg0_alias_info(dest) + if dest_alias_info is None or dest_alias_info.is_write: + break + dest = dest.args[0] + # Only a write through a view needs its chain left unobserved: an + # in-place op on the tensor itself (e.g. relu_) writes the observer's + # output just as well. + if len(chain) > 2: + self.chain_edges.update(zip(chain[1:], chain[:-1])) + base = self.base.get(chain[-1], chain[-1]) + for alias in chain: + self.base[alias] = base + self.writes.setdefault(base, []).append(node) + + def last_write_before(self, arg: Node, user: Node) -> Optional[Node]: + last_write = None + for write in self.writes.get(self.base.get(arg, arg), ()): + if write < user: + last_write = write + return last_write + + def _maybe_insert_input_observer_for_arg_or_kwarg( node: Union[Node, Any], arg: Argument, @@ -462,6 +515,7 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, + in_place_writes: Optional[_InPlaceWrites] = None, ) -> Argument: """ Given a `node` and an `arg`, inserts an input observer between @@ -482,6 +536,7 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( buffer_names, is_qat, model_device, + in_place_writes, ) new_arg_to_return.append(new_inner_arg) return type(arg)(new_arg_to_return) @@ -507,6 +562,14 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( if _is_mutable_buffer_arg(node, original_arg, buffer_names): return new_arg + if in_place_writes is not None and input_edge in in_place_writes.chain_edges: + return new_arg + last_write = ( + in_place_writes.last_write_before(original_arg, node) + if in_place_writes is not None + else None + ) + # input_edge needs to be observed input_edge_obs_or_fq = obs_or_fq_map[input_edge] if input_edge_obs_or_fq is None: @@ -533,6 +596,8 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( for maybe_obs_node in arg.users.keys(): if not _is_activation_post_process_node(maybe_obs_node, named_modules): continue + if last_write is not None and maybe_obs_node < last_write: + continue maybe_obs_mod = named_modules[maybe_obs_node.target] # type: ignore[index] if id(maybe_obs_mod) == id(input_edge_obs_or_fq): return maybe_obs_node @@ -542,6 +607,8 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( new_arg = _insert_obs_or_fq( arg, input_edge_obs_or_fq, model, named_modules, model.graph ) + if last_write is not None: + last_write.append(new_arg) return new_arg @@ -554,6 +621,7 @@ def _maybe_insert_input_observers_for_node( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, + in_place_writes: Optional[_InPlaceWrites] = None, ) -> None: """ If needed, inserts observers to the input args and kwargs of `node`. @@ -582,6 +650,7 @@ def _maybe_insert_input_observers_for_node( buffer_names, is_qat, model_device, + in_place_writes, ) new_args.append(new_arg) @@ -641,6 +710,7 @@ def _maybe_insert_input_and_output_observers_for_node( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, + in_place_writes: Optional[_InPlaceWrites] = None, ): this_node_quantization_annotation = ( node.meta[Q_ANNOTATION_KEY] if Q_ANNOTATION_KEY in node.meta else None @@ -657,6 +727,7 @@ def _maybe_insert_input_and_output_observers_for_node( buffer_names, is_qat, model_device, + in_place_writes, ) output_is_a_tensor = "val" in node.meta and isinstance(node.meta["val"], FakeTensor) @@ -695,6 +766,11 @@ def _maybe_insert_input_and_output_observers_for_node( for user_node in orig_users: if user_node is maybe_output_obs_node: continue + if ( + in_place_writes is not None + and (node, user_node) in in_place_writes.chain_edges + ): + continue user_node.replace_input_with(node, maybe_output_obs_node) @@ -722,6 +798,7 @@ def prepare( model_device = _assert_and_get_unique_device(model) named_modules = dict(model.named_modules(remove_duplicate=False)) buffer_names = {name for name, _ in model.named_buffers(remove_duplicate=False)} + in_place_writes = _InPlaceWrites(model.graph) for node in nodes_before_observation: # TODO: simplify logic for inserting observers @@ -733,6 +810,7 @@ def prepare( buffer_names, is_qat, model_device, + in_place_writes, ) model = GraphModule(model, model.graph) From 5c87717fdc1afbe9e4cf9317e7abb6bc92f9b9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Mon, 2 Nov 2026 18:59:22 +0100 Subject: [PATCH 2/2] Replace in-place ops with functional variants before pt2e prepare --- torchao/quantization/pt2e/prepare.py | 78 ------------------- torchao/quantization/pt2e/quantize_pt2e.py | 3 + torchao/quantization/pt2e/utils.py | 90 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 78 deletions(-) diff --git a/torchao/quantization/pt2e/prepare.py b/torchao/quantization/pt2e/prepare.py index 00e0f4ca75e..d6098a75a82 100644 --- a/torchao/quantization/pt2e/prepare.py +++ b/torchao/quantization/pt2e/prepare.py @@ -452,59 +452,6 @@ def _is_mutable_buffer_arg( ) -def _arg0_alias_info(node: Node): - if ( - node.op != "call_function" - or not isinstance(node.target, torch._ops.OpOverload) - or not node.args - or not node.target._schema.arguments - ): - return None - return node.target._schema.arguments[0].alias_info - - -class _InPlaceWrites: - """Tensors written in place through a chain of views, e.g. `buf.select(0, 0).copy_(x)`. - - An observer on an edge of such a chain becomes a quantize/dequantize copy after - convert, so the write would land on the copy instead of `buf`. And an observer - reading `buf` must run after the write, or it sees `buf` before it is filled. - """ - - def __init__(self, graph: Graph): - self.chain_edges: set[tuple[Node, Node]] = set() - self.base: dict[Node, Node] = {} - self.writes: dict[Node, list[Node]] = {} - for node in graph.nodes: - alias_info = _arg0_alias_info(node) - if alias_info is None or not alias_info.is_write: - continue - chain = [node] - dest = node.args[0] - while isinstance(dest, Node): - chain.append(dest) - dest_alias_info = _arg0_alias_info(dest) - if dest_alias_info is None or dest_alias_info.is_write: - break - dest = dest.args[0] - # Only a write through a view needs its chain left unobserved: an - # in-place op on the tensor itself (e.g. relu_) writes the observer's - # output just as well. - if len(chain) > 2: - self.chain_edges.update(zip(chain[1:], chain[:-1])) - base = self.base.get(chain[-1], chain[-1]) - for alias in chain: - self.base[alias] = base - self.writes.setdefault(base, []).append(node) - - def last_write_before(self, arg: Node, user: Node) -> Optional[Node]: - last_write = None - for write in self.writes.get(self.base.get(arg, arg), ()): - if write < user: - last_write = write - return last_write - - def _maybe_insert_input_observer_for_arg_or_kwarg( node: Union[Node, Any], arg: Argument, @@ -515,7 +462,6 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, - in_place_writes: Optional[_InPlaceWrites] = None, ) -> Argument: """ Given a `node` and an `arg`, inserts an input observer between @@ -536,7 +482,6 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( buffer_names, is_qat, model_device, - in_place_writes, ) new_arg_to_return.append(new_inner_arg) return type(arg)(new_arg_to_return) @@ -562,14 +507,6 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( if _is_mutable_buffer_arg(node, original_arg, buffer_names): return new_arg - if in_place_writes is not None and input_edge in in_place_writes.chain_edges: - return new_arg - last_write = ( - in_place_writes.last_write_before(original_arg, node) - if in_place_writes is not None - else None - ) - # input_edge needs to be observed input_edge_obs_or_fq = obs_or_fq_map[input_edge] if input_edge_obs_or_fq is None: @@ -596,8 +533,6 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( for maybe_obs_node in arg.users.keys(): if not _is_activation_post_process_node(maybe_obs_node, named_modules): continue - if last_write is not None and maybe_obs_node < last_write: - continue maybe_obs_mod = named_modules[maybe_obs_node.target] # type: ignore[index] if id(maybe_obs_mod) == id(input_edge_obs_or_fq): return maybe_obs_node @@ -607,8 +542,6 @@ def _maybe_insert_input_observer_for_arg_or_kwarg( new_arg = _insert_obs_or_fq( arg, input_edge_obs_or_fq, model, named_modules, model.graph ) - if last_write is not None: - last_write.append(new_arg) return new_arg @@ -621,7 +554,6 @@ def _maybe_insert_input_observers_for_node( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, - in_place_writes: Optional[_InPlaceWrites] = None, ) -> None: """ If needed, inserts observers to the input args and kwargs of `node`. @@ -650,7 +582,6 @@ def _maybe_insert_input_observers_for_node( buffer_names, is_qat, model_device, - in_place_writes, ) new_args.append(new_arg) @@ -710,7 +641,6 @@ def _maybe_insert_input_and_output_observers_for_node( buffer_names: set[str], is_qat: bool, model_device: Optional[torch.device] = None, - in_place_writes: Optional[_InPlaceWrites] = None, ): this_node_quantization_annotation = ( node.meta[Q_ANNOTATION_KEY] if Q_ANNOTATION_KEY in node.meta else None @@ -727,7 +657,6 @@ def _maybe_insert_input_and_output_observers_for_node( buffer_names, is_qat, model_device, - in_place_writes, ) output_is_a_tensor = "val" in node.meta and isinstance(node.meta["val"], FakeTensor) @@ -766,11 +695,6 @@ def _maybe_insert_input_and_output_observers_for_node( for user_node in orig_users: if user_node is maybe_output_obs_node: continue - if ( - in_place_writes is not None - and (node, user_node) in in_place_writes.chain_edges - ): - continue user_node.replace_input_with(node, maybe_output_obs_node) @@ -798,7 +722,6 @@ def prepare( model_device = _assert_and_get_unique_device(model) named_modules = dict(model.named_modules(remove_duplicate=False)) buffer_names = {name for name, _ in model.named_buffers(remove_duplicate=False)} - in_place_writes = _InPlaceWrites(model.graph) for node in nodes_before_observation: # TODO: simplify logic for inserting observers @@ -810,7 +733,6 @@ def prepare( buffer_names, is_qat, model_device, - in_place_writes, ) model = GraphModule(model, model.graph) diff --git a/torchao/quantization/pt2e/quantize_pt2e.py b/torchao/quantization/pt2e/quantize_pt2e.py index 8d9f0b97f03..53b744f9b05 100644 --- a/torchao/quantization/pt2e/quantize_pt2e.py +++ b/torchao/quantization/pt2e/quantize_pt2e.py @@ -22,6 +22,7 @@ _fuse_conv_bn_, _fuse_linear_bn_, _get_node_name_to_scope, + _replace_in_place_ops, get_arg, ) @@ -122,6 +123,7 @@ def calibrate(model, data_loader): _reject_set_grad_enabled_subgraph(model, "prepare_pt2e") original_graph_meta = model.meta node_name_to_scope = _get_node_name_to_scope(model) + _replace_in_place_ops(model) # TODO: check qconfig_mapping to make sure conv and bn are both configured # to be quantized before fusion # TODO: (maybe) rewrite this with subgraph_rewriter @@ -227,6 +229,7 @@ def train_loop(model, train_data): _reject_set_grad_enabled_subgraph(model, "prepare_qat_pt2e") original_graph_meta = model.meta node_name_to_scope = _get_node_name_to_scope(model) + _replace_in_place_ops(model) model = quantizer.transform_for_annotation(model) quantizer.annotate(model) quantizer.validate(model) diff --git a/torchao/quantization/pt2e/utils.py b/torchao/quantization/pt2e/utils.py index f5632ffd629..39ea913161d 100644 --- a/torchao/quantization/pt2e/utils.py +++ b/torchao/quantization/pt2e/utils.py @@ -1093,6 +1093,96 @@ def _fuse_linear_bn_(m: GraphModule) -> None: m.recompile() +# Views that an in-place write can go through, mapped to the op that writes the +# result back into the viewed tensor. +_VIEW_TO_SCATTER = { + torch.ops.aten.select.int: torch.ops.aten.select_scatter.default, + torch.ops.aten.slice.Tensor: torch.ops.aten.slice_scatter.default, + torch.ops.aten.diagonal.default: torch.ops.aten.diagonal_scatter.default, + torch.ops.aten.as_strided.default: torch.ops.aten.as_strided_scatter.default, +} + + +def _get_functional_op(op: torch._ops.OpOverload) -> Optional[torch._ops.OpOverload]: + name = op._schema.name.split("::")[-1] + if not name.endswith("_"): + return None + packet = getattr(getattr(torch.ops, op.namespace), name[:-1], None) + if packet is None or op._overloadname not in packet.overloads(): + return None + return getattr(packet, op._overloadname) + + +def _get_arg0_alias_info(n: Node): + if not isinstance(n.target, torch._ops.OpOverload) or not n.args: + return None + return n.target._schema.arguments[0].alias_info + + +def _replace_in_place_ops(m: GraphModule) -> None: + """Replace in-place ops on tensors created in the graph with their functional + variants, e.g. `buf.select(0, 0).copy_(x)` with + `select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`. + + An observer reads its input where it is inserted, so it misses a later + in-place write to that input. Writes to graph inputs and buffers are kept, + as they have to stay visible outside the graph. + """ + for node in list(m.graph.nodes): + alias_info = _get_arg0_alias_info(node) + if alias_info is None or not alias_info.is_write: + continue + if not isinstance(node.args[0], Node): + continue + functional_op = _get_functional_op(node.target) + views = [] + base = node.args[0] + while base.target in _VIEW_TO_SCATTER: + views.append(base) + base = base.args[0] + if ( + functional_op is None + or not isinstance(base.target, torch._ops.OpOverload) + or any(r.alias_info for r in base.target._schema.returns) + ): + continue + # Aliases of `base` created before the write and used after it would + # need to be recomputed from the new value; leave those writes in place. + aliases = [] + to_visit = [base] + while to_visit: + n = to_visit.pop() + for user in n.users: + if _get_arg0_alias_info(user) is not None and user.args[0] is n: + aliases.append(user) + to_visit.append(user) + if any( + user > node + for alias in aliases + if alias < node and alias not in views + for user in alias.users + ): + continue + + with m.graph.inserting_before(node): + new_value = m.graph.call_function(functional_op, node.args, node.kwargs) + new_value.meta = dict(node.meta) + replacements = [(node, new_value), (node.args[0], new_value)] + for view in views: + parent = view.args[0] + new_value = m.graph.call_function( + _VIEW_TO_SCATTER[view.target], + (parent, new_value, *view.args[1:]), + view.kwargs, + ) + new_value.meta = {**node.meta, "val": parent.meta.get("val")} + replacements.append((parent, new_value)) + for old, new in replacements: + old.replace_all_uses_with(new, delete_user_cb=lambda user: user > node) + m.graph.erase_node(node) + m.recompile() + + def _get_node_name_to_scope(model: GraphModule) -> dict[str, tuple[str, type]]: # TODO: move this information to fx node itself node_name_to_scope: dict[str, tuple[str, type]] = {}