diff --git a/test/quantization/pt2e/test_quantize_pt2e.py b/test/quantization/pt2e/test_quantize_pt2e.py index f0cffccd3f..ab34afc39d 100644 --- a/test/quantization/pt2e/test_quantize_pt2e.py +++ b/test/quantization/pt2e/test_quantize_pt2e.py @@ -3523,6 +3523,62 @@ def forward(self, x: torch.Tensor, idx: torch.Tensor) -> torch.Tensor: ViewM(), example_inputs, {"select_scatter": "buf"} ) + def test_quantize_buffer_filled_in_place_through_view(self): + class M(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = torch.nn.Conv2d(3, 3, 3) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + buf = x.new_zeros(1, 3, 8, 8) + buf.select(0, 0).copy_(x) + return self.conv(buf) + + class ViewQuantizer(Quantizer): + # Also quantizes the view, so its observers sit on the written chain. + def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: + act_qspec = QuantizationSpec( + dtype=torch.int8, + quant_min=-128, + quant_max=127, + qscheme=torch.per_tensor_affine, + is_dynamic=False, + observer_or_fake_quant_ctr=observer.default_observer, + ) + for node in model.graph.nodes: + if node.target in ( + torch.ops.aten.select.int, + torch.ops.aten.conv2d.default, + ): + node.meta["quantization_annotation"] = QuantizationAnnotation( + input_qspec_map={node.args[0]: act_qspec}, + output_qspec=act_qspec, + _annotated=True, + ) + return model + + def validate(self, model: torch.fx.GraphModule) -> None: + pass + + quantizers = { + "xnnpack": XNNPACKQuantizer().set_global( + get_symmetric_quantization_config() + ), + "view": ViewQuantizer(), + } + for name, quantizer in quantizers.items(): + with self.subTest(quantizer=name): + torch.manual_seed(0) + eager = M().eval() + x = torch.randn(3, 8, 8) + m = torch.export.export(eager, (x,), strict=True).module() + m = prepare_pt2e(m, quantizer) + for _ in range(4): + m(torch.randn(3, 8, 8)) + m = convert_pt2e(m) + # Without the image in `buf`, the output would not depend on x. + self.assertGreater(compute_error(eager(x), m(x)).item(), 20) + def _prepare_mutable_buffer_quantization( self, module: torch.nn.Module, diff --git a/torchao/quantization/pt2e/quantize_pt2e.py b/torchao/quantization/pt2e/quantize_pt2e.py index 8d9f0b97f0..53b744f9b0 100644 --- a/torchao/quantization/pt2e/quantize_pt2e.py +++ b/torchao/quantization/pt2e/quantize_pt2e.py @@ -22,6 +22,7 @@ _fuse_conv_bn_, _fuse_linear_bn_, _get_node_name_to_scope, + _replace_in_place_ops, get_arg, ) @@ -122,6 +123,7 @@ def calibrate(model, data_loader): _reject_set_grad_enabled_subgraph(model, "prepare_pt2e") original_graph_meta = model.meta node_name_to_scope = _get_node_name_to_scope(model) + _replace_in_place_ops(model) # TODO: check qconfig_mapping to make sure conv and bn are both configured # to be quantized before fusion # TODO: (maybe) rewrite this with subgraph_rewriter @@ -227,6 +229,7 @@ def train_loop(model, train_data): _reject_set_grad_enabled_subgraph(model, "prepare_qat_pt2e") original_graph_meta = model.meta node_name_to_scope = _get_node_name_to_scope(model) + _replace_in_place_ops(model) model = quantizer.transform_for_annotation(model) quantizer.annotate(model) quantizer.validate(model) diff --git a/torchao/quantization/pt2e/utils.py b/torchao/quantization/pt2e/utils.py index f5632ffd62..39ea913161 100644 --- a/torchao/quantization/pt2e/utils.py +++ b/torchao/quantization/pt2e/utils.py @@ -1093,6 +1093,96 @@ def _fuse_linear_bn_(m: GraphModule) -> None: m.recompile() +# Views that an in-place write can go through, mapped to the op that writes the +# result back into the viewed tensor. +_VIEW_TO_SCATTER = { + torch.ops.aten.select.int: torch.ops.aten.select_scatter.default, + torch.ops.aten.slice.Tensor: torch.ops.aten.slice_scatter.default, + torch.ops.aten.diagonal.default: torch.ops.aten.diagonal_scatter.default, + torch.ops.aten.as_strided.default: torch.ops.aten.as_strided_scatter.default, +} + + +def _get_functional_op(op: torch._ops.OpOverload) -> Optional[torch._ops.OpOverload]: + name = op._schema.name.split("::")[-1] + if not name.endswith("_"): + return None + packet = getattr(getattr(torch.ops, op.namespace), name[:-1], None) + if packet is None or op._overloadname not in packet.overloads(): + return None + return getattr(packet, op._overloadname) + + +def _get_arg0_alias_info(n: Node): + if not isinstance(n.target, torch._ops.OpOverload) or not n.args: + return None + return n.target._schema.arguments[0].alias_info + + +def _replace_in_place_ops(m: GraphModule) -> None: + """Replace in-place ops on tensors created in the graph with their functional + variants, e.g. `buf.select(0, 0).copy_(x)` with + `select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`. + + An observer reads its input where it is inserted, so it misses a later + in-place write to that input. Writes to graph inputs and buffers are kept, + as they have to stay visible outside the graph. + """ + for node in list(m.graph.nodes): + alias_info = _get_arg0_alias_info(node) + if alias_info is None or not alias_info.is_write: + continue + if not isinstance(node.args[0], Node): + continue + functional_op = _get_functional_op(node.target) + views = [] + base = node.args[0] + while base.target in _VIEW_TO_SCATTER: + views.append(base) + base = base.args[0] + if ( + functional_op is None + or not isinstance(base.target, torch._ops.OpOverload) + or any(r.alias_info for r in base.target._schema.returns) + ): + continue + # Aliases of `base` created before the write and used after it would + # need to be recomputed from the new value; leave those writes in place. + aliases = [] + to_visit = [base] + while to_visit: + n = to_visit.pop() + for user in n.users: + if _get_arg0_alias_info(user) is not None and user.args[0] is n: + aliases.append(user) + to_visit.append(user) + if any( + user > node + for alias in aliases + if alias < node and alias not in views + for user in alias.users + ): + continue + + with m.graph.inserting_before(node): + new_value = m.graph.call_function(functional_op, node.args, node.kwargs) + new_value.meta = dict(node.meta) + replacements = [(node, new_value), (node.args[0], new_value)] + for view in views: + parent = view.args[0] + new_value = m.graph.call_function( + _VIEW_TO_SCATTER[view.target], + (parent, new_value, *view.args[1:]), + view.kwargs, + ) + new_value.meta = {**node.meta, "val": parent.meta.get("val")} + replacements.append((parent, new_value)) + for old, new in replacements: + old.replace_all_uses_with(new, delete_user_cb=lambda user: user > node) + m.graph.erase_node(node) + m.recompile() + + def _get_node_name_to_scope(model: GraphModule) -> dict[str, tuple[str, type]]: # TODO: move this information to fx node itself node_name_to_scope: dict[str, tuple[str, type]] = {}