Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions test/quantization/pt2e/test_quantize_pt2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -3523,6 +3523,62 @@ def forward(self, x: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
ViewM(), example_inputs, {"select_scatter": "buf"}
)

def test_quantize_buffer_filled_in_place_through_view(self):
class M(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.conv = torch.nn.Conv2d(3, 3, 3)

def forward(self, x: torch.Tensor) -> torch.Tensor:
buf = x.new_zeros(1, 3, 8, 8)
buf.select(0, 0).copy_(x)
return self.conv(buf)

class ViewQuantizer(Quantizer):
# Also quantizes the view, so its observers sit on the written chain.
def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
act_qspec = QuantizationSpec(
dtype=torch.int8,
quant_min=-128,
quant_max=127,
qscheme=torch.per_tensor_affine,
is_dynamic=False,
observer_or_fake_quant_ctr=observer.default_observer,
)
for node in model.graph.nodes:
if node.target in (
torch.ops.aten.select.int,
torch.ops.aten.conv2d.default,
):
node.meta["quantization_annotation"] = QuantizationAnnotation(
input_qspec_map={node.args[0]: act_qspec},
output_qspec=act_qspec,
_annotated=True,
)
return model

def validate(self, model: torch.fx.GraphModule) -> None:
pass

quantizers = {
"xnnpack": XNNPACKQuantizer().set_global(
get_symmetric_quantization_config()
),
"view": ViewQuantizer(),
}
for name, quantizer in quantizers.items():
with self.subTest(quantizer=name):
torch.manual_seed(0)
eager = M().eval()
x = torch.randn(3, 8, 8)
m = torch.export.export(eager, (x,), strict=True).module()
m = prepare_pt2e(m, quantizer)
for _ in range(4):
m(torch.randn(3, 8, 8))
m = convert_pt2e(m)
# Without the image in `buf`, the output would not depend on x.
self.assertGreater(compute_error(eager(x), m(x)).item(), 20)

def _prepare_mutable_buffer_quantization(
self,
module: torch.nn.Module,
Expand Down
3 changes: 3 additions & 0 deletions torchao/quantization/pt2e/quantize_pt2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
_fuse_conv_bn_,
_fuse_linear_bn_,
_get_node_name_to_scope,
_replace_in_place_ops,
get_arg,
)

Expand Down Expand Up @@ -122,6 +123,7 @@ def calibrate(model, data_loader):
_reject_set_grad_enabled_subgraph(model, "prepare_pt2e")
original_graph_meta = model.meta
node_name_to_scope = _get_node_name_to_scope(model)
_replace_in_place_ops(model)
# TODO: check qconfig_mapping to make sure conv and bn are both configured
# to be quantized before fusion
# TODO: (maybe) rewrite this with subgraph_rewriter
Expand Down Expand Up @@ -227,6 +229,7 @@ def train_loop(model, train_data):
_reject_set_grad_enabled_subgraph(model, "prepare_qat_pt2e")
original_graph_meta = model.meta
node_name_to_scope = _get_node_name_to_scope(model)
_replace_in_place_ops(model)
model = quantizer.transform_for_annotation(model)
quantizer.annotate(model)
quantizer.validate(model)
Expand Down
90 changes: 90 additions & 0 deletions torchao/quantization/pt2e/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1093,6 +1093,96 @@ def _fuse_linear_bn_(m: GraphModule) -> None:
m.recompile()


# Views that an in-place write can go through, mapped to the op that writes the
# result back into the viewed tensor.
_VIEW_TO_SCATTER = {
torch.ops.aten.select.int: torch.ops.aten.select_scatter.default,
torch.ops.aten.slice.Tensor: torch.ops.aten.slice_scatter.default,
torch.ops.aten.diagonal.default: torch.ops.aten.diagonal_scatter.default,
torch.ops.aten.as_strided.default: torch.ops.aten.as_strided_scatter.default,
}


def _get_functional_op(op: torch._ops.OpOverload) -> Optional[torch._ops.OpOverload]:
name = op._schema.name.split("::")[-1]
if not name.endswith("_"):
return None
packet = getattr(getattr(torch.ops, op.namespace), name[:-1], None)
if packet is None or op._overloadname not in packet.overloads():
return None
return getattr(packet, op._overloadname)


def _get_arg0_alias_info(n: Node):
if not isinstance(n.target, torch._ops.OpOverload) or not n.args:
return None
return n.target._schema.arguments[0].alias_info


def _replace_in_place_ops(m: GraphModule) -> None:
"""Replace in-place ops on tensors created in the graph with their functional
variants, e.g. `buf.select(0, 0).copy_(x)` with
`select_scatter(buf, copy(buf.select(0, 0), x), 0, 0)`.

An observer reads its input where it is inserted, so it misses a later
in-place write to that input. Writes to graph inputs and buffers are kept,
as they have to stay visible outside the graph.
"""
for node in list(m.graph.nodes):
alias_info = _get_arg0_alias_info(node)
if alias_info is None or not alias_info.is_write:
continue
if not isinstance(node.args[0], Node):
continue
functional_op = _get_functional_op(node.target)
views = []
base = node.args[0]
while base.target in _VIEW_TO_SCATTER:
views.append(base)
base = base.args[0]
if (
functional_op is None
or not isinstance(base.target, torch._ops.OpOverload)
or any(r.alias_info for r in base.target._schema.returns)
):
continue
# Aliases of `base` created before the write and used after it would
# need to be recomputed from the new value; leave those writes in place.
aliases = []
to_visit = [base]
while to_visit:
n = to_visit.pop()
for user in n.users:
if _get_arg0_alias_info(user) is not None and user.args[0] is n:
aliases.append(user)
to_visit.append(user)
if any(
user > node
for alias in aliases
if alias < node and alias not in views
for user in alias.users
):
continue

with m.graph.inserting_before(node):
new_value = m.graph.call_function(functional_op, node.args, node.kwargs)
new_value.meta = dict(node.meta)
replacements = [(node, new_value), (node.args[0], new_value)]
for view in views:
parent = view.args[0]
new_value = m.graph.call_function(
_VIEW_TO_SCATTER[view.target],
(parent, new_value, *view.args[1:]),
view.kwargs,
)
new_value.meta = {**node.meta, "val": parent.meta.get("val")}
replacements.append((parent, new_value))
for old, new in replacements:
old.replace_all_uses_with(new, delete_user_cb=lambda user: user > node)
m.graph.erase_node(node)
m.recompile()


def _get_node_name_to_scope(model: GraphModule) -> dict[str, tuple[str, type]]:
# TODO: move this information to fx node itself
node_name_to_scope: dict[str, tuple[str, type]] = {}
Expand Down