Skip to content

Commit 5c3ef5f

Browse files
committed
Preserve FQ linear and quantize constant bias for ESP
Preserve linear through quantization and shared FQ fusion, using per-channel int8 weights and constant int32 biases derived from input and weight scales. Separate shared constant-bias uses before observation so each call gets its own qparams while float consumers retain the original bias. Runtime weights are rejected before shared per-channel fusion. Runtime biases remain float and convolution policy is unchanged. Review this quantizer contract before the following linear implementation commit. The two are intended to land together: shared FQ decomposition alone restores aten.linear, which has no portable kernel. The following commit lowers supported linear calls and reports unsupported ones at export. Validation: 12 focused quantizer tests and 110 existing pass-manager/operator tests passed. Lintrunner passed. Authored with OpenAI Codex.
1 parent a3ded58 commit 5c3ef5f

2 files changed

Lines changed: 289 additions & 11 deletions

File tree

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 71 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# LICENSE file in the root directory of this source tree.
66

77
import torch
8-
from executorch.backends.transforms.permute_pass_utils import get_arg
8+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
99
from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
1010
get_symmetric_quantization_config,
1111
)
@@ -14,8 +14,67 @@
1414
get_core_aten_quantizers,
1515
OpQuantizer,
1616
)
17-
from torchao.quantization.pt2e.quantizer import SharedQuantizationSpec
18-
from torchao.quantization.pt2e.quantizer.utils import annotate_output_qspec
17+
from torchao.quantization.pt2e.quantizer import (
18+
DerivedQuantizationSpec,
19+
SharedQuantizationSpec,
20+
)
21+
from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY
22+
from torchao.quantization.pt2e.quantizer.utils import (
23+
annotate_input_qspec_map,
24+
annotate_output_qspec,
25+
)
26+
27+
28+
def _derive_bias_qparams(observers):
29+
input_scale, _ = observers[0].calculate_qparams()
30+
weight_scale, _ = observers[1].calculate_qparams()
31+
return input_scale * weight_scale, torch.zeros_like(weight_scale, dtype=torch.int32)
32+
33+
34+
class _LinearQuantizer(OpQuantizer):
35+
def __init__(self, config):
36+
super().__init__(
37+
torch.ops.aten.linear.default,
38+
torch.ops.fused_quant.linear.default,
39+
config,
40+
weight_names=("weight",),
41+
other_names=("bias",),
42+
)
43+
44+
def annotate_node(self, node: torch.fx.Node) -> None:
45+
if get_arg(node, "weight", torch.fx.Node).op != "get_attr":
46+
raise ValueError(f"ESP-NN linear '{node.name}' requires constant weights")
47+
super().annotate_node(node)
48+
bias = get_arg(node, "bias")
49+
if not isinstance(bias, torch.fx.Node) or bias.op != "get_attr":
50+
return
51+
if len(bias.users) > 1:
52+
# PT2E's implicit sharing ignores DerivedQuantizationSpec inputs.
53+
with node.graph.inserting_before(node):
54+
bias_copy = node.graph.call_function(
55+
torch.ops.aten.clone.default, (bias,)
56+
)
57+
bias_copy.meta = bias.meta.copy()
58+
bias_copy.meta.pop(Q_ANNOTATION_KEY, None)
59+
set_arg(node, "bias", bias_copy)
60+
bias = bias_copy
61+
annotate_input_qspec_map(
62+
node,
63+
bias,
64+
DerivedQuantizationSpec(
65+
derived_from=[
66+
(get_arg(node, "input"), node),
67+
(get_arg(node, "weight"), node),
68+
],
69+
derive_qparams_fn=_derive_bias_qparams,
70+
dtype=torch.int32,
71+
quant_min=-(1 << 31),
72+
# INT32_MAX rounds to 2**31 in PT2E's float32 clamp before the cast.
73+
quant_max=(1 << 31) - 128,
74+
qscheme=torch.per_channel_symmetric,
75+
ch_axis=0,
76+
),
77+
)
1978

2079

2180
class _AvgPoolQuantizer(OpQuantizer):
@@ -28,26 +87,27 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
2887
"""Fused Quant quantizer for the Espressif operator-library backend.
2988
3089
Uses int8 per-tensor affine activations and symmetric weights, with
31-
per-channel weights for convolution. AvgPool2d without divisor_override
32-
shares input and output qparams. EspPassManager lowers supported
33-
Fused Quant operators to ESP-NN and decomposes the rest for portable
34-
execution. Use EspNNQuantizer for the EspNNBackend delegate instead.
90+
per-channel weights for linear and convolution. Constant linear biases use
91+
int32 with scales derived from the input and weight observers.
92+
AvgPool2d without divisor_override shares input and output qparams.
93+
EspPassManager lowers supported Fused Quant operators to ESP-NN and
94+
decomposes the rest for portable execution. Use EspNNQuantizer for the
95+
EspNNBackend delegate instead.
3596
"""
3697

3798
def __init__(self) -> None:
3899
config = get_symmetric_quantization_config(is_per_channel=False)
100+
weighted_config = get_symmetric_quantization_config(is_per_channel=True)
39101
super().__init__(
40102
[
103+
_LinearQuantizer(weighted_config),
41104
_AvgPoolQuantizer(
42105
torch.ops.aten.avg_pool2d.default,
43106
torch.ops.fused_quant.avg_pool2d.default,
44107
config,
45108
node_filter=lambda node: get_arg(node, "divisor_override") is None,
46109
),
47-
*get_core_aten_quantizers(
48-
config,
49-
get_symmetric_quantization_config(is_per_channel=True),
50-
),
110+
*get_core_aten_quantizers(config, weighted_config),
51111
]
52112
)
53113

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import copy
8+
import math
9+
10+
import pytest
11+
import torch
12+
from executorch.backends.espressif.quantizer import get_esp_fused_quant_quantizer
13+
from executorch.backends.espressif.test.tester import EspTester
14+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
15+
from executorch.backends.transforms.permute_pass_utils import get_arg
16+
from torchao.quantization.pt2e.quantizer import QuantizationAnnotation
17+
from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY
18+
19+
20+
@pytest.fixture
21+
def linear():
22+
model = torch.nn.Linear(2, 3).eval()
23+
with torch.no_grad():
24+
model.weight.copy_(torch.tensor([[0.2, -0.1], [0.03, 0.05], [-0.4, 0.1]]))
25+
model.bias.copy_(torch.tensor([0.125, -0.3, 0.6]))
26+
return model
27+
28+
29+
def _quantize(model, inputs):
30+
program = EspTester(model, inputs).export_fused_quant().get_artifact()
31+
interpreter = torch.fx.Interpreter(program.module(), garbage_collect_values=False)
32+
interpreter.run(*inputs)
33+
return interpreter
34+
35+
36+
def _check_bias(interpreter, node, original_bias):
37+
values = interpreter.env
38+
inp = get_qparams_from_node(node, "inp")
39+
weight = get_qparams_from_node(node, "weight")
40+
bias = get_qparams_from_node(node, "bias")
41+
scales = values[bias.scale]
42+
torch.testing.assert_close(
43+
scales, values[inp.scale] * values[weight.scale].flatten(), atol=0, rtol=0
44+
)
45+
assert torch.count_nonzero(values[bias.zero_point]) == 0
46+
stored = values[get_arg(node, "bias")]
47+
assert stored.dtype == torch.int32
48+
expected = torch.round(original_bias / scales).clamp(bias.quant_min, bias.quant_max)
49+
torch.testing.assert_close(stored, expected.to(torch.int32), atol=0, rtol=0)
50+
51+
52+
@pytest.mark.parametrize("shape", [(2,), (3, 2), (2, 3, 2)])
53+
def test_linear_preservation_and_int32_bias(linear, shape):
54+
inputs = (torch.linspace(-2, 3, math.prod(shape)).reshape(shape),)
55+
interpreter = _quantize(linear, inputs)
56+
(node,) = interpreter.module.graph.find_nodes(
57+
op="call_function", target=torch.ops.fused_quant.linear.default
58+
)
59+
weight = interpreter.env[get_arg(node, "weight")]
60+
weight_qparams = get_qparams_from_node(node, "weight")
61+
assert weight.dtype == torch.int8
62+
assert interpreter.env[weight_qparams.scale].shape == (3, 1)
63+
assert torch.count_nonzero(interpreter.env[weight_qparams.zero_point]) == 0
64+
_check_bias(interpreter, node, linear.bias)
65+
assert interpreter.env[node].shape == (*shape[:-1], 3)
66+
67+
68+
def test_linear_without_bias():
69+
interpreter = _quantize(torch.nn.Linear(2, 3, bias=False), (torch.ones(1, 2),))
70+
(node,) = interpreter.module.graph.find_nodes(
71+
op="call_function", target=torch.ops.fused_quant.linear.default
72+
)
73+
assert get_arg(node, "bias") is None
74+
assert get_qparams_from_node(node, "bias") is None
75+
76+
77+
def test_runtime_linear_weights_report_error():
78+
class Model(torch.nn.Module):
79+
def forward(self, x, weight):
80+
return torch.nn.functional.linear(x, weight)
81+
82+
module = torch.export.export(Model(), (torch.ones(2, 4), torch.ones(3, 4))).module()
83+
with pytest.raises(ValueError, match="ESP-NN linear.*requires constant weights"):
84+
get_esp_fused_quant_quantizer().annotate(module)
85+
86+
87+
@pytest.mark.parametrize("shared_weight", [False, True])
88+
def test_shared_bias_has_independent_scales_and_preserves_float_user(
89+
linear, shared_weight
90+
):
91+
class Model(torch.nn.Module):
92+
def __init__(self):
93+
super().__init__()
94+
self.first = linear
95+
self.second = copy.deepcopy(linear)
96+
self.second.bias = linear.bias
97+
if shared_weight:
98+
self.second.weight = linear.weight
99+
100+
def forward(self, x, y):
101+
return self.first(x), self.second(y), self.first.bias
102+
103+
x = torch.tensor([[-2.0, 3.0]])
104+
interpreter = _quantize(Model().eval(), (x, x * 8))
105+
nodes = list(
106+
interpreter.module.graph.find_nodes(
107+
op="call_function", target=torch.ops.fused_quant.linear.default
108+
)
109+
)
110+
assert len(nodes) == 2
111+
for node in nodes:
112+
_check_bias(interpreter, node, linear.bias)
113+
assert get_arg(nodes[0], "bias") is not get_arg(nodes[1], "bias")
114+
scales = [interpreter.env[get_qparams_from_node(n, "bias").scale] for n in nodes]
115+
torch.testing.assert_close(scales[1], scales[0] * 8, atol=0, rtol=0)
116+
assert not list(
117+
interpreter.module.graph.find_nodes(
118+
op="call_function", target=torch.ops.aten.clone.default
119+
)
120+
)
121+
output = next(n for n in interpreter.module.graph.nodes if n.op == "output")
122+
torch.testing.assert_close(interpreter.env[output][2], linear.bias, atol=0, rtol=0)
123+
124+
125+
def test_shared_runtime_bias_stays_float(linear):
126+
class Model(torch.nn.Module):
127+
def __init__(self):
128+
super().__init__()
129+
self.weight = linear.weight
130+
131+
def forward(self, x, y, bias):
132+
return (
133+
torch.nn.functional.linear(x, self.weight, bias),
134+
torch.nn.functional.linear(y, self.weight, bias),
135+
bias,
136+
)
137+
138+
x = torch.tensor([[-2.0, 3.0]])
139+
interpreter = _quantize(Model().eval(), (x, x * 8, linear.bias.detach()))
140+
nodes = list(
141+
interpreter.module.graph.find_nodes(
142+
op="call_function", target=torch.ops.fused_quant.linear.default
143+
)
144+
)
145+
assert len(nodes) == 2
146+
for node in nodes:
147+
bias = get_arg(node, "bias")
148+
assert bias.op == "placeholder"
149+
assert interpreter.env[bias].dtype == torch.float32
150+
assert get_qparams_from_node(node, "bias") is None
151+
assert get_arg(nodes[0], "bias") is get_arg(nodes[1], "bias")
152+
assert not list(
153+
interpreter.module.graph.find_nodes(
154+
op="call_function", target=torch.ops.aten.clone.default
155+
)
156+
)
157+
158+
159+
def test_bias_int32_boundaries(linear):
160+
with torch.no_grad():
161+
linear.weight.zero_()
162+
linear.bias.copy_(torch.tensor([127.0, 128.0, -128.0]))
163+
interpreter = _quantize(linear, (torch.zeros(1, 2),))
164+
(node,) = interpreter.module.graph.find_nodes(
165+
op="call_function", target=torch.ops.fused_quant.linear.default
166+
)
167+
torch.testing.assert_close(
168+
interpreter.env[get_arg(node, "bias")],
169+
torch.tensor([2130706432, 2147483520, -2147483648], dtype=torch.int32),
170+
atol=0,
171+
rtol=0,
172+
)
173+
174+
175+
def test_bias_rounding_is_visible_in_fused_reference(linear):
176+
with torch.no_grad():
177+
linear.weight.copy_(torch.tensor([[1.0, 0.0]]).expand(3, -1))
178+
linear.bias.fill_(0.1)
179+
inputs = (torch.tensor([[0.0, 100000.0], [0.0, -100000.0]]),)
180+
interpreter = _quantize(linear, inputs)
181+
torch.testing.assert_close(linear(*inputs), torch.full((2, 3), 0.1), atol=0, rtol=0)
182+
torch.testing.assert_close(
183+
interpreter.module(*inputs), torch.zeros(2, 3), atol=0, rtol=0
184+
)
185+
186+
187+
def test_bias_annotation_preserves_other_quantizers(linear):
188+
class Model(torch.nn.Module):
189+
def __init__(self):
190+
super().__init__()
191+
self.linear = linear
192+
193+
def forward(self, x):
194+
return self.linear(x), self.linear.bias
195+
196+
module = torch.export.export(Model().eval(), (torch.ones(1, 2),)).module()
197+
(node,) = module.graph.find_nodes(
198+
op="call_function", target=torch.ops.aten.linear.default
199+
)
200+
annotation = QuantizationAnnotation(_annotated=True)
201+
node.meta[Q_ANNOTATION_KEY] = annotation
202+
bias = get_arg(node, "bias")
203+
get_esp_fused_quant_quantizer().annotate(module)
204+
assert node.meta[Q_ANNOTATION_KEY] is annotation
205+
assert not annotation.input_qspec_map
206+
assert get_arg(node, "bias") is bias
207+
assert not list(
208+
module.graph.find_nodes(op="call_function", target=torch.ops.aten.clone.default)
209+
)
210+
211+
212+
def test_convolution_bias_remains_float():
213+
interpreter = _quantize(torch.nn.Conv2d(2, 3, 1), (torch.ones(1, 2, 2, 2),))
214+
(node,) = interpreter.module.graph.find_nodes(
215+
op="call_function", target=torch.ops.fused_quant.convolution.default
216+
)
217+
assert interpreter.env[get_arg(node, "bias")].dtype == torch.float32
218+
assert get_qparams_from_node(node, "bias") is None

0 commit comments

Comments
 (0)