Skip to content

Commit 79f0568

Browse files
committed
[Espressif] Prepare Fused Quant linear annotation and int32 bias
Preserve linear through quantization and shared FQ fusion, using per-channel int8 weights and constant int32 biases derived from input and weight scales. Separate shared constant-bias uses before observation so each call gets its own qparams while float consumers retain the original bias. Runtime weights stay float because shared per-channel fusion requires constants; runtime biases remain float. Enable this in the public quantizer and use shared portable decomposition until the following ESP-NN kernel is introduced. Register portable matrix kernels in the S3 test runtime. Bias tests exercise the public quantizer directly. Authored with OpenAI Codex.
1 parent b4cf440 commit 79f0568

4 files changed

Lines changed: 300 additions & 13 deletions

File tree

‎.ci/scripts/test_espressif.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ python examples/espressif/export_espnn_test_programs.py \
6868
espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \
6969
| tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -)
7070
if [[ "${espnn_kernels}" == optimized ]]; then
71-
espnn_ops="${espnn_ops},aten::sin.out,aten::relu.out,aten::max_pool2d_with_indices.out,aten::avg_pool2d.out,aten::permute_copy.out,aten::mul.Scalar_out"
71+
espnn_ops="${espnn_ops},aten::sin.out,aten::relu.out,aten::max_pool2d_with_indices.out,aten::avg_pool2d.out,aten::permute_copy.out,aten::addmm.out,aten::mm.out,aten::mul.Scalar_out"
7272
fi
7373

7474
cmake --preset esp-baremetal -G Ninja -B "${build_dir}/runtime" \

‎backends/espressif/README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,10 @@ explicit layout operations rather than dim-order operators, matching the
167167
ESP-IDF runtime and the operator implementation tests.
168168
When checking supported int8 operators, lowering rejects malformed per-tensor
169169
qparams, such as non-positive scales or out-of-range zero points. Valid
170-
unsupported configurations remain portable.
170+
unsupported configurations use shared portable fallback, including linear calls
171+
with runtime weights or bias and shapes outside ESP-NN's bounds. Linear calls
172+
without constant weights stay unquantized because FQ's per-channel fusion
173+
requires constant weight tensors.
171174
ESP-NN pooling requires matching input and output qparams. A shared optimization
172175
that absorbs a following requantization can change the output qparams and leave
173176
that pool on the portable path.

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 72 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# LICENSE file in the root directory of this source tree.
66

77
import torch
8-
from executorch.backends.transforms.permute_pass_utils import get_arg
8+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
99
from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
1010
get_symmetric_quantization_config,
1111
)
@@ -14,8 +14,68 @@
1414
get_core_aten_quantizers,
1515
OpQuantizer,
1616
)
17-
from torchao.quantization.pt2e.quantizer import SharedQuantizationSpec
18-
from torchao.quantization.pt2e.quantizer.utils import annotate_output_qspec
17+
from torchao.quantization.pt2e.quantizer import (
18+
DerivedQuantizationSpec,
19+
SharedQuantizationSpec,
20+
)
21+
from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY
22+
from torchao.quantization.pt2e.quantizer.utils import (
23+
annotate_input_qspec_map,
24+
annotate_output_qspec,
25+
)
26+
27+
28+
def _derive_bias_qparams(observers):
29+
input_scale, _ = observers[0].calculate_qparams()
30+
weight_scale, _ = observers[1].calculate_qparams()
31+
return input_scale * weight_scale, torch.zeros_like(weight_scale, dtype=torch.int32)
32+
33+
34+
class _LinearQuantizer(OpQuantizer):
35+
def __init__(self, config):
36+
super().__init__(
37+
torch.ops.aten.linear.default,
38+
torch.ops.fused_quant.linear.default,
39+
config,
40+
weight_names=("weight",),
41+
other_names=("bias",),
42+
)
43+
44+
def annotate_node(self, node: torch.fx.Node) -> None:
45+
# FQ fusion requires constant per-channel weight tensors.
46+
if get_arg(node, "weight", torch.fx.Node).op != "get_attr":
47+
return
48+
super().annotate_node(node)
49+
bias = get_arg(node, "bias")
50+
if not isinstance(bias, torch.fx.Node) or bias.op != "get_attr":
51+
return
52+
if len(bias.users) > 1:
53+
# PT2E's implicit sharing ignores DerivedQuantizationSpec inputs.
54+
with node.graph.inserting_before(node):
55+
bias_copy = node.graph.call_function(
56+
torch.ops.aten.clone.default, (bias,)
57+
)
58+
bias_copy.meta = bias.meta.copy()
59+
bias_copy.meta.pop(Q_ANNOTATION_KEY, None)
60+
set_arg(node, "bias", bias_copy)
61+
bias = bias_copy
62+
annotate_input_qspec_map(
63+
node,
64+
bias,
65+
DerivedQuantizationSpec(
66+
derived_from=[
67+
(get_arg(node, "input"), node),
68+
(get_arg(node, "weight"), node),
69+
],
70+
derive_qparams_fn=_derive_bias_qparams,
71+
dtype=torch.int32,
72+
quant_min=-(1 << 31),
73+
# INT32_MAX rounds to 2**31 in PT2E's float32 clamp before the cast.
74+
quant_max=(1 << 31) - 128,
75+
qscheme=torch.per_channel_symmetric,
76+
ch_axis=0,
77+
),
78+
)
1979

2080

2181
class _AvgPoolQuantizer(OpQuantizer):
@@ -28,26 +88,27 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
2888
"""Fused Quant quantizer for the Espressif operator-library backend.
2989
3090
Uses int8 per-tensor affine activations and symmetric weights, with
31-
per-channel weights for convolution. AvgPool2d without divisor_override
32-
shares input and output qparams. EspPassManager lowers supported
33-
Fused Quant operators to ESP-NN and decomposes the rest for portable
34-
execution. Use EspNNQuantizer for the EspNNBackend delegate instead.
91+
per-channel weights for linear and convolution. Constant linear biases use
92+
int32 with scales derived from the input and weight observers.
93+
AvgPool2d without divisor_override shares input and output qparams.
94+
EspPassManager lowers supported Fused Quant operators to ESP-NN and
95+
decomposes the rest for portable execution. Use EspNNQuantizer for the
96+
EspNNBackend delegate instead.
3597
"""
3698

3799
def __init__(self) -> None:
38100
config = get_symmetric_quantization_config(is_per_channel=False)
101+
weighted_config = get_symmetric_quantization_config(is_per_channel=True)
39102
super().__init__(
40103
[
104+
_LinearQuantizer(weighted_config),
41105
_AvgPoolQuantizer(
42106
torch.ops.aten.avg_pool2d.default,
43107
torch.ops.fused_quant.avg_pool2d.default,
44108
config,
45109
node_filter=lambda node: get_arg(node, "divisor_override") is None,
46110
),
47-
*get_core_aten_quantizers(
48-
config,
49-
get_symmetric_quantization_config(is_per_channel=True),
50-
),
111+
*get_core_aten_quantizers(config, weighted_config),
51112
]
52113
)
53114

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import copy
8+
import math
9+
10+
import pytest
11+
import torch
12+
from executorch.backends.espressif.quantizer import get_esp_fused_quant_quantizer
13+
from executorch.backends.espressif.test.tester import EspTester
14+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
15+
from executorch.backends.transforms.permute_pass_utils import get_arg
16+
from torchao.quantization.pt2e.quantizer import QuantizationAnnotation
17+
from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY
18+
19+
20+
@pytest.fixture
21+
def linear():
22+
model = torch.nn.Linear(2, 3).eval()
23+
with torch.no_grad():
24+
model.weight.copy_(torch.tensor([[0.2, -0.1], [0.03, 0.05], [-0.4, 0.1]]))
25+
model.bias.copy_(torch.tensor([0.125, -0.3, 0.6]))
26+
return model
27+
28+
29+
def _quantize(model, inputs):
30+
program = EspTester(model, inputs).export_fused_quant().get_artifact()
31+
interpreter = torch.fx.Interpreter(program.module(), garbage_collect_values=False)
32+
interpreter.run(*inputs)
33+
return interpreter
34+
35+
36+
def _check_bias(interpreter, node, original_bias):
37+
values = interpreter.env
38+
inp = get_qparams_from_node(node, "inp")
39+
weight = get_qparams_from_node(node, "weight")
40+
bias = get_qparams_from_node(node, "bias")
41+
scales = values[bias.scale]
42+
torch.testing.assert_close(
43+
scales, values[inp.scale] * values[weight.scale].flatten(), atol=0, rtol=0
44+
)
45+
assert torch.count_nonzero(values[bias.zero_point]) == 0
46+
stored = values[get_arg(node, "bias")]
47+
assert stored.dtype == torch.int32
48+
expected = torch.round(original_bias / scales).clamp(bias.quant_min, bias.quant_max)
49+
torch.testing.assert_close(stored, expected.to(torch.int32), atol=0, rtol=0)
50+
51+
52+
@pytest.mark.parametrize("shape", [(2,), (3, 2), (2, 3, 2)])
53+
def test_linear_preservation_and_int32_bias(linear, shape):
54+
inputs = (torch.linspace(-2, 3, math.prod(shape)).reshape(shape),)
55+
interpreter = _quantize(linear, inputs)
56+
(node,) = interpreter.module.graph.find_nodes(
57+
op="call_function", target=torch.ops.fused_quant.linear.default
58+
)
59+
weight = interpreter.env[get_arg(node, "weight")]
60+
weight_qparams = get_qparams_from_node(node, "weight")
61+
assert weight.dtype == torch.int8
62+
assert interpreter.env[weight_qparams.scale].shape == (3, 1)
63+
assert torch.count_nonzero(interpreter.env[weight_qparams.zero_point]) == 0
64+
_check_bias(interpreter, node, linear.bias)
65+
assert interpreter.env[node].shape == (*shape[:-1], 3)
66+
67+
68+
def test_linear_without_bias():
69+
interpreter = _quantize(torch.nn.Linear(2, 3, bias=False), (torch.ones(1, 2),))
70+
(node,) = interpreter.module.graph.find_nodes(
71+
op="call_function", target=torch.ops.fused_quant.linear.default
72+
)
73+
assert get_arg(node, "bias") is None
74+
assert get_qparams_from_node(node, "bias") is None
75+
76+
77+
def test_runtime_linear_weights_stay_float():
78+
class Model(torch.nn.Module):
79+
def forward(self, x, weight):
80+
return torch.nn.functional.linear(x, weight)
81+
82+
interpreter = _quantize(Model(), (torch.ones(2, 4), torch.ones(3, 4)))
83+
(node,) = interpreter.module.graph.find_nodes(
84+
op="call_function", target=torch.ops.aten.linear.default
85+
)
86+
weight = get_arg(node, "weight")
87+
assert weight.op == "placeholder"
88+
assert interpreter.env[weight].dtype == torch.float32
89+
torch.testing.assert_close(interpreter.env[node], torch.full((2, 3), 4.0))
90+
91+
92+
@pytest.mark.parametrize("shared_weight", [False, True])
93+
def test_shared_bias_has_independent_scales_and_preserves_float_user(
94+
linear, shared_weight
95+
):
96+
class Model(torch.nn.Module):
97+
def __init__(self):
98+
super().__init__()
99+
self.first = linear
100+
self.second = copy.deepcopy(linear)
101+
self.second.bias = linear.bias
102+
if shared_weight:
103+
self.second.weight = linear.weight
104+
105+
def forward(self, x, y):
106+
return self.first(x), self.second(y), self.first.bias
107+
108+
x = torch.tensor([[-2.0, 3.0]])
109+
interpreter = _quantize(Model().eval(), (x, x * 8))
110+
nodes = list(
111+
interpreter.module.graph.find_nodes(
112+
op="call_function", target=torch.ops.fused_quant.linear.default
113+
)
114+
)
115+
assert len(nodes) == 2
116+
for node in nodes:
117+
_check_bias(interpreter, node, linear.bias)
118+
assert get_arg(nodes[0], "bias") is not get_arg(nodes[1], "bias")
119+
scales = [interpreter.env[get_qparams_from_node(n, "bias").scale] for n in nodes]
120+
torch.testing.assert_close(scales[1], scales[0] * 8, atol=0, rtol=0)
121+
assert not list(
122+
interpreter.module.graph.find_nodes(
123+
op="call_function", target=torch.ops.aten.clone.default
124+
)
125+
)
126+
output = next(n for n in interpreter.module.graph.nodes if n.op == "output")
127+
torch.testing.assert_close(interpreter.env[output][2], linear.bias, atol=0, rtol=0)
128+
129+
130+
def test_shared_runtime_bias_stays_float(linear):
131+
class Model(torch.nn.Module):
132+
def __init__(self):
133+
super().__init__()
134+
self.weight = linear.weight
135+
136+
def forward(self, x, y, bias):
137+
return (
138+
torch.nn.functional.linear(x, self.weight, bias),
139+
torch.nn.functional.linear(y, self.weight, bias),
140+
bias,
141+
)
142+
143+
x = torch.tensor([[-2.0, 3.0]])
144+
interpreter = _quantize(Model().eval(), (x, x * 8, linear.bias.detach()))
145+
nodes = list(
146+
interpreter.module.graph.find_nodes(
147+
op="call_function", target=torch.ops.fused_quant.linear.default
148+
)
149+
)
150+
assert len(nodes) == 2
151+
for node in nodes:
152+
bias = get_arg(node, "bias")
153+
assert bias.op == "placeholder"
154+
assert interpreter.env[bias].dtype == torch.float32
155+
assert get_qparams_from_node(node, "bias") is None
156+
assert get_arg(nodes[0], "bias") is get_arg(nodes[1], "bias")
157+
assert not list(
158+
interpreter.module.graph.find_nodes(
159+
op="call_function", target=torch.ops.aten.clone.default
160+
)
161+
)
162+
163+
164+
def test_bias_int32_boundaries(linear):
165+
with torch.no_grad():
166+
linear.weight.zero_()
167+
linear.bias.copy_(torch.tensor([127.0, 128.0, -128.0]))
168+
interpreter = _quantize(linear, (torch.zeros(1, 2),))
169+
(node,) = interpreter.module.graph.find_nodes(
170+
op="call_function", target=torch.ops.fused_quant.linear.default
171+
)
172+
torch.testing.assert_close(
173+
interpreter.env[get_arg(node, "bias")],
174+
torch.tensor([2130706432, 2147483520, -2147483648], dtype=torch.int32),
175+
atol=0,
176+
rtol=0,
177+
)
178+
179+
180+
def test_bias_rounding_is_visible_in_fused_reference(linear):
181+
with torch.no_grad():
182+
linear.weight.copy_(torch.tensor([[1.0, 0.0]]).expand(3, -1))
183+
linear.bias.fill_(0.1)
184+
inputs = (torch.tensor([[0.0, 100000.0], [0.0, -100000.0]]),)
185+
interpreter = _quantize(linear, inputs)
186+
torch.testing.assert_close(linear(*inputs), torch.full((2, 3), 0.1), atol=0, rtol=0)
187+
torch.testing.assert_close(
188+
interpreter.module(*inputs), torch.zeros(2, 3), atol=0, rtol=0
189+
)
190+
191+
192+
def test_bias_annotation_preserves_other_quantizers(linear):
193+
class Model(torch.nn.Module):
194+
def __init__(self):
195+
super().__init__()
196+
self.linear = linear
197+
198+
def forward(self, x):
199+
return self.linear(x), self.linear.bias
200+
201+
module = torch.export.export(Model().eval(), (torch.ones(1, 2),)).module()
202+
(node,) = module.graph.find_nodes(
203+
op="call_function", target=torch.ops.aten.linear.default
204+
)
205+
annotation = QuantizationAnnotation(_annotated=True)
206+
node.meta[Q_ANNOTATION_KEY] = annotation
207+
bias = get_arg(node, "bias")
208+
get_esp_fused_quant_quantizer().annotate(module)
209+
assert node.meta[Q_ANNOTATION_KEY] is annotation
210+
assert not annotation.input_qspec_map
211+
assert get_arg(node, "bias") is bias
212+
assert not list(
213+
module.graph.find_nodes(op="call_function", target=torch.ops.aten.clone.default)
214+
)
215+
216+
217+
def test_convolution_bias_remains_float():
218+
interpreter = _quantize(torch.nn.Conv2d(2, 3, 1), (torch.ones(1, 2, 2, 2),))
219+
(node,) = interpreter.module.graph.find_nodes(
220+
op="call_function", target=torch.ops.fused_quant.convolution.default
221+
)
222+
assert interpreter.env[get_arg(node, "bias")].dtype == torch.float32
223+
assert get_qparams_from_node(node, "bias") is None

0 commit comments

Comments
 (0)