Skip to content

Commit 56c4efc

Browse files
committed
[Espressif] Fuse linear ReLU and HardTanh patterns
Quantize single-consumer linear followed by ReLU or HardTanh, including ReLU6, at the pattern boundaries. Fold the final output qparams and clamp bounds into the existing ESP-NN fully connected call, preserving per-channel weights and derived int32 bias. Forked and separately quantized activations retain their original quantization boundaries. Read the quantizer and activation fold before the linear lowering and schema/wrapper updates. Shared dialect and S3 cases cover bias/no bias, forks, saturation, vector and higher-rank inputs, and narrower clamp grids with asymmetric, positive-only and negative-only HardTanh bounds. Authored with OpenAI Codex.
1 parent 2597b2d commit 56c4efc

8 files changed

Lines changed: 195 additions & 30 deletions

File tree

‎backends/espressif/ops/op_quantized_linear.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_linear_int8_out(
3030
const Tensor& output_shift,
3131
int64_t input_offset,
3232
int64_t output_offset,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
3537
const auto channels = weight.size(0);
@@ -71,8 +73,8 @@ Tensor& quantized_linear_int8_out(
7173
static_cast<int32_t>(output_offset),
7274
output_shift.const_data_ptr<int32_t>(),
7375
output_multiplier.const_data_ptr<int32_t>(),
74-
std::numeric_limits<int8_t>::min(),
75-
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(activation_min),
77+
static_cast<int32_t>(activation_max),
7678
static_cast<int32_t>(batches));
7779
return out;
7880
}

‎backends/espressif/ops/operators.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,13 @@
6868
)
6969
lib.define(
7070
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
71-
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
71+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
72+
"int activation_min, int activation_max) -> Tensor"
7273
)
7374
lib.define(
7475
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
7576
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
76-
"*, Tensor(a!) out) -> Tensor(a!)"
77+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
7778
)
7879

7980
lib.define(
@@ -247,14 +248,30 @@ def quantized_add_int8(
247248

248249
@register_fake("esp_nn::quantized_linear_int8")
249250
def quantized_linear_int8_meta(
250-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
251+
input,
252+
weight,
253+
bias,
254+
output_multiplier,
255+
output_shift,
256+
input_offset,
257+
output_offset,
258+
activation_min,
259+
activation_max,
251260
):
252261
return input.new_empty((*input.shape[:-1], weight.shape[0]))
253262

254263

255264
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
256265
def quantized_linear_int8(
257-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
266+
input,
267+
weight,
268+
bias,
269+
output_multiplier,
270+
output_shift,
271+
input_offset,
272+
output_offset,
273+
activation_min,
274+
activation_max,
258275
):
259276
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
260277
if bias is not None:
@@ -268,7 +285,7 @@ def quantized_linear_int8(
268285
],
269286
dim=-1,
270287
)
271-
return (result + output_offset).clamp(-128, 127).to(torch.int8)
288+
return (result + output_offset).clamp(activation_min, activation_max).to(torch.int8)
272289

273290

274291
@register_fake("esp_nn::quantized_conv2d_int8")
@@ -325,6 +342,8 @@ def quantized_conv2d_int8(
325342
output_shift,
326343
0,
327344
output_offset,
345+
-128,
346+
127,
328347
)
329348

330349

‎backends/espressif/ops/operators.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
- arg_meta: null
4141
kernel_name: espressif::quantized_avg_pool2d_int8_out
4242

43-
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
43+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
4444
variants: function
4545
kernels:
4646
- arg_meta: null

‎backends/espressif/passes/fold_activations.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717

1818
class FoldActivations(ExportedProgramPassBase):
19-
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
19+
"""Fold grouped clamps into the producer's output quantization."""
2020

2121
def call(self, exported_program):
2222
graph = exported_program.graph
@@ -43,6 +43,7 @@ def call(self, exported_program):
4343
exir_ops.edge.fused_quant.add.default,
4444
exir_ops.edge.fused_quant.sub.default,
4545
exir_ops.edge.fused_quant.mul.default,
46+
exir_ops.edge.fused_quant.linear.default,
4647
)
4748
or len(producer.users) != 1
4849
or get_qparams_from_node(producer, "out") is not None

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,8 @@ def _linear_kernel_args(program, node, constants):
404404
"input and output must be nonempty, static, contiguous int8 tensors"
405405
)
406406
input_params, output_params = (
407-
_qparams(program, node, name) for name in ("inp", "out")
407+
_qparams(program, node, name, allow_output_clamp=True)
408+
for name in ("inp", "out")
408409
)
409410
if input_params is None or output_params is None:
410411
raise ValueError("input and output require constant per-tensor int8 qparams")
@@ -436,6 +437,8 @@ def _linear_kernel_args(program, node, constants):
436437
shift_node,
437438
-input_params.zp,
438439
output_params.zp,
440+
get_arg(node, "out_quant_min"),
441+
get_arg(node, "out_quant_max"),
439442
)
440443

441444

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
107107
Conv2d, and depthwise Conv2d biases use
108108
int32 with scales derived from the input and weight observers.
109109
AvgPool2d without divisor_override shares input and output qparams.
110-
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
111-
only their boundaries.
110+
Single-consumer binary or linear followed by ReLU or HardTanh (including
111+
ReLU6) patterns observe only their boundaries.
112112
EspPassManager lowers supported Fused Quant operators to ESP-NN and
113113
decomposes the rest for portable execution. Use EspNNQuantizer for the
114114
EspNNBackend delegate instead.
@@ -163,16 +163,19 @@ def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
163163
torch.ops.aten.add.Tensor,
164164
torch.ops.aten.sub.Tensor,
165165
torch.ops.aten.mul.Tensor,
166+
torch.ops.aten.linear.default,
166167
)
167168
or len(producer.users) != 1
168-
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
169169
or Q_ANNOTATION_KEY in producer.meta
170170
or Q_ANNOTATION_KEY in activation.meta
171171
):
172172
continue
173-
if (
174-
producer.target != torch.ops.aten.mul.Tensor
175-
and get_arg(producer, "alpha") != 1
173+
if producer.target != torch.ops.aten.linear.default and (
174+
not isinstance(get_arg(producer, "other"), torch.fx.Node)
175+
or (
176+
producer.target != torch.ops.aten.mul.Tensor
177+
and get_arg(producer, "alpha") != 1
178+
)
176179
):
177180
continue
178181
groups.append((producer, activation))

‎backends/espressif/test/ops/test_linear.py‎

Lines changed: 146 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,24 @@
55
# LICENSE file in the root directory of this source tree.
66

77
from dataclasses import replace
8+
from functools import partial
89

910
import pytest
1011
import torch
1112
from executorch.backends.espressif.lowering import (
1213
get_esp_fused_quant_recipe,
1314
get_operator_library_report,
1415
)
16+
from executorch.backends.espressif.passes.fold_activations import FoldActivations
17+
from executorch.backends.espressif.quantizer import get_esp_fused_quant_quantizer
1518
from executorch.backends.espressif.test.tester import (
1619
EspTestCase,
1720
EspTester,
1821
ramp_tensor,
1922
)
2023
from executorch.backends.test.suite.operators.test_linear import Model
21-
from executorch.backends.transforms.permute_pass_utils import get_arg
24+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
25+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
2226
from executorch.backends.transforms.utils import get_constant
2327
from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
2428
get_symmetric_quantization_config,
@@ -33,6 +37,21 @@
3337

3438
_FQ_LINEAR = exir_ops.edge.fused_quant.linear.default
3539
_ESP_LINEAR = exir_ops.edge.esp_nn.quantized_linear_int8.default
40+
_ESP_LUT = exir_ops.edge.esp_nn.quantized_lut_int8.default
41+
_ACTIVATIONS = {
42+
"relu": (torch.nn.ReLU, (-7, 127)),
43+
"relu6": (torch.nn.ReLU6, (-7, 41)),
44+
"hardtanh": (torch.nn.Hardtanh, (-15, 1)),
45+
"hardtanh_asymmetric": (
46+
partial(torch.nn.Hardtanh, min_val=-2.0625, max_val=3.1875),
47+
(-23, 19),
48+
),
49+
"hardtanh_positive": (partial(torch.nn.Hardtanh, min_val=1, max_val=4), (1, 25)),
50+
"hardtanh_negative": (
51+
partial(torch.nn.Hardtanh, min_val=-5, max_val=-2),
52+
(-47, -23),
53+
),
54+
}
3655

3756

3857
class EspLinear(Model):
@@ -58,16 +77,27 @@ def forward(self, x):
5877
return result, torch.sin(result), torch.sin(x), self.linear.bias
5978

6079

61-
class EspLinearRelu(EspLinear):
62-
ops_before_transforms = {_FQ_LINEAR: 1, exir_ops.edge.fused_quant.relu.default: 1}
63-
ops_after_transforms = {
64-
_ESP_LINEAR: 1,
65-
exir_ops.edge.esp_nn.quantized_lut_int8.default: 1,
66-
exir_ops.edge.aten.relu.default: 0,
67-
}
80+
class EspLinearActivation(EspLinear):
81+
def __init__(self, activation, *, bias=True, fork=False):
82+
super().__init__(bias=bias)
83+
self.activation = _ACTIVATIONS[activation][0]()
84+
self.fused_activation = (
85+
exir_ops.edge.fused_quant.relu.default
86+
if activation == "relu"
87+
else exir_ops.edge.fused_quant.hard_tanh.default
88+
)
89+
self.fork = fork
90+
self.ops_before_transforms = {_FQ_LINEAR: 1, self.fused_activation: 1}
91+
self.ops_after_transforms = {
92+
_ESP_LINEAR: 1,
93+
self.fused_activation: 0,
94+
_ESP_LUT: int(fork),
95+
}
6896

6997
def forward(self, x):
70-
return torch.relu(self.linear(x))
98+
result = self.linear(x)
99+
activated = self.activation(result)
100+
return (activated, result, torch.sin(result)) if self.fork else activated
71101

72102

73103
class EspSharedLinear(EspLinear):
@@ -132,7 +162,6 @@ def forward(self, x, weight):
132162
"float_users": EspTestCase(
133163
EspLinearWithFloatUsers(), (ramp_tensor(-3, 5, (2, 17)),)
134164
),
135-
"relu": EspTestCase(EspLinearRelu(), (ramp_tensor(-3, 5, (2, 17)),)),
136165
"shared_bias": EspTestCase(
137166
EspSharedLinear(), (ramp_tensor(-3, 5, (2, 17)), ramp_tensor(-24, 40, (2, 17)))
138167
),
@@ -152,6 +181,38 @@ def forward(self, x, weight):
152181
calibration_samples=[(ramp_tensor(-3, 5, (2, 17)),)],
153182
),
154183
}
184+
test_cases.update(
185+
{
186+
f"{activation}{'_no_bias' if not bias else ''}": EspTestCase(
187+
EspLinearActivation(activation, bias=bias),
188+
(ramp_tensor(-30, 50, (5, 17)),),
189+
)
190+
for activation in _ACTIVATIONS
191+
for bias in (True, False)
192+
}
193+
)
194+
test_cases.update(
195+
{
196+
"relu_fork": EspTestCase(
197+
EspLinearActivation("relu", fork=True), (ramp_tensor(-3, 5, (2, 17)),)
198+
),
199+
"hardtanh_fork": EspTestCase(
200+
EspLinearActivation("hardtanh", fork=True),
201+
(ramp_tensor(-3, 5, (2, 17)),),
202+
),
203+
"relu6_vector": EspTestCase(
204+
EspLinearActivation("relu6"), (ramp_tensor(-30, 50, (17,)),)
205+
),
206+
"hardtanh_rank_3": EspTestCase(
207+
EspLinearActivation("hardtanh"), (ramp_tensor(-30, 50, (2, 3, 17)),)
208+
),
209+
"relu6_saturation": EspTestCase(
210+
EspLinearActivation("relu6"),
211+
(ramp_tensor(-30, 50, (2, 17)),),
212+
calibration_samples=[(ramp_tensor(-3, 5, (2, 17)),)],
213+
),
214+
}
215+
)
155216

156217

157218
@pytest.mark.parametrize("test_case", test_cases.values(), ids=test_cases.keys())
@@ -185,7 +246,7 @@ def test_implementation_linear(test_case, pytestconfig):
185246
test_case.model, test_case.example_inputs, test_case.calibration_samples
186247
)
187248
tester.test_implementation(
188-
pytestconfig.rootpath, rtol=torch.finfo(torch.float32).eps
249+
pytestconfig.rootpath, rtol=torch.finfo(torch.float32).eps, num_inferences=2
189250
)
190251

191252

@@ -204,7 +265,7 @@ def test_public_recipe_linear():
204265
assert session.get_executorch_program_manager().buffer
205266

206267

207-
@pytest.mark.parametrize("case_name", ["shared_bias", "large_bias"])
268+
@pytest.mark.parametrize("case_name", ["shared_bias", "large_bias", "relu", "hardtanh"])
208269
def test_lowering_reuses_quantized_bias_and_preserves_source_program(case_name):
209270
case = test_cases[case_name]
210271
tester = EspTester(case.model, case.example_inputs).export_fused_quant().to_edge()
@@ -226,6 +287,79 @@ def test_lowering_reuses_quantized_bias_and_preserves_source_program(case_name):
226287
source.validate()
227288

228289

290+
@pytest.mark.parametrize("activation", ["relu", "relu6", "hardtanh"])
291+
@pytest.mark.parametrize("fork", [False, True])
292+
def test_linear_activation_quantization_boundary(activation, fork):
293+
model = EspLinearActivation(activation, fork=fork)
294+
tester = EspTester(model, (ramp_tensor(-3, 5, (2, 17)),))
295+
tester.export_fused_quant().to_edge()
296+
graph = tester.get_artifact().exported_program().graph
297+
(producer,) = graph.find_nodes(op="call_function", target=_FQ_LINEAR)
298+
(activation_node,) = graph.find_nodes(
299+
op="call_function", target=model.fused_activation
300+
)
301+
assert (get_qparams_from_node(producer, "out") is not None) == fork
302+
assert (get_qparams_from_node(activation_node, "inp") is not None) == fork
303+
assert get_qparams_from_node(activation_node, "out") is not None
304+
305+
306+
@pytest.mark.parametrize("activation", ["relu", "hardtanh"])
307+
def test_separately_quantized_linear_activation_keeps_rounding(activation):
308+
model = EspLinearActivation(activation)
309+
quantizer = FusedQuantQuantizer(get_esp_fused_quant_quantizer().child_quantizers)
310+
tester = EspTester(model, (ramp_tensor(-3, 5, (2, 17)),))
311+
tester.export_fused_quant(quantizer).to_edge()
312+
assert not FoldActivations()(tester.get_artifact().exported_program()).modified
313+
tester.run_passes().check_node_count({_ESP_LINEAR: 1, _ESP_LUT: 1})
314+
315+
316+
def _linear_activation_clamp_tester(activation):
317+
model = EspLinearActivation(activation)
318+
tester = EspTester(model, (ramp_tensor(-80, 80, (9, 17)),))
319+
tester.export_fused_quant().to_edge()
320+
program = tester.get_artifact().exported_program()
321+
(node,) = program.graph.find_nodes(
322+
op="call_function", target=model.fused_activation
323+
)
324+
params = get_qparams_from_node(node, "out")
325+
get_constant(program, params.scale).fill_(0.125)
326+
get_constant(program, params.zero_point).fill_(-7)
327+
(output,) = program.graph.find_nodes(op="output")
328+
dequant = output.args[0][0]
329+
set_arg(dequant, "scale", 0.125)
330+
set_arg(dequant, "zero_point", -7)
331+
program.graph_module.recompile()
332+
return tester
333+
334+
335+
@pytest.mark.parametrize("activation", _ACTIVATIONS)
336+
def test_fold_linear_uses_final_qparams_and_clamp(activation):
337+
tester = _linear_activation_clamp_tester(activation)
338+
program = tester.get_artifact().exported_program()
339+
expected = program.module()(*tester.example_inputs)
340+
assert FoldActivations()(program).modified
341+
torch.testing.assert_close(
342+
program.module()(*tester.example_inputs), expected, atol=0, rtol=0
343+
)
344+
tester.run_passes().check_node_count({_ESP_LINEAR: 1, _ESP_LUT: 0})
345+
lowered = tester.get_artifact().exported_program()
346+
(node,) = lowered.graph.find_nodes(op="call_function", target=_ESP_LINEAR)
347+
assert get_arg(node, "output_offset") == -7
348+
assert (get_arg(node, "activation_min"), get_arg(node, "activation_max")) == (
349+
_ACTIVATIONS[activation][1]
350+
)
351+
torch.testing.assert_close(
352+
lowered.module()(*tester.example_inputs), expected, atol=0.125, rtol=0
353+
)
354+
355+
356+
@pytest.mark.parametrize("activation", _ACTIVATIONS)
357+
def test_implementation_linear_activation_clamp(activation, pytestconfig):
358+
tester = _linear_activation_clamp_tester(activation)
359+
tester.run_passes()
360+
tester.test_implementation(pytestconfig.rootpath, num_inferences=2)
361+
362+
229363
@pytest.mark.parametrize(
230364
"model, inputs",
231365
[

0 commit comments

Comments
 (0)