Skip to content

Commit 14266b4

Browse files
committed
[Espressif] Fuse linear ReLU and HardTanh patterns
Quantize single-consumer linear followed by ReLU or HardTanh, including ReLU6, at the pattern boundaries. Fold the final output qparams and clamp bounds into the existing ESP-NN fully connected call, preserving per-channel weights and derived int32 bias. Forked and separately quantized activations retain their original quantization boundaries. Read the quantizer and activation fold before the linear lowering and schema/wrapper updates. Shared dialect and S3 cases cover bias/no bias, forks, saturation, vector and higher-rank inputs, and narrower clamp grids with asymmetric, positive-only and negative-only HardTanh bounds. Authored with OpenAI Codex.
1 parent cc91377 commit 14266b4

8 files changed

Lines changed: 195 additions & 30 deletions

File tree

‎backends/espressif/ops/op_quantized_linear.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_linear_int8_out(
3030
const Tensor& output_shift,
3131
int64_t input_offset,
3232
int64_t output_offset,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
3537
const auto channels = weight.size(0);
@@ -71,8 +73,8 @@ Tensor& quantized_linear_int8_out(
7173
static_cast<int32_t>(output_offset),
7274
output_shift.const_data_ptr<int32_t>(),
7375
output_multiplier.const_data_ptr<int32_t>(),
74-
std::numeric_limits<int8_t>::min(),
75-
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(activation_min),
77+
static_cast<int32_t>(activation_max),
7678
static_cast<int32_t>(batches));
7779
return out;
7880
}

‎backends/espressif/ops/operators.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,13 @@
6868
)
6969
lib.define(
7070
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
71-
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
71+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
72+
"int activation_min, int activation_max) -> Tensor"
7273
)
7374
lib.define(
7475
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
7576
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
76-
"*, Tensor(a!) out) -> Tensor(a!)"
77+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
7778
)
7879

7980
lib.define(
@@ -247,14 +248,30 @@ def quantized_add_int8(
247248

248249
@register_fake("esp_nn::quantized_linear_int8")
249250
def quantized_linear_int8_meta(
250-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
251+
input,
252+
weight,
253+
bias,
254+
output_multiplier,
255+
output_shift,
256+
input_offset,
257+
output_offset,
258+
activation_min,
259+
activation_max,
251260
):
252261
return input.new_empty((*input.shape[:-1], weight.shape[0]))
253262

254263

255264
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
256265
def quantized_linear_int8(
257-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
266+
input,
267+
weight,
268+
bias,
269+
output_multiplier,
270+
output_shift,
271+
input_offset,
272+
output_offset,
273+
activation_min,
274+
activation_max,
258275
):
259276
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
260277
if bias is not None:
@@ -268,7 +285,7 @@ def quantized_linear_int8(
268285
],
269286
dim=-1,
270287
)
271-
return (result + output_offset).clamp(-128, 127).to(torch.int8)
288+
return (result + output_offset).clamp(activation_min, activation_max).to(torch.int8)
272289

273290

274291
@register_fake("esp_nn::quantized_conv2d_int8")
@@ -325,6 +342,8 @@ def quantized_conv2d_int8(
325342
output_shift,
326343
0,
327344
output_offset,
345+
-128,
346+
127,
328347
)
329348

330349

‎backends/espressif/ops/operators.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
- arg_meta: null
4141
kernel_name: espressif::quantized_avg_pool2d_int8_out
4242

43-
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
43+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
4444
variants: function
4545
kernels:
4646
- arg_meta: null

‎backends/espressif/passes/fold_activations.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717

1818
class FoldActivations(ExportedProgramPassBase):
19-
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
19+
"""Fold grouped clamps into the producer's output quantization."""
2020

2121
def call(self, exported_program):
2222
graph = exported_program.graph
@@ -43,6 +43,7 @@ def call(self, exported_program):
4343
exir_ops.edge.fused_quant.add.default,
4444
exir_ops.edge.fused_quant.sub.default,
4545
exir_ops.edge.fused_quant.mul.default,
46+
exir_ops.edge.fused_quant.linear.default,
4647
)
4748
or len(producer.users) != 1
4849
or get_qparams_from_node(producer, "out") is not None

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,8 @@ def _linear_kernel_args(program, node, constants):
404404
"input and output must be nonempty, static, contiguous int8 tensors"
405405
)
406406
input_params, output_params = (
407-
_qparams(program, node, name) for name in ("inp", "out")
407+
_qparams(program, node, name, allow_output_clamp=True)
408+
for name in ("inp", "out")
408409
)
409410
if input_params is None or output_params is None:
410411
raise ValueError("input and output require constant per-tensor int8 qparams")
@@ -436,6 +437,8 @@ def _linear_kernel_args(program, node, constants):
436437
shift_node,
437438
-input_params.zp,
438439
output_params.zp,
440+
get_arg(node, "out_quant_min"),
441+
get_arg(node, "out_quant_max"),
439442
)
440443

441444

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
106106
Conv2d, and depthwise Conv2d biases use
107107
int32 with scales derived from the input and weight observers.
108108
AvgPool2d without divisor_override shares input and output qparams.
109-
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
110-
only their boundaries.
109+
Single-consumer binary or linear followed by ReLU or HardTanh (including
110+
ReLU6) patterns observe only their boundaries.
111111
EspPassManager lowers supported Fused Quant operators to ESP-NN and
112112
decomposes the rest for portable execution. Use EspNNQuantizer for the
113113
EspNNBackend delegate instead.
@@ -162,16 +162,19 @@ def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
162162
torch.ops.aten.add.Tensor,
163163
torch.ops.aten.sub.Tensor,
164164
torch.ops.aten.mul.Tensor,
165+
torch.ops.aten.linear.default,
165166
)
166167
or len(producer.users) != 1
167-
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
168168
or Q_ANNOTATION_KEY in producer.meta
169169
or Q_ANNOTATION_KEY in activation.meta
170170
):
171171
continue
172-
if (
173-
producer.target != torch.ops.aten.mul.Tensor
174-
and get_arg(producer, "alpha") != 1
172+
if producer.target != torch.ops.aten.linear.default and (
173+
not isinstance(get_arg(producer, "other"), torch.fx.Node)
174+
or (
175+
producer.target != torch.ops.aten.mul.Tensor
176+
and get_arg(producer, "alpha") != 1
177+
)
175178
):
176179
continue
177180
groups.append((producer, activation))

‎backends/espressif/test/ops/test_linear.py‎

Lines changed: 146 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,24 @@
66

77
import re
88
from dataclasses import replace
9+
from functools import partial
910

1011
import pytest
1112
import torch
1213
from executorch.backends.espressif.lowering import (
1314
get_esp_fused_quant_recipe,
1415
get_operator_library_report,
1516
)
17+
from executorch.backends.espressif.passes.fold_activations import FoldActivations
18+
from executorch.backends.espressif.quantizer import get_esp_fused_quant_quantizer
1619
from executorch.backends.espressif.test.tester import (
1720
EspTestCase,
1821
EspTester,
1922
ramp_tensor,
2023
)
2124
from executorch.backends.test.suite.operators.test_linear import Model
22-
from executorch.backends.transforms.permute_pass_utils import get_arg
25+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
26+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
2327
from executorch.backends.transforms.utils import get_constant
2428
from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
2529
get_symmetric_quantization_config,
@@ -34,6 +38,21 @@
3438

3539
_FQ_LINEAR = exir_ops.edge.fused_quant.linear.default
3640
_ESP_LINEAR = exir_ops.edge.esp_nn.quantized_linear_int8.default
41+
_ESP_LUT = exir_ops.edge.esp_nn.quantized_lut_int8.default
42+
_ACTIVATIONS = {
43+
"relu": (torch.nn.ReLU, (-7, 127)),
44+
"relu6": (torch.nn.ReLU6, (-7, 41)),
45+
"hardtanh": (torch.nn.Hardtanh, (-15, 1)),
46+
"hardtanh_asymmetric": (
47+
partial(torch.nn.Hardtanh, min_val=-2.0625, max_val=3.1875),
48+
(-23, 19),
49+
),
50+
"hardtanh_positive": (partial(torch.nn.Hardtanh, min_val=1, max_val=4), (1, 25)),
51+
"hardtanh_negative": (
52+
partial(torch.nn.Hardtanh, min_val=-5, max_val=-2),
53+
(-47, -23),
54+
),
55+
}
3756

3857

3958
class EspLinear(Model):
@@ -59,16 +78,27 @@ def forward(self, x):
5978
return result, torch.sin(result), torch.sin(x), self.linear.bias
6079

6180

62-
class EspLinearRelu(EspLinear):
63-
ops_before_transforms = {_FQ_LINEAR: 1, exir_ops.edge.fused_quant.relu.default: 1}
64-
ops_after_transforms = {
65-
_ESP_LINEAR: 1,
66-
exir_ops.edge.esp_nn.quantized_lut_int8.default: 1,
67-
exir_ops.edge.aten.relu.default: 0,
68-
}
81+
class EspLinearActivation(EspLinear):
82+
def __init__(self, activation, *, bias=True, fork=False):
83+
super().__init__(bias=bias)
84+
self.activation = _ACTIVATIONS[activation][0]()
85+
self.fused_activation = (
86+
exir_ops.edge.fused_quant.relu.default
87+
if activation == "relu"
88+
else exir_ops.edge.fused_quant.hard_tanh.default
89+
)
90+
self.fork = fork
91+
self.ops_before_transforms = {_FQ_LINEAR: 1, self.fused_activation: 1}
92+
self.ops_after_transforms = {
93+
_ESP_LINEAR: 1,
94+
self.fused_activation: 0,
95+
_ESP_LUT: int(fork),
96+
}
6997

7098
def forward(self, x):
71-
return torch.relu(self.linear(x))
99+
result = self.linear(x)
100+
activated = self.activation(result)
101+
return (activated, result, torch.sin(result)) if self.fork else activated
72102

73103

74104
class EspSharedLinear(EspLinear):
@@ -115,7 +145,6 @@ def __init__(self):
115145
"float_users": EspTestCase(
116146
EspLinearWithFloatUsers(), (ramp_tensor(-3, 5, (2, 17)),)
117147
),
118-
"relu": EspTestCase(EspLinearRelu(), (ramp_tensor(-3, 5, (2, 17)),)),
119148
"shared_bias": EspTestCase(
120149
EspSharedLinear(), (ramp_tensor(-3, 5, (2, 17)), ramp_tensor(-24, 40, (2, 17)))
121150
),
@@ -127,6 +156,38 @@ def __init__(self):
127156
calibration_samples=[(ramp_tensor(-3, 5, (2, 17)),)],
128157
),
129158
}
159+
test_cases.update(
160+
{
161+
f"{activation}{'_no_bias' if not bias else ''}": EspTestCase(
162+
EspLinearActivation(activation, bias=bias),
163+
(ramp_tensor(-30, 50, (5, 17)),),
164+
)
165+
for activation in _ACTIVATIONS
166+
for bias in (True, False)
167+
}
168+
)
169+
test_cases.update(
170+
{
171+
"relu_fork": EspTestCase(
172+
EspLinearActivation("relu", fork=True), (ramp_tensor(-3, 5, (2, 17)),)
173+
),
174+
"hardtanh_fork": EspTestCase(
175+
EspLinearActivation("hardtanh", fork=True),
176+
(ramp_tensor(-3, 5, (2, 17)),),
177+
),
178+
"relu6_vector": EspTestCase(
179+
EspLinearActivation("relu6"), (ramp_tensor(-30, 50, (17,)),)
180+
),
181+
"hardtanh_rank_3": EspTestCase(
182+
EspLinearActivation("hardtanh"), (ramp_tensor(-30, 50, (2, 3, 17)),)
183+
),
184+
"relu6_saturation": EspTestCase(
185+
EspLinearActivation("relu6"),
186+
(ramp_tensor(-30, 50, (2, 17)),),
187+
calibration_samples=[(ramp_tensor(-3, 5, (2, 17)),)],
188+
),
189+
}
190+
)
130191

131192

132193
@pytest.mark.parametrize("test_case", test_cases.values(), ids=test_cases.keys())
@@ -158,7 +219,7 @@ def test_implementation_linear(test_case, pytestconfig):
158219
test_case.model, test_case.example_inputs, test_case.calibration_samples
159220
)
160221
tester.test_implementation(
161-
pytestconfig.rootpath, rtol=torch.finfo(torch.float32).eps
222+
pytestconfig.rootpath, rtol=torch.finfo(torch.float32).eps, num_inferences=2
162223
)
163224

164225

@@ -177,7 +238,7 @@ def test_public_recipe_linear():
177238
assert session.get_executorch_program_manager().buffer
178239

179240

180-
@pytest.mark.parametrize("case_name", ["shared_bias", "large_bias"])
241+
@pytest.mark.parametrize("case_name", ["shared_bias", "large_bias", "relu", "hardtanh"])
181242
def test_lowering_reuses_quantized_bias_and_preserves_source_program(case_name):
182243
case = test_cases[case_name]
183244
tester = EspTester(case.model, case.example_inputs).export_fused_quant().to_edge()
@@ -199,6 +260,79 @@ def test_lowering_reuses_quantized_bias_and_preserves_source_program(case_name):
199260
source.validate()
200261

201262

263+
@pytest.mark.parametrize("activation", ["relu", "relu6", "hardtanh"])
264+
@pytest.mark.parametrize("fork", [False, True])
265+
def test_linear_activation_quantization_boundary(activation, fork):
266+
model = EspLinearActivation(activation, fork=fork)
267+
tester = EspTester(model, (ramp_tensor(-3, 5, (2, 17)),))
268+
tester.export_fused_quant().to_edge()
269+
graph = tester.get_artifact().exported_program().graph
270+
(producer,) = graph.find_nodes(op="call_function", target=_FQ_LINEAR)
271+
(activation_node,) = graph.find_nodes(
272+
op="call_function", target=model.fused_activation
273+
)
274+
assert (get_qparams_from_node(producer, "out") is not None) == fork
275+
assert (get_qparams_from_node(activation_node, "inp") is not None) == fork
276+
assert get_qparams_from_node(activation_node, "out") is not None
277+
278+
279+
@pytest.mark.parametrize("activation", ["relu", "hardtanh"])
280+
def test_separately_quantized_linear_activation_keeps_rounding(activation):
281+
model = EspLinearActivation(activation)
282+
quantizer = FusedQuantQuantizer(get_esp_fused_quant_quantizer().child_quantizers)
283+
tester = EspTester(model, (ramp_tensor(-3, 5, (2, 17)),))
284+
tester.export_fused_quant(quantizer).to_edge()
285+
assert not FoldActivations()(tester.get_artifact().exported_program()).modified
286+
tester.run_passes().check_node_count({_ESP_LINEAR: 1, _ESP_LUT: 1})
287+
288+
289+
def _linear_activation_clamp_tester(activation):
290+
model = EspLinearActivation(activation)
291+
tester = EspTester(model, (ramp_tensor(-80, 80, (9, 17)),))
292+
tester.export_fused_quant().to_edge()
293+
program = tester.get_artifact().exported_program()
294+
(node,) = program.graph.find_nodes(
295+
op="call_function", target=model.fused_activation
296+
)
297+
params = get_qparams_from_node(node, "out")
298+
get_constant(program, params.scale).fill_(0.125)
299+
get_constant(program, params.zero_point).fill_(-7)
300+
(output,) = program.graph.find_nodes(op="output")
301+
dequant = output.args[0][0]
302+
set_arg(dequant, "scale", 0.125)
303+
set_arg(dequant, "zero_point", -7)
304+
program.graph_module.recompile()
305+
return tester
306+
307+
308+
@pytest.mark.parametrize("activation", _ACTIVATIONS)
309+
def test_fold_linear_uses_final_qparams_and_clamp(activation):
310+
tester = _linear_activation_clamp_tester(activation)
311+
program = tester.get_artifact().exported_program()
312+
expected = program.module()(*tester.example_inputs)
313+
assert FoldActivations()(program).modified
314+
torch.testing.assert_close(
315+
program.module()(*tester.example_inputs), expected, atol=0, rtol=0
316+
)
317+
tester.run_passes().check_node_count({_ESP_LINEAR: 1, _ESP_LUT: 0})
318+
lowered = tester.get_artifact().exported_program()
319+
(node,) = lowered.graph.find_nodes(op="call_function", target=_ESP_LINEAR)
320+
assert get_arg(node, "output_offset") == -7
321+
assert (get_arg(node, "activation_min"), get_arg(node, "activation_max")) == (
322+
_ACTIVATIONS[activation][1]
323+
)
324+
torch.testing.assert_close(
325+
lowered.module()(*tester.example_inputs), expected, atol=0.125, rtol=0
326+
)
327+
328+
329+
@pytest.mark.parametrize("activation", _ACTIVATIONS)
330+
def test_implementation_linear_activation_clamp(activation, pytestconfig):
331+
tester = _linear_activation_clamp_tester(activation)
332+
tester.run_passes()
333+
tester.test_implementation(pytestconfig.rootpath, num_inferences=2)
334+
335+
202336
class RuntimeBias(EspLinear):
203337
def forward(self, x, bias):
204338
return torch.nn.functional.linear(x, self.linear.weight, bias)

0 commit comments

Comments
 (0)