Skip to content

Commit 6f4aa7d

Browse files
committed
[Espressif] Fuse binary ReLU and HardTanh patterns
Quantize single-consumer add, subtract and multiply followed by ReLU or HardTanh, including ReLU6, at their pattern boundaries so the reference has no intermediate rounding. Fold the activation into Fused Quant output qparams and clamp bounds before ESP replacement. Reuse existing shape and arithmetic eligibility checks with the final output scale; unsupported producers retain portable execution. Forked and separately quantized activations keep their original boundaries. Clamp bounds use the producer's floating dtype to preserve BF16 and FP16 portable fallback results. Read the quantizer and FoldActivations first, then the binary lowering and schema/wrapper changes. Shared dialect and S3 implementation cases cover forks, broadcasts, saturation, fallback and narrower clamp grids, including asymmetric, positive and negative HardTanh bounds and rounding at endpoints. Authored with OpenAI Codex.
1 parent 1526b41 commit 6f4aa7d

15 files changed

Lines changed: 550 additions & 32 deletions

‎backends/espressif/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -198,9 +198,9 @@ Use the shared export API with `get_esp_fused_quant_recipe()`. The recipe
198198
configures `get_esp_fused_quant_quantizer`, calibrates the model, fuses Q/DQ into Fused
199199
Quant operators, and runs `EspPassManager`. The pass manager uses the shared Fused
200200
Quant pipeline with `channels_last=True` to convert spatial operators to NHWC,
201-
simplify layout operations, and fold scalar multiplies before ESP lowering.
202-
Other shared opt-in rewrites keep their default settings. Unlowered Fused Quant
203-
operators decompose to portable kernels before serialization.
201+
simplify layout operations, and fold scalar multiplies before ESP activation
202+
folding and lowering. Other shared opt-in rewrites keep their default settings.
203+
Unlowered Fused Quant operators decompose to portable kernels before serialization.
204204
No delegate partitions are created.
205205

206206
```python

‎backends/espressif/ops/op_quantized_add.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Tensor& quantized_add_int8_out(
3535
int64_t left_shift,
3636
int64_t output_multiplier,
3737
int64_t output_shift,
38+
int64_t activation_min,
39+
int64_t activation_max,
3840
Tensor& out) {
3941
const auto numel = input1.numel();
4042
ET_KERNEL_CHECK(
@@ -64,8 +66,8 @@ Tensor& quantized_add_int8_out(
6466
static_cast<int32_t>(output_offset),
6567
static_cast<int32_t>(output_multiplier),
6668
static_cast<int32_t>(output_shift),
67-
std::numeric_limits<int8_t>::min(),
68-
std::numeric_limits<int8_t>::max(),
69+
static_cast<int32_t>(activation_min),
70+
static_cast<int32_t>(activation_max),
6971
static_cast<int32_t>(numel));
7072
return out;
7173
}

‎backends/espressif/ops/op_quantized_mul.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
ET_KERNEL_CHECK(
@@ -54,8 +56,8 @@ Tensor& quantized_mul_int8_out(
5456
static_cast<int32_t>(output_offset),
5557
static_cast<int32_t>(output_multiplier),
5658
static_cast<int32_t>(output_shift),
57-
std::numeric_limits<int8_t>::min(),
58-
std::numeric_limits<int8_t>::max(),
59+
static_cast<int32_t>(activation_min),
60+
static_cast<int32_t>(activation_max),
5961
static_cast<int32_t>(numel));
6062
return out;
6163
}

‎backends/espressif/ops/op_quantized_mul_broadcast.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_broadcast_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
const auto block_size = input2.numel();
@@ -57,8 +59,8 @@ Tensor& quantized_mul_broadcast_int8_out(
5759
static_cast<int32_t>(output_offset),
5860
static_cast<int32_t>(output_multiplier),
5961
static_cast<int32_t>(output_shift),
60-
std::numeric_limits<int8_t>::min(),
61-
std::numeric_limits<int8_t>::max(),
62+
static_cast<int32_t>(activation_min),
63+
static_cast<int32_t>(activation_max),
6264
static_cast<int32_t>(numel / block_size),
6365
static_cast<int32_t>(block_size));
6466
return out;

‎backends/espressif/ops/operators.py‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,33 +16,37 @@
1616
)
1717
lib.define(
1818
"quantized_mul_int8(Tensor input1, Tensor input2, int input1_offset, "
19-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
19+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
20+
"int activation_min, int activation_max) -> Tensor"
2021
)
2122
lib.define(
2223
"quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, "
2324
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
24-
"*, Tensor(a!) out) -> Tensor(a!)"
25+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
2526
)
2627
lib.define(
2728
"quantized_mul_broadcast_int8(Tensor input1, Tensor input2, int input1_offset, "
28-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
29+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
30+
"int activation_min, int activation_max) -> Tensor"
2931
)
3032
lib.define(
3133
"quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, "
3234
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
33-
"*, Tensor(a!) out) -> Tensor(a!)"
35+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
3436
)
3537
lib.define(
3638
"quantized_add_int8(Tensor input1, Tensor input2, int input1_offset, "
3739
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
3840
"int input2_multiplier, int input2_shift, int left_shift, "
39-
"int output_multiplier, int output_shift) -> Tensor"
41+
"int output_multiplier, int output_shift, "
42+
"int activation_min, int activation_max) -> Tensor"
4043
)
4144
lib.define(
4245
"quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, "
4346
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
4447
"int input2_multiplier, int input2_shift, int left_shift, "
45-
"int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)"
48+
"int output_multiplier, int output_shift, int activation_min, int activation_max, "
49+
"*, Tensor(a!) out) -> Tensor(a!)"
4650
)
4751

4852

@@ -154,6 +158,8 @@ def quantized_mul_int8_meta(
154158
output_offset,
155159
output_multiplier,
156160
output_shift,
161+
activation_min,
162+
activation_max,
157163
):
158164
return torch.empty_like(input1)
159165

@@ -167,6 +173,8 @@ def quantized_mul_broadcast_int8_meta(
167173
output_offset,
168174
output_multiplier,
169175
output_shift,
176+
activation_min,
177+
activation_max,
170178
):
171179
return input1.new_empty(torch.broadcast_shapes(input1.shape, input2.shape))
172180

@@ -181,12 +189,14 @@ def quantized_mul_int8(
181189
output_offset,
182190
output_multiplier,
183191
output_shift,
192+
activation_min,
193+
activation_max,
184194
):
185195
product = (input1.to(torch.int32) + input1_offset) * (
186196
input2.to(torch.int32) + input2_offset
187197
)
188198
result = requantize(product, output_multiplier, output_shift) + output_offset
189-
return result.clamp(-128, 127).to(torch.int8)
199+
return result.clamp(activation_min, activation_max).to(torch.int8)
190200

191201

192202
@register_fake("esp_nn::quantized_add_int8")
@@ -203,6 +213,8 @@ def quantized_add_int8_meta(
203213
left_shift,
204214
output_multiplier,
205215
output_shift,
216+
activation_min,
217+
activation_max,
206218
):
207219
return torch.empty_like(input1)
208220

@@ -221,14 +233,16 @@ def quantized_add_int8(
221233
left_shift,
222234
output_multiplier,
223235
output_shift,
236+
activation_min,
237+
activation_max,
224238
):
225239
lhs = (input1.to(torch.int32) + input1_offset) << left_shift
226240
rhs = (input2.to(torch.int32) + input2_offset) << left_shift
227241
total = requantize(lhs, input1_multiplier, input1_shift) + requantize(
228242
rhs, input2_multiplier, input2_shift
229243
)
230244
result = requantize(total, output_multiplier, output_shift) + output_offset
231-
return result.clamp(-128, 127).to(torch.int8)
245+
return result.clamp(activation_min, activation_max).to(torch.int8)
232246

233247

234248
@register_fake("esp_nn::quantized_linear_int8")

‎backends/espressif/ops/operators.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,19 @@
1010
- arg_meta: null
1111
kernel_name: espressif::quantized_lut_int8_out
1212

13-
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
13+
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
1414
variants: function
1515
kernels:
1616
- arg_meta: null
1717
kernel_name: espressif::quantized_mul_int8_out
1818

19-
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
19+
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2020
variants: function
2121
kernels:
2222
- arg_meta: null
2323
kernel_name: espressif::quantized_mul_broadcast_int8_out
2424

25-
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
25+
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2626
variants: function
2727
kernels:
2828
- arg_meta: null

‎backends/espressif/passes/esp_pass_manager.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
from torch.fx import GraphModule
2828

29+
from .fold_activations import FoldActivations
2930
from .lower_to_esp import LowerToEsp
3031

3132

@@ -43,6 +44,7 @@ def __init__(
4344
*get_fused_quant_passes(channels_last=True),
4445
# Refresh contiguous tensor metadata after the layout rewrites.
4546
ExportPass(),
47+
FoldActivations(),
4648
LowerToEsp(target=target, kernel_mode=kernel_mode),
4749
ToChannelsFirst(),
4850
DecomposeFusedQuant(),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import math
8+
from dataclasses import replace
9+
10+
import torch
11+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
12+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
13+
from executorch.backends.transforms.utils import get_constant
14+
from executorch.exir.dialects._ops import ops as exir_ops
15+
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
16+
17+
18+
class FoldActivations(ExportedProgramPassBase):
19+
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
20+
21+
def call(self, exported_program):
22+
graph = exported_program.graph
23+
modified = False
24+
for activation in list(graph.nodes):
25+
if activation.target not in (
26+
exir_ops.edge.fused_quant.relu.default,
27+
exir_ops.edge.fused_quant.hard_tanh.default,
28+
):
29+
continue
30+
bounds = (
31+
(get_arg(activation, "min_val"), get_arg(activation, "max_val"))
32+
if activation.target == exir_ops.edge.fused_quant.hard_tanh.default
33+
else (0.0, float("inf"))
34+
)
35+
if not all(isinstance(bound, (int, float)) for bound in bounds) or not (
36+
bounds[0] <= bounds[1]
37+
):
38+
continue
39+
producer = get_arg(activation, "inp")
40+
if (
41+
producer.target
42+
not in (
43+
exir_ops.edge.fused_quant.add.default,
44+
exir_ops.edge.fused_quant.sub.default,
45+
exir_ops.edge.fused_quant.mul.default,
46+
)
47+
or len(producer.users) != 1
48+
or get_qparams_from_node(producer, "out") is not None
49+
or get_qparams_from_node(activation, "inp") is not None
50+
):
51+
continue
52+
params = get_qparams_from_node(activation, "out")
53+
if (
54+
params is None
55+
or not params.is_per_tensor()
56+
or params.dtype != torch.int8
57+
or (params.quant_min, params.quant_max) != (-128, 127)
58+
):
59+
continue
60+
scale = get_constant(exported_program, params.scale)
61+
zero_point = get_constant(exported_program, params.zero_point)
62+
if (
63+
scale is None
64+
or zero_point is None
65+
or not math.isfinite(scale.item())
66+
or scale.item() <= 0
67+
or not -128 <= zero_point.item() <= 127
68+
):
69+
continue
70+
concrete = replace(params, scale=scale, zero_point=zero_point)
71+
lower, upper = concrete.quantize(
72+
torch.tensor(bounds, dtype=producer.meta["val"].dtype)
73+
).tolist()
74+
for field in ("scale", "zero_point", "dtype"):
75+
set_arg(producer, f"out_{field}", get_arg(activation, f"out_{field}"))
76+
set_arg(producer, "out_quant_min", lower)
77+
set_arg(producer, "out_quant_max", upper)
78+
producer.meta["val"] = activation.meta["val"]
79+
activation.replace_all_uses_with(producer)
80+
graph.erase_node(activation)
81+
modified = True
82+
if modified:
83+
graph.eliminate_dead_code()
84+
graph.lint()
85+
exported_program.graph_module.recompile()
86+
return ExportedProgramPassResult(exported_program, modified)

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def _prepare_binary(node, params, target, rescale_fn) -> tuple | None:
148148
)
149149

150150

151-
def _qparams(program, node, prefix):
151+
def _qparams(program, node, prefix, *, allow_output_clamp=False):
152152
params = get_qparams_from_node(node, prefix)
153153
dtype = torch.int8 if prefix == "out" else torch.float32
154154
if params is None or not params.is_per_tensor() or params.dtype != dtype:
@@ -168,7 +168,10 @@ def _qparams(program, node, prefix):
168168
raise ValueError(f"{node.name}: {prefix} scale must be finite and positive")
169169
if not -128 <= zero_point <= 127:
170170
raise ValueError(f"{node.name}: {prefix} has an invalid int8 zero point")
171-
if (params.quant_min, params.quant_max) != (-128, 127):
171+
if not (prefix == "out" and allow_output_clamp) and (
172+
params.quant_min,
173+
params.quant_max,
174+
) != (-128, 127):
172175
return None
173176
return _QParams(scale, zero_point)
174177

@@ -746,7 +749,7 @@ def call(self, exported_program):
746749
continue
747750
target, rescale_fn = lowering
748751
params = [
749-
_qparams(exported_program, node, prefix)
752+
_qparams(exported_program, node, prefix, allow_output_clamp=True)
750753
for prefix in ("inp", "other", "out")
751754
]
752755
if any(param is None for param in params):
@@ -756,7 +759,14 @@ def call(self, exported_program):
756759
continue
757760
target, args = prepared
758761
with graph.inserting_before(node):
759-
replacement = graph.call_function(target, args)
762+
replacement = graph.call_function(
763+
target,
764+
(
765+
*args,
766+
get_arg(node, "out_quant_min"),
767+
get_arg(node, "out_quant_max"),
768+
),
769+
)
760770
replacement.meta = node.meta.copy()
761771
node.replace_all_uses_with(replacement)
762772
graph.erase_node(node)

0 commit comments

Comments
 (0)