Skip to content

Commit ff96319

Browse files
committed
[Espressif] Fuse binary ReLU and HardTanh patterns
Quantize single-consumer add, subtract and multiply followed by ReLU or HardTanh, including ReLU6, at their pattern boundaries so the reference has no intermediate rounding. Fold the activation into Fused Quant output qparams and clamp bounds before ESP replacement. Reuse existing shape and arithmetic eligibility checks with the final output scale; unsupported producers retain portable execution. Forked and separately quantized activations keep their original boundaries. Clamp bounds use the producer's floating dtype to preserve BF16 and FP16 portable fallback results. Read the quantizer and FoldActivations first, then the binary lowering and schema/wrapper changes. Shared dialect and S3 implementation cases cover forks, broadcasts, saturation, fallback and narrower clamp grids, including asymmetric, positive and negative HardTanh bounds and rounding at endpoints. Authored with OpenAI Codex.
1 parent 0867d1a commit ff96319

15 files changed

Lines changed: 554 additions & 30 deletions

‎backends/espressif/README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,13 @@ Both flows use ESP-NN 1.5.0 kernels. The operator library currently accelerates
2828
int8 tensor add and subtract with equal input shapes and `alpha=1`. Subtraction
2929
uses the add kernel with a negative second-input multiplier. Multiply supports
3030
equal shapes or a contiguous trailing block broadcast across the full tensor,
31-
including one-element tensors. Sigmoid, tanh, GELU, SiLU, HardSwish, HardSigmoid,
31+
including one-element tensors. A single-consumer add, subtract, or multiply
32+
followed by ReLU or HardTanh (including ReLU6) is quantized as one pattern,
33+
with no intermediate rounding. HardTanh supports constant lower and upper
34+
bounds. The activation's output qparams and clamp bounds are folded into
35+
the binary kernel; unsupported binary configurations retain portable execution.
36+
Forked or separately quantized activations keep their quantization boundaries.
37+
Sigmoid, tanh, GELU, SiLU, HardSwish, HardSigmoid,
3238
ReLU, and HardTanh (including ReLU6) support static, contiguous int8 tensors with
3339
constant per-tensor input and output qparams. Export evaluates all 256 input
3440
codes with the FQ reference, using calibrated qparams, and stores the results

‎backends/espressif/ops/op_quantized_add.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Tensor& quantized_add_int8_out(
3535
int64_t left_shift,
3636
int64_t output_multiplier,
3737
int64_t output_shift,
38+
int64_t activation_min,
39+
int64_t activation_max,
3840
Tensor& out) {
3941
const auto numel = input1.numel();
4042
ET_KERNEL_CHECK(
@@ -64,8 +66,8 @@ Tensor& quantized_add_int8_out(
6466
static_cast<int32_t>(output_offset),
6567
static_cast<int32_t>(output_multiplier),
6668
static_cast<int32_t>(output_shift),
67-
std::numeric_limits<int8_t>::min(),
68-
std::numeric_limits<int8_t>::max(),
69+
static_cast<int32_t>(activation_min),
70+
static_cast<int32_t>(activation_max),
6971
static_cast<int32_t>(numel));
7072
return out;
7173
}

‎backends/espressif/ops/op_quantized_mul.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
ET_KERNEL_CHECK(
@@ -54,8 +56,8 @@ Tensor& quantized_mul_int8_out(
5456
static_cast<int32_t>(output_offset),
5557
static_cast<int32_t>(output_multiplier),
5658
static_cast<int32_t>(output_shift),
57-
std::numeric_limits<int8_t>::min(),
58-
std::numeric_limits<int8_t>::max(),
59+
static_cast<int32_t>(activation_min),
60+
static_cast<int32_t>(activation_max),
5961
static_cast<int32_t>(numel));
6062
return out;
6163
}

‎backends/espressif/ops/op_quantized_mul_broadcast.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_broadcast_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
const auto block_size = input2.numel();
@@ -57,8 +59,8 @@ Tensor& quantized_mul_broadcast_int8_out(
5759
static_cast<int32_t>(output_offset),
5860
static_cast<int32_t>(output_multiplier),
5961
static_cast<int32_t>(output_shift),
60-
std::numeric_limits<int8_t>::min(),
61-
std::numeric_limits<int8_t>::max(),
62+
static_cast<int32_t>(activation_min),
63+
static_cast<int32_t>(activation_max),
6264
static_cast<int32_t>(numel / block_size),
6365
static_cast<int32_t>(block_size));
6466
return out;

‎backends/espressif/ops/operators.py‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,33 +16,37 @@
1616
)
1717
lib.define(
1818
"quantized_mul_int8(Tensor input1, Tensor input2, int input1_offset, "
19-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
19+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
20+
"int activation_min, int activation_max) -> Tensor"
2021
)
2122
lib.define(
2223
"quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, "
2324
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
24-
"*, Tensor(a!) out) -> Tensor(a!)"
25+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
2526
)
2627
lib.define(
2728
"quantized_mul_broadcast_int8(Tensor input1, Tensor input2, int input1_offset, "
28-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
29+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
30+
"int activation_min, int activation_max) -> Tensor"
2931
)
3032
lib.define(
3133
"quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, "
3234
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
33-
"*, Tensor(a!) out) -> Tensor(a!)"
35+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
3436
)
3537
lib.define(
3638
"quantized_add_int8(Tensor input1, Tensor input2, int input1_offset, "
3739
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
3840
"int input2_multiplier, int input2_shift, int left_shift, "
39-
"int output_multiplier, int output_shift) -> Tensor"
41+
"int output_multiplier, int output_shift, "
42+
"int activation_min, int activation_max) -> Tensor"
4043
)
4144
lib.define(
4245
"quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, "
4346
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
4447
"int input2_multiplier, int input2_shift, int left_shift, "
45-
"int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)"
48+
"int output_multiplier, int output_shift, int activation_min, int activation_max, "
49+
"*, Tensor(a!) out) -> Tensor(a!)"
4650
)
4751

4852

@@ -154,6 +158,8 @@ def quantized_mul_int8_meta(
154158
output_offset,
155159
output_multiplier,
156160
output_shift,
161+
activation_min,
162+
activation_max,
157163
):
158164
return torch.empty_like(input1)
159165

@@ -167,6 +173,8 @@ def quantized_mul_broadcast_int8_meta(
167173
output_offset,
168174
output_multiplier,
169175
output_shift,
176+
activation_min,
177+
activation_max,
170178
):
171179
return input1.new_empty(torch.broadcast_shapes(input1.shape, input2.shape))
172180

@@ -181,12 +189,14 @@ def quantized_mul_int8(
181189
output_offset,
182190
output_multiplier,
183191
output_shift,
192+
activation_min,
193+
activation_max,
184194
):
185195
product = (input1.to(torch.int32) + input1_offset) * (
186196
input2.to(torch.int32) + input2_offset
187197
)
188198
result = requantize(product, output_multiplier, output_shift) + output_offset
189-
return result.clamp(-128, 127).to(torch.int8)
199+
return result.clamp(activation_min, activation_max).to(torch.int8)
190200

191201

192202
@register_fake("esp_nn::quantized_add_int8")
@@ -203,6 +213,8 @@ def quantized_add_int8_meta(
203213
left_shift,
204214
output_multiplier,
205215
output_shift,
216+
activation_min,
217+
activation_max,
206218
):
207219
return torch.empty_like(input1)
208220

@@ -221,14 +233,16 @@ def quantized_add_int8(
221233
left_shift,
222234
output_multiplier,
223235
output_shift,
236+
activation_min,
237+
activation_max,
224238
):
225239
lhs = (input1.to(torch.int32) + input1_offset) << left_shift
226240
rhs = (input2.to(torch.int32) + input2_offset) << left_shift
227241
total = requantize(lhs, input1_multiplier, input1_shift) + requantize(
228242
rhs, input2_multiplier, input2_shift
229243
)
230244
result = requantize(total, output_multiplier, output_shift) + output_offset
231-
return result.clamp(-128, 127).to(torch.int8)
245+
return result.clamp(activation_min, activation_max).to(torch.int8)
232246

233247

234248
@register_fake("esp_nn::quantized_linear_int8")

‎backends/espressif/ops/operators.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,19 @@
1010
- arg_meta: null
1111
kernel_name: espressif::quantized_lut_int8_out
1212

13-
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
13+
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
1414
variants: function
1515
kernels:
1616
- arg_meta: null
1717
kernel_name: espressif::quantized_mul_int8_out
1818

19-
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
19+
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2020
variants: function
2121
kernels:
2222
- arg_meta: null
2323
kernel_name: espressif::quantized_mul_broadcast_int8_out
2424

25-
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
25+
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2626
variants: function
2727
kernels:
2828
- arg_meta: null

‎backends/espressif/passes/esp_pass_manager.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from torch.fx import GraphModule
3030

3131
from .decompose_activations import DecomposeActivations
32+
from .fold_activations import FoldActivations
3233
from .lower_to_esp import LowerToEsp
3334

3435

@@ -46,6 +47,7 @@ def __init__(
4647
ToChannelsLast(),
4748
# Refresh contiguous tensor metadata after the layout rewrites.
4849
ExportPass(),
50+
FoldActivations(),
4951
LowerToEsp(target=target, kernel_mode=kernel_mode),
5052
ToChannelsFirst(),
5153
DecomposeFusedQuant(),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import math
8+
from dataclasses import replace
9+
10+
import torch
11+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
12+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
13+
from executorch.backends.transforms.utils import get_constant
14+
from executorch.exir.dialects._ops import ops as exir_ops
15+
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
16+
17+
18+
class FoldActivations(ExportedProgramPassBase):
19+
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
20+
21+
def call(self, exported_program):
22+
graph = exported_program.graph
23+
modified = False
24+
for activation in list(graph.nodes):
25+
if activation.target not in (
26+
exir_ops.edge.fused_quant.relu.default,
27+
exir_ops.edge.fused_quant.hard_tanh.default,
28+
):
29+
continue
30+
bounds = (
31+
(get_arg(activation, "min_val"), get_arg(activation, "max_val"))
32+
if activation.target == exir_ops.edge.fused_quant.hard_tanh.default
33+
else (0.0, float("inf"))
34+
)
35+
if not all(isinstance(bound, (int, float)) for bound in bounds) or not (
36+
bounds[0] <= bounds[1]
37+
):
38+
continue
39+
producer = get_arg(activation, "inp")
40+
if (
41+
producer.target
42+
not in (
43+
exir_ops.edge.fused_quant.add.default,
44+
exir_ops.edge.fused_quant.sub.default,
45+
exir_ops.edge.fused_quant.mul.default,
46+
)
47+
or len(producer.users) != 1
48+
or get_qparams_from_node(producer, "out") is not None
49+
or get_qparams_from_node(activation, "inp") is not None
50+
):
51+
continue
52+
params = get_qparams_from_node(activation, "out")
53+
if (
54+
params is None
55+
or not params.is_per_tensor()
56+
or params.dtype != torch.int8
57+
or (params.quant_min, params.quant_max) != (-128, 127)
58+
):
59+
continue
60+
scale = get_constant(exported_program, params.scale)
61+
zero_point = get_constant(exported_program, params.zero_point)
62+
if (
63+
scale is None
64+
or zero_point is None
65+
or not math.isfinite(scale.item())
66+
or scale.item() <= 0
67+
or not -128 <= zero_point.item() <= 127
68+
):
69+
continue
70+
concrete = replace(params, scale=scale, zero_point=zero_point)
71+
lower, upper = concrete.quantize(
72+
torch.tensor(bounds, dtype=producer.meta["val"].dtype)
73+
).tolist()
74+
for field in ("scale", "zero_point", "dtype"):
75+
set_arg(producer, f"out_{field}", get_arg(activation, f"out_{field}"))
76+
set_arg(producer, "out_quant_min", lower)
77+
set_arg(producer, "out_quant_max", upper)
78+
producer.meta["val"] = activation.meta["val"]
79+
activation.replace_all_uses_with(producer)
80+
graph.erase_node(activation)
81+
modified = True
82+
if modified:
83+
graph.eliminate_dead_code()
84+
graph.lint()
85+
exported_program.graph_module.recompile()
86+
return ExportedProgramPassResult(exported_program, modified)

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ def _binary_kernel_args(node, params, rescale_fn) -> tuple | None:
146146
)
147147

148148

149-
def _qparams(program, node, prefix):
149+
def _qparams(program, node, prefix, *, allow_output_clamp=False):
150150
params = get_qparams_from_node(node, prefix)
151151
dtype = torch.int8 if prefix == "out" else torch.float32
152152
if params is None or not params.is_per_tensor() or params.dtype != dtype:
@@ -166,7 +166,10 @@ def _qparams(program, node, prefix):
166166
raise ValueError(f"{node.name}: {prefix} scale must be finite and positive")
167167
if not -128 <= zero_point <= 127:
168168
raise ValueError(f"{node.name}: {prefix} has an invalid int8 zero point")
169-
if (params.quant_min, params.quant_max) != (-128, 127):
169+
if not (prefix == "out" and allow_output_clamp) and (
170+
params.quant_min,
171+
params.quant_max,
172+
) != (-128, 127):
170173
return None
171174
return _QParams(scale, zero_point)
172175

@@ -748,7 +751,7 @@ def call(self, exported_program):
748751
continue
749752
target, rescale_fn = lowering
750753
params = [
751-
_qparams(exported_program, node, prefix)
754+
_qparams(exported_program, node, prefix, allow_output_clamp=True)
752755
for prefix in ("inp", "other", "out")
753756
]
754757
if any(param is None for param in params):
@@ -762,7 +765,14 @@ def call(self, exported_program):
762765
):
763766
target = exir_ops.edge.esp_nn.quantized_mul_broadcast_int8.default
764767
with graph.inserting_before(node):
765-
replacement = graph.call_function(target, args)
768+
replacement = graph.call_function(
769+
target,
770+
(
771+
*args,
772+
get_arg(node, "out_quant_min"),
773+
get_arg(node, "out_quant_max"),
774+
),
775+
)
766776
replacement.meta = node.meta.copy()
767777
node.replace_all_uses_with(replacement)
768778
graph.erase_node(node)

0 commit comments

Comments
 (0)