Skip to content

Commit 5fe214a

Browse files
committed
[Espressif] Fuse binary ReLU and HardTanh patterns
Quantize single-consumer add, subtract and multiply followed by ReLU or HardTanh, including ReLU6, at their pattern boundaries so the reference has no intermediate rounding. Fold the activation into Fused Quant output qparams and clamp bounds before ESP replacement. Reuse existing shape and arithmetic eligibility checks with the final output scale; unsupported producers retain portable execution. Forked and separately quantized activations keep their original boundaries. Clamp bounds use the producer's floating dtype to preserve BF16 and FP16 portable fallback results. Read the quantizer and FoldActivations first, then the binary lowering and schema/wrapper changes. Shared dialect and S3 implementation cases cover forks, broadcasts, saturation, fallback and narrower clamp grids, including asymmetric, positive and negative HardTanh bounds and rounding at endpoints. Authored with OpenAI Codex.
1 parent 3025afd commit 5fe214a

14 files changed

Lines changed: 547 additions & 29 deletions

‎backends/espressif/ops/op_quantized_add.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Tensor& quantized_add_int8_out(
3535
int64_t left_shift,
3636
int64_t output_multiplier,
3737
int64_t output_shift,
38+
int64_t activation_min,
39+
int64_t activation_max,
3840
Tensor& out) {
3941
const auto numel = input1.numel();
4042
ET_KERNEL_CHECK(
@@ -64,8 +66,8 @@ Tensor& quantized_add_int8_out(
6466
static_cast<int32_t>(output_offset),
6567
static_cast<int32_t>(output_multiplier),
6668
static_cast<int32_t>(output_shift),
67-
std::numeric_limits<int8_t>::min(),
68-
std::numeric_limits<int8_t>::max(),
69+
static_cast<int32_t>(activation_min),
70+
static_cast<int32_t>(activation_max),
6971
static_cast<int32_t>(numel));
7072
return out;
7173
}

‎backends/espressif/ops/op_quantized_mul.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
ET_KERNEL_CHECK(
@@ -54,8 +56,8 @@ Tensor& quantized_mul_int8_out(
5456
static_cast<int32_t>(output_offset),
5557
static_cast<int32_t>(output_multiplier),
5658
static_cast<int32_t>(output_shift),
57-
std::numeric_limits<int8_t>::min(),
58-
std::numeric_limits<int8_t>::max(),
59+
static_cast<int32_t>(activation_min),
60+
static_cast<int32_t>(activation_max),
5961
static_cast<int32_t>(numel));
6062
return out;
6163
}

‎backends/espressif/ops/op_quantized_mul_broadcast.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_broadcast_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
const auto block_size = input2.numel();
@@ -57,8 +59,8 @@ Tensor& quantized_mul_broadcast_int8_out(
5759
static_cast<int32_t>(output_offset),
5860
static_cast<int32_t>(output_multiplier),
5961
static_cast<int32_t>(output_shift),
60-
std::numeric_limits<int8_t>::min(),
61-
std::numeric_limits<int8_t>::max(),
62+
static_cast<int32_t>(activation_min),
63+
static_cast<int32_t>(activation_max),
6264
static_cast<int32_t>(numel / block_size),
6365
static_cast<int32_t>(block_size));
6466
return out;

‎backends/espressif/ops/operators.py‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,33 +16,37 @@
1616
)
1717
lib.define(
1818
"quantized_mul_int8(Tensor input1, Tensor input2, int input1_offset, "
19-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
19+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
20+
"int activation_min, int activation_max) -> Tensor"
2021
)
2122
lib.define(
2223
"quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, "
2324
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
24-
"*, Tensor(a!) out) -> Tensor(a!)"
25+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
2526
)
2627
lib.define(
2728
"quantized_mul_broadcast_int8(Tensor input1, Tensor input2, int input1_offset, "
28-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
29+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
30+
"int activation_min, int activation_max) -> Tensor"
2931
)
3032
lib.define(
3133
"quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, "
3234
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
33-
"*, Tensor(a!) out) -> Tensor(a!)"
35+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
3436
)
3537
lib.define(
3638
"quantized_add_int8(Tensor input1, Tensor input2, int input1_offset, "
3739
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
3840
"int input2_multiplier, int input2_shift, int left_shift, "
39-
"int output_multiplier, int output_shift) -> Tensor"
41+
"int output_multiplier, int output_shift, "
42+
"int activation_min, int activation_max) -> Tensor"
4043
)
4144
lib.define(
4245
"quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, "
4346
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
4447
"int input2_multiplier, int input2_shift, int left_shift, "
45-
"int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)"
48+
"int output_multiplier, int output_shift, int activation_min, int activation_max, "
49+
"*, Tensor(a!) out) -> Tensor(a!)"
4650
)
4751

4852

@@ -154,6 +158,8 @@ def quantized_mul_int8_meta(
154158
output_offset,
155159
output_multiplier,
156160
output_shift,
161+
activation_min,
162+
activation_max,
157163
):
158164
return torch.empty_like(input1)
159165

@@ -167,6 +173,8 @@ def quantized_mul_broadcast_int8_meta(
167173
output_offset,
168174
output_multiplier,
169175
output_shift,
176+
activation_min,
177+
activation_max,
170178
):
171179
return input1.new_empty(torch.broadcast_shapes(input1.shape, input2.shape))
172180

@@ -181,12 +189,14 @@ def quantized_mul_int8(
181189
output_offset,
182190
output_multiplier,
183191
output_shift,
192+
activation_min,
193+
activation_max,
184194
):
185195
product = (input1.to(torch.int32) + input1_offset) * (
186196
input2.to(torch.int32) + input2_offset
187197
)
188198
result = requantize(product, output_multiplier, output_shift) + output_offset
189-
return result.clamp(-128, 127).to(torch.int8)
199+
return result.clamp(activation_min, activation_max).to(torch.int8)
190200

191201

192202
@register_fake("esp_nn::quantized_add_int8")
@@ -203,6 +213,8 @@ def quantized_add_int8_meta(
203213
left_shift,
204214
output_multiplier,
205215
output_shift,
216+
activation_min,
217+
activation_max,
206218
):
207219
return torch.empty_like(input1)
208220

@@ -221,14 +233,16 @@ def quantized_add_int8(
221233
left_shift,
222234
output_multiplier,
223235
output_shift,
236+
activation_min,
237+
activation_max,
224238
):
225239
lhs = (input1.to(torch.int32) + input1_offset) << left_shift
226240
rhs = (input2.to(torch.int32) + input2_offset) << left_shift
227241
total = requantize(lhs, input1_multiplier, input1_shift) + requantize(
228242
rhs, input2_multiplier, input2_shift
229243
)
230244
result = requantize(total, output_multiplier, output_shift) + output_offset
231-
return result.clamp(-128, 127).to(torch.int8)
245+
return result.clamp(activation_min, activation_max).to(torch.int8)
232246

233247

234248
@register_fake("esp_nn::quantized_linear_int8")

‎backends/espressif/ops/operators.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,19 @@
1010
- arg_meta: null
1111
kernel_name: espressif::quantized_lut_int8_out
1212

13-
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
13+
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
1414
variants: function
1515
kernels:
1616
- arg_meta: null
1717
kernel_name: espressif::quantized_mul_int8_out
1818

19-
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
19+
- func: esp_nn::quantized_mul_broadcast_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2020
variants: function
2121
kernels:
2222
- arg_meta: null
2323
kernel_name: espressif::quantized_mul_broadcast_int8_out
2424

25-
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
25+
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2626
variants: function
2727
kernels:
2828
- arg_meta: null

‎backends/espressif/passes/esp_pass_manager.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from torch.fx import GraphModule
3030

3131
from .decompose_activations import DecomposeActivations
32+
from .fold_activations import FoldActivations
3233
from .lower_to_esp import LowerToEsp
3334

3435

@@ -46,6 +47,7 @@ def __init__(
4647
ToChannelsLast(),
4748
# Refresh contiguous tensor metadata after the layout rewrites.
4849
ExportPass(),
50+
FoldActivations(),
4951
LowerToEsp(target=target, kernel_mode=kernel_mode),
5052
ToChannelsFirst(),
5153
DecomposeFusedQuant(),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import math
8+
from dataclasses import replace
9+
10+
import torch
11+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
12+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
13+
from executorch.backends.transforms.utils import get_constant
14+
from executorch.exir.dialects._ops import ops as exir_ops
15+
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
16+
17+
18+
class FoldActivations(ExportedProgramPassBase):
19+
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
20+
21+
def call(self, exported_program):
22+
graph = exported_program.graph
23+
modified = False
24+
for activation in list(graph.nodes):
25+
if activation.target not in (
26+
exir_ops.edge.fused_quant.relu.default,
27+
exir_ops.edge.fused_quant.hard_tanh.default,
28+
):
29+
continue
30+
bounds = (
31+
(get_arg(activation, "min_val"), get_arg(activation, "max_val"))
32+
if activation.target == exir_ops.edge.fused_quant.hard_tanh.default
33+
else (0.0, float("inf"))
34+
)
35+
if not all(isinstance(bound, (int, float)) for bound in bounds) or not (
36+
bounds[0] <= bounds[1]
37+
):
38+
continue
39+
producer = get_arg(activation, "inp")
40+
if (
41+
producer.target
42+
not in (
43+
exir_ops.edge.fused_quant.add.default,
44+
exir_ops.edge.fused_quant.sub.default,
45+
exir_ops.edge.fused_quant.mul.default,
46+
)
47+
or len(producer.users) != 1
48+
or get_qparams_from_node(producer, "out") is not None
49+
or get_qparams_from_node(activation, "inp") is not None
50+
):
51+
continue
52+
params = get_qparams_from_node(activation, "out")
53+
if (
54+
params is None
55+
or not params.is_per_tensor()
56+
or params.dtype != torch.int8
57+
or (params.quant_min, params.quant_max) != (-128, 127)
58+
):
59+
continue
60+
scale = get_constant(exported_program, params.scale)
61+
zero_point = get_constant(exported_program, params.zero_point)
62+
if (
63+
scale is None
64+
or zero_point is None
65+
or not math.isfinite(scale.item())
66+
or scale.item() <= 0
67+
or not -128 <= zero_point.item() <= 127
68+
):
69+
continue
70+
concrete = replace(params, scale=scale, zero_point=zero_point)
71+
lower, upper = concrete.quantize(
72+
torch.tensor(bounds, dtype=producer.meta["val"].dtype)
73+
).tolist()
74+
for field in ("scale", "zero_point", "dtype"):
75+
set_arg(producer, f"out_{field}", get_arg(activation, f"out_{field}"))
76+
set_arg(producer, "out_quant_min", lower)
77+
set_arg(producer, "out_quant_max", upper)
78+
producer.meta["val"] = activation.meta["val"]
79+
activation.replace_all_uses_with(producer)
80+
graph.erase_node(activation)
81+
modified = True
82+
if modified:
83+
graph.eliminate_dead_code()
84+
graph.lint()
85+
exported_program.graph_module.recompile()
86+
return ExportedProgramPassResult(exported_program, modified)

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def _prepare_binary(node, params, target, rescale_fn) -> tuple | None:
148148
)
149149

150150

151-
def _qparams(program, node, prefix):
151+
def _qparams(program, node, prefix, *, allow_output_clamp=False):
152152
params = get_qparams_from_node(node, prefix)
153153
dtype = torch.int8 if prefix == "out" else torch.float32
154154
if params is None or not params.is_per_tensor() or params.dtype != dtype:
@@ -168,7 +168,10 @@ def _qparams(program, node, prefix):
168168
raise ValueError(f"{node.name}: {prefix} scale must be finite and positive")
169169
if not -128 <= zero_point <= 127:
170170
raise ValueError(f"{node.name}: {prefix} has an invalid int8 zero point")
171-
if (params.quant_min, params.quant_max) != (-128, 127):
171+
if not (prefix == "out" and allow_output_clamp) and (
172+
params.quant_min,
173+
params.quant_max,
174+
) != (-128, 127):
172175
return None
173176
return _QParams(scale, zero_point)
174177

@@ -749,7 +752,7 @@ def call(self, exported_program):
749752
continue
750753
target, rescale_fn = lowering
751754
params = [
752-
_qparams(exported_program, node, prefix)
755+
_qparams(exported_program, node, prefix, allow_output_clamp=True)
753756
for prefix in ("inp", "other", "out")
754757
]
755758
if any(param is None for param in params):
@@ -759,7 +762,14 @@ def call(self, exported_program):
759762
continue
760763
target, args = prepared
761764
with graph.inserting_before(node):
762-
replacement = graph.call_function(target, args)
765+
replacement = graph.call_function(
766+
target,
767+
(
768+
*args,
769+
get_arg(node, "out_quant_min"),
770+
get_arg(node, "out_quant_max"),
771+
),
772+
)
763773
replacement.meta = node.meta.copy()
764774
node.replace_all_uses_with(replacement)
765775
graph.erase_node(node)

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
106106
Conv2d, and depthwise Conv2d biases use
107107
int32 with scales derived from the input and weight observers.
108108
AvgPool2d without divisor_override shares input and output qparams.
109+
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
110+
only their boundaries.
109111
EspPassManager lowers supported Fused Quant operators to ESP-NN and
110112
decomposes the rest for portable execution. Use EspNNQuantizer for the
111113
EspNNBackend delegate instead.
@@ -145,6 +147,45 @@ def __init__(self) -> None:
145147
]
146148
)
147149

150+
def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
151+
groups = []
152+
for activation in model.graph.nodes:
153+
if activation.target not in (
154+
torch.ops.aten.relu.default,
155+
torch.ops.aten.hardtanh.default,
156+
):
157+
continue
158+
producer = activation.args[0]
159+
if (
160+
producer.target
161+
not in (
162+
torch.ops.aten.add.Tensor,
163+
torch.ops.aten.sub.Tensor,
164+
torch.ops.aten.mul.Tensor,
165+
)
166+
or len(producer.users) != 1
167+
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
168+
or Q_ANNOTATION_KEY in producer.meta
169+
or Q_ANNOTATION_KEY in activation.meta
170+
):
171+
continue
172+
if (
173+
producer.target != torch.ops.aten.mul.Tensor
174+
and get_arg(producer, "alpha") != 1
175+
):
176+
continue
177+
groups.append((producer, activation))
178+
179+
super().annotate(model)
180+
for producer, activation in groups:
181+
producer_annotation = producer.meta.get(Q_ANNOTATION_KEY)
182+
activation_annotation = activation.meta.get(Q_ANNOTATION_KEY)
183+
if producer_annotation is not None and activation_annotation is not None:
184+
# Observe the pattern's output, without rounding its internal edge.
185+
producer_annotation.output_qspec = None
186+
activation_annotation.input_qspec_map[producer] = None
187+
return model
188+
148189

149190
def get_esp_fused_quant_quantizer() -> FusedQuantQuantizer:
150191
"""Create the quantizer for the Espressif Fused Quant operator-library flow."""

0 commit comments

Comments
 (0)