Skip to content

Commit e5cd902

Browse files
committed
[Espressif] Fuse binary ReLU and HardTanh patterns
Quantize single-consumer add, subtract and multiply followed by ReLU or HardTanh, including ReLU6, at their pattern boundaries so the reference has no intermediate rounding. Fold the activation into Fused Quant output qparams and clamp bounds before ESP replacement. Reuse existing shape and arithmetic eligibility checks with the final output scale; unsupported producers retain portable execution. Forked and separately quantized activations keep their original boundaries. Clamp bounds use the producer's floating dtype to preserve BF16 and FP16 portable fallback results. Read the quantizer and FoldActivations first, then the binary lowering and schema/wrapper changes. Shared dialect and S3 implementation cases cover forks, broadcasts, saturation, fallback and narrower clamp grids, including asymmetric, positive and negative HardTanh bounds and rounding at endpoints. Authored with OpenAI Codex.
1 parent 4c6048b commit e5cd902

14 files changed

Lines changed: 526 additions & 27 deletions

‎backends/espressif/README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,13 @@ Both flows use ESP-NN 1.5.0 kernels. The operator library currently accelerates
2828
int8 tensor add and subtract with equal input shapes and `alpha=1`. Subtraction
2929
uses the add kernel with a negative second-input multiplier. Multiply supports
3030
equal shapes or a contiguous trailing block broadcast across the full tensor,
31-
including one-element tensors. Sigmoid, tanh, GELU, SiLU, HardSwish, HardSigmoid,
31+
including one-element tensors. A single-consumer add, subtract, or multiply
32+
followed by ReLU or HardTanh (including ReLU6) is quantized as one pattern,
33+
with no intermediate rounding. HardTanh supports constant lower and upper
34+
bounds. The activation's output qparams and clamp bounds are folded into
35+
the binary kernel; unsupported binary configurations retain portable execution.
36+
Forked or separately quantized activations keep their quantization boundaries.
37+
Sigmoid, tanh, GELU, SiLU, HardSwish, HardSigmoid,
3238
ReLU, and HardTanh (including ReLU6) support static, contiguous int8 tensors with
3339
constant per-tensor input and output qparams. Export evaluates all 256 input
3440
codes with the FQ reference, using calibrated qparams, and stores the results

‎backends/espressif/ops/op_quantized_add.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Tensor& quantized_add_int8_out(
3535
int64_t left_shift,
3636
int64_t output_multiplier,
3737
int64_t output_shift,
38+
int64_t activation_min,
39+
int64_t activation_max,
3840
Tensor& out) {
3941
const auto numel = input1.numel();
4042
ET_KERNEL_CHECK(
@@ -64,8 +66,8 @@ Tensor& quantized_add_int8_out(
6466
static_cast<int32_t>(output_offset),
6567
static_cast<int32_t>(output_multiplier),
6668
static_cast<int32_t>(output_shift),
67-
std::numeric_limits<int8_t>::min(),
68-
std::numeric_limits<int8_t>::max(),
69+
static_cast<int32_t>(activation_min),
70+
static_cast<int32_t>(activation_max),
6971
static_cast<int32_t>(numel));
7072
return out;
7173
}

‎backends/espressif/ops/op_quantized_mul.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_mul_int8_out(
3030
int64_t output_offset,
3131
int64_t output_multiplier,
3232
int64_t output_shift,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
const auto numel = input1.numel();
3537
const auto block_size = input2.numel();
@@ -58,8 +60,8 @@ Tensor& quantized_mul_int8_out(
5860
static_cast<int32_t>(output_offset),
5961
static_cast<int32_t>(output_multiplier),
6062
static_cast<int32_t>(output_shift),
61-
std::numeric_limits<int8_t>::min(),
62-
std::numeric_limits<int8_t>::max(),
63+
static_cast<int32_t>(activation_min),
64+
static_cast<int32_t>(activation_max),
6365
static_cast<int32_t>(numel));
6466
} else {
6567
esp_nn_mul_broadcast_channel_s8(
@@ -71,8 +73,8 @@ Tensor& quantized_mul_int8_out(
7173
static_cast<int32_t>(output_offset),
7274
static_cast<int32_t>(output_multiplier),
7375
static_cast<int32_t>(output_shift),
74-
std::numeric_limits<int8_t>::min(),
75-
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(activation_min),
77+
static_cast<int32_t>(activation_max),
7678
static_cast<int32_t>(numel / block_size),
7779
static_cast<int32_t>(block_size));
7880
}

‎backends/espressif/ops/operators.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,27 @@
1616
)
1717
lib.define(
1818
"quantized_mul_int8(Tensor input1, Tensor input2, int input1_offset, "
19-
"int input2_offset, int output_offset, int output_multiplier, int output_shift) -> Tensor"
19+
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
20+
"int activation_min, int activation_max) -> Tensor"
2021
)
2122
lib.define(
2223
"quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, "
2324
"int input2_offset, int output_offset, int output_multiplier, int output_shift, "
24-
"*, Tensor(a!) out) -> Tensor(a!)"
25+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
2526
)
2627
lib.define(
2728
"quantized_add_int8(Tensor input1, Tensor input2, int input1_offset, "
2829
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
2930
"int input2_multiplier, int input2_shift, int left_shift, "
30-
"int output_multiplier, int output_shift) -> Tensor"
31+
"int output_multiplier, int output_shift, "
32+
"int activation_min, int activation_max) -> Tensor"
3133
)
3234
lib.define(
3335
"quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, "
3436
"int input2_offset, int output_offset, int input1_multiplier, int input1_shift, "
3537
"int input2_multiplier, int input2_shift, int left_shift, "
36-
"int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)"
38+
"int output_multiplier, int output_shift, int activation_min, int activation_max, "
39+
"*, Tensor(a!) out) -> Tensor(a!)"
3740
)
3841

3942

@@ -145,6 +148,8 @@ def quantized_mul_int8_meta(
145148
output_offset,
146149
output_multiplier,
147150
output_shift,
151+
activation_min,
152+
activation_max,
148153
):
149154
return input1.new_empty(torch.broadcast_shapes(input1.shape, input2.shape))
150155

@@ -158,12 +163,14 @@ def quantized_mul_int8(
158163
output_offset,
159164
output_multiplier,
160165
output_shift,
166+
activation_min,
167+
activation_max,
161168
):
162169
product = (input1.to(torch.int32) + input1_offset) * (
163170
input2.to(torch.int32) + input2_offset
164171
)
165172
result = requantize(product, output_multiplier, output_shift) + output_offset
166-
return result.clamp(-128, 127).to(torch.int8)
173+
return result.clamp(activation_min, activation_max).to(torch.int8)
167174

168175

169176
@register_fake("esp_nn::quantized_add_int8")
@@ -180,6 +187,8 @@ def quantized_add_int8_meta(
180187
left_shift,
181188
output_multiplier,
182189
output_shift,
190+
activation_min,
191+
activation_max,
183192
):
184193
return torch.empty_like(input1)
185194

@@ -198,14 +207,16 @@ def quantized_add_int8(
198207
left_shift,
199208
output_multiplier,
200209
output_shift,
210+
activation_min,
211+
activation_max,
201212
):
202213
lhs = (input1.to(torch.int32) + input1_offset) << left_shift
203214
rhs = (input2.to(torch.int32) + input2_offset) << left_shift
204215
total = requantize(lhs, input1_multiplier, input1_shift) + requantize(
205216
rhs, input2_multiplier, input2_shift
206217
)
207218
result = requantize(total, output_multiplier, output_shift) + output_offset
208-
return result.clamp(-128, 127).to(torch.int8)
219+
return result.clamp(activation_min, activation_max).to(torch.int8)
209220

210221

211222
@register_fake("esp_nn::quantized_linear_int8")

‎backends/espressif/ops/operators.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@
1010
- arg_meta: null
1111
kernel_name: espressif::quantized_lut_int8_out
1212

13-
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
13+
- func: esp_nn::quantized_mul_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
1414
variants: function
1515
kernels:
1616
- arg_meta: null
1717
kernel_name: espressif::quantized_mul_int8_out
1818

19-
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
19+
- func: esp_nn::quantized_add_int8.out(Tensor input1, Tensor input2, int input1_offset, int input2_offset, int output_offset, int input1_multiplier, int input1_shift, int input2_multiplier, int input2_shift, int left_shift, int output_multiplier, int output_shift, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
2020
variants: function
2121
kernels:
2222
- arg_meta: null

‎backends/espressif/passes/esp_pass_manager.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from torch.fx import GraphModule
2929

3030
from .decompose_activations import DecomposeActivations
31+
from .fold_activations import FoldActivations
3132
from .lower_to_esp import LowerToEsp
3233

3334

@@ -45,6 +46,7 @@ def __init__(
4546
ToChannelsLast(),
4647
# Refresh contiguous tensor metadata after the layout rewrites.
4748
ExportPass(),
49+
FoldActivations(),
4850
LowerToEsp(target=target, kernel_mode=kernel_mode),
4951
ToChannelsFirst(),
5052
DecomposeFusedQuant(),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import math
8+
from dataclasses import replace
9+
10+
import torch
11+
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
12+
from executorch.backends.transforms.permute_pass_utils import get_arg, set_arg
13+
from executorch.backends.transforms.utils import get_constant
14+
from executorch.exir.dialects._ops import ops as exir_ops
15+
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
16+
17+
18+
class FoldActivations(ExportedProgramPassBase):
19+
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
20+
21+
def call(self, exported_program):
22+
graph = exported_program.graph
23+
modified = False
24+
for activation in list(graph.nodes):
25+
if activation.target not in (
26+
exir_ops.edge.fused_quant.relu.default,
27+
exir_ops.edge.fused_quant.hard_tanh.default,
28+
):
29+
continue
30+
bounds = (
31+
(get_arg(activation, "min_val"), get_arg(activation, "max_val"))
32+
if activation.target == exir_ops.edge.fused_quant.hard_tanh.default
33+
else (0.0, float("inf"))
34+
)
35+
if not all(isinstance(bound, (int, float)) for bound in bounds) or not (
36+
bounds[0] <= bounds[1]
37+
):
38+
continue
39+
producer = get_arg(activation, "inp")
40+
if (
41+
producer.target
42+
not in (
43+
exir_ops.edge.fused_quant.add.default,
44+
exir_ops.edge.fused_quant.sub.default,
45+
exir_ops.edge.fused_quant.mul.default,
46+
)
47+
or len(producer.users) != 1
48+
or get_qparams_from_node(producer, "out") is not None
49+
or get_qparams_from_node(activation, "inp") is not None
50+
):
51+
continue
52+
params = get_qparams_from_node(activation, "out")
53+
if (
54+
params is None
55+
or not params.is_per_tensor()
56+
or params.dtype != torch.int8
57+
or (params.quant_min, params.quant_max) != (-128, 127)
58+
):
59+
continue
60+
scale = get_constant(exported_program, params.scale)
61+
zero_point = get_constant(exported_program, params.zero_point)
62+
if (
63+
scale is None
64+
or zero_point is None
65+
or not math.isfinite(scale.item())
66+
or scale.item() <= 0
67+
or not -128 <= zero_point.item() <= 127
68+
):
69+
continue
70+
concrete = replace(params, scale=scale, zero_point=zero_point)
71+
lower, upper = concrete.quantize(
72+
torch.tensor(bounds, dtype=producer.meta["val"].dtype)
73+
).tolist()
74+
for field in ("scale", "zero_point", "dtype"):
75+
set_arg(producer, f"out_{field}", get_arg(activation, f"out_{field}"))
76+
set_arg(producer, "out_quant_min", lower)
77+
set_arg(producer, "out_quant_max", upper)
78+
producer.meta["val"] = activation.meta["val"]
79+
activation.replace_all_uses_with(producer)
80+
graph.erase_node(activation)
81+
modified = True
82+
if modified:
83+
graph.eliminate_dead_code()
84+
graph.lint()
85+
exported_program.graph_module.recompile()
86+
return ExportedProgramPassResult(exported_program, modified)

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ def _binary_kernel_args(node, params, rescale_fn) -> tuple | None:
146146
)
147147

148148

149-
def _qparams(program, node, prefix):
149+
def _qparams(program, node, prefix, *, allow_output_clamp=False):
150150
params = get_qparams_from_node(node, prefix)
151151
dtype = torch.int8 if prefix == "out" else torch.float32
152152
if params is None or not params.is_per_tensor() or params.dtype != dtype:
@@ -166,7 +166,10 @@ def _qparams(program, node, prefix):
166166
raise ValueError(f"{node.name}: {prefix} scale must be finite and positive")
167167
if not -128 <= zero_point <= 127:
168168
raise ValueError(f"{node.name}: {prefix} has an invalid int8 zero point")
169-
if (params.quant_min, params.quant_max) != (-128, 127):
169+
if not (prefix == "out" and allow_output_clamp) and (
170+
params.quant_min,
171+
params.quant_max,
172+
) != (-128, 127):
170173
return None
171174
return _QParams(scale, zero_point)
172175

@@ -748,7 +751,7 @@ def call(self, exported_program):
748751
continue
749752
target, rescale_fn = lowering
750753
params = [
751-
_qparams(exported_program, node, prefix)
754+
_qparams(exported_program, node, prefix, allow_output_clamp=True)
752755
for prefix in ("inp", "other", "out")
753756
]
754757
if any(param is None for param in params):
@@ -757,7 +760,14 @@ def call(self, exported_program):
757760
if args is None:
758761
continue
759762
with graph.inserting_before(node):
760-
replacement = graph.call_function(target, args)
763+
replacement = graph.call_function(
764+
target,
765+
(
766+
*args,
767+
get_arg(node, "out_quant_min"),
768+
get_arg(node, "out_quant_max"),
769+
),
770+
)
761771
replacement.meta = node.meta.copy()
762772
node.replace_all_uses_with(replacement)
763773
graph.erase_node(node)

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
106106
Conv2d, and depthwise Conv2d biases use
107107
int32 with scales derived from the input and weight observers.
108108
AvgPool2d without divisor_override shares input and output qparams.
109+
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
110+
only their boundaries.
109111
EspPassManager lowers supported Fused Quant operators to ESP-NN and
110112
decomposes the rest for portable execution. Use EspNNQuantizer for the
111113
EspNNBackend delegate instead.
@@ -145,6 +147,45 @@ def __init__(self) -> None:
145147
]
146148
)
147149

150+
def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
151+
groups = []
152+
for activation in model.graph.nodes:
153+
if activation.target not in (
154+
torch.ops.aten.relu.default,
155+
torch.ops.aten.hardtanh.default,
156+
):
157+
continue
158+
producer = activation.args[0]
159+
if (
160+
producer.target
161+
not in (
162+
torch.ops.aten.add.Tensor,
163+
torch.ops.aten.sub.Tensor,
164+
torch.ops.aten.mul.Tensor,
165+
)
166+
or len(producer.users) != 1
167+
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
168+
or Q_ANNOTATION_KEY in producer.meta
169+
or Q_ANNOTATION_KEY in activation.meta
170+
):
171+
continue
172+
if (
173+
producer.target != torch.ops.aten.mul.Tensor
174+
and get_arg(producer, "alpha") != 1
175+
):
176+
continue
177+
groups.append((producer, activation))
178+
179+
super().annotate(model)
180+
for producer, activation in groups:
181+
producer_annotation = producer.meta.get(Q_ANNOTATION_KEY)
182+
activation_annotation = activation.meta.get(Q_ANNOTATION_KEY)
183+
if producer_annotation is not None and activation_annotation is not None:
184+
# Observe the pattern's output, without rounding its internal edge.
185+
producer_annotation.output_qspec = None
186+
activation_annotation.input_qspec_map[producer] = None
187+
return model
188+
148189

149190
def get_esp_fused_quant_quantizer() -> FusedQuantQuantizer:
150191
"""Create the quantizer for the Espressif Fused Quant operator-library flow."""

‎backends/espressif/test/ops/test_add.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ class EspAddRelu(EspAdd):
7777
ops_after_transforms = {
7878
_ADD: 0,
7979
_ESP_ADD: 1,
80-
exir_ops.edge.esp_nn.quantized_lut_int8.default: 1,
80+
exir_ops.edge.esp_nn.quantized_lut_int8.default: 0,
8181
exir_ops.edge.aten.relu.default: 0,
8282
}
8383

0 commit comments

Comments
 (0)