Skip to content

Commit fe3fa73

Browse files
committed
[Espressif] Fuse linear ReLU and HardTanh patterns
Quantize single-consumer linear followed by ReLU or HardTanh, including ReLU6, at the pattern boundaries. Fold the final output qparams and clamp bounds into the existing ESP-NN fully connected call, preserving per-channel weights and derived int32 bias. Forked and separately quantized activations retain their original quantization boundaries. Read the quantizer and activation fold before the linear lowering and schema/wrapper updates. Shared dialect and S3 cases cover bias/no bias, forks, saturation, vector and higher-rank inputs, and narrower clamp grids with asymmetric, positive-only and negative-only HardTanh bounds. Authored with OpenAI Codex.
1 parent 7cf3893 commit fe3fa73

9 files changed

Lines changed: 200 additions & 31 deletions

File tree

‎backends/espressif/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,11 @@ Linear supports static, contiguous int8 inputs of rank one or higher, constant
5353
per-output-channel symmetric int8 weights, and optional constant int32 bias.
5454
`get_esp_fused_quant_quantizer` derives the bias scales from input and weight scales.
5555
Leading input dimensions are treated as batch rows by ESP-NN's batched fully
56-
connected API. Unsupported linear calls fail at export with the reason;
56+
connected API. Single-consumer linear followed by ReLU or HardTanh (including
57+
ReLU6) is quantized as one pattern. Its final output qparams and clamp bounds
58+
are folded into the linear kernel, preserving the derived int32 bias. Forked
59+
or separately quantized activations retain their boundaries.
60+
Unsupported linear calls fail at export with the reason;
5761
portable linear fallback is not available yet, even when CPU fallback is enabled.
5862
Conv2d supports static 4-D int8 inputs, constant per-output-channel symmetric
5963
weights, optional derived int32 bias, and unit dilation. Ordinary convolution

‎backends/espressif/ops/op_quantized_linear.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_linear_int8_out(
3030
const Tensor& output_shift,
3131
int64_t input_offset,
3232
int64_t output_offset,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
3537
const auto channels = weight.size(0);
@@ -71,8 +73,8 @@ Tensor& quantized_linear_int8_out(
7173
static_cast<int32_t>(output_offset),
7274
output_shift.const_data_ptr<int32_t>(),
7375
output_multiplier.const_data_ptr<int32_t>(),
74-
std::numeric_limits<int8_t>::min(),
75-
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(activation_min),
77+
static_cast<int32_t>(activation_max),
7678
static_cast<int32_t>(batches));
7779
return out;
7880
}

‎backends/espressif/ops/operators.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,13 @@
5858
)
5959
lib.define(
6060
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
61-
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
61+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
62+
"int activation_min, int activation_max) -> Tensor"
6263
)
6364
lib.define(
6465
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
6566
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
66-
"*, Tensor(a!) out) -> Tensor(a!)"
67+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
6768
)
6869

6970
lib.define(
@@ -221,14 +222,30 @@ def quantized_add_int8(
221222

222223
@register_fake("esp_nn::quantized_linear_int8")
223224
def quantized_linear_int8_meta(
224-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
225+
input,
226+
weight,
227+
bias,
228+
output_multiplier,
229+
output_shift,
230+
input_offset,
231+
output_offset,
232+
activation_min,
233+
activation_max,
225234
):
226235
return input.new_empty((*input.shape[:-1], weight.shape[0]))
227236

228237

229238
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
230239
def quantized_linear_int8(
231-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
240+
input,
241+
weight,
242+
bias,
243+
output_multiplier,
244+
output_shift,
245+
input_offset,
246+
output_offset,
247+
activation_min,
248+
activation_max,
232249
):
233250
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
234251
if bias is not None:
@@ -242,7 +259,7 @@ def quantized_linear_int8(
242259
],
243260
dim=-1,
244261
)
245-
return (result + output_offset).clamp(-128, 127).to(torch.int8)
262+
return (result + output_offset).clamp(activation_min, activation_max).to(torch.int8)
246263

247264

248265
@register_fake("esp_nn::quantized_conv2d_int8")
@@ -299,6 +316,8 @@ def quantized_conv2d_int8(
299316
output_shift,
300317
0,
301318
output_offset,
319+
-128,
320+
127,
302321
)
303322

304323

‎backends/espressif/ops/operators.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
- arg_meta: null
3535
kernel_name: espressif::quantized_avg_pool2d_int8_out
3636

37-
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
37+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
3838
variants: function
3939
kernels:
4040
- arg_meta: null

‎backends/espressif/passes/fold_activations.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717

1818
class FoldActivations(ExportedProgramPassBase):
19-
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
19+
"""Fold grouped clamps into the producer's output quantization."""
2020

2121
def call(self, exported_program):
2222
graph = exported_program.graph
@@ -43,6 +43,7 @@ def call(self, exported_program):
4343
exir_ops.edge.fused_quant.add.default,
4444
exir_ops.edge.fused_quant.sub.default,
4545
exir_ops.edge.fused_quant.mul.default,
46+
exir_ops.edge.fused_quant.linear.default,
4647
)
4748
or len(producer.users) != 1
4849
or get_qparams_from_node(producer, "out") is not None

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -402,7 +402,8 @@ def _linear_kernel_args(program, node, constants):
402402
"input and output must be nonempty, static, contiguous int8 tensors"
403403
)
404404
input_params, output_params = (
405-
_qparams(program, node, name) for name in ("inp", "out")
405+
_qparams(program, node, name, allow_output_clamp=True)
406+
for name in ("inp", "out")
406407
)
407408
if input_params is None or output_params is None:
408409
raise ValueError("input and output require constant per-tensor int8 qparams")
@@ -434,6 +435,8 @@ def _linear_kernel_args(program, node, constants):
434435
shift_node,
435436
-input_params.zp,
436437
output_params.zp,
438+
get_arg(node, "out_quant_min"),
439+
get_arg(node, "out_quant_max"),
437440
)
438441

439442

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
106106
Conv2d, and depthwise Conv2d biases use
107107
int32 with scales derived from the input and weight observers.
108108
AvgPool2d without divisor_override shares input and output qparams.
109-
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
110-
only their boundaries.
109+
Single-consumer binary or linear followed by ReLU or HardTanh (including
110+
ReLU6) patterns observe only their boundaries.
111111
EspPassManager lowers supported Fused Quant operators to ESP-NN and
112112
decomposes the rest for portable execution. Use EspNNQuantizer for the
113113
EspNNBackend delegate instead.
@@ -162,16 +162,19 @@ def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
162162
torch.ops.aten.add.Tensor,
163163
torch.ops.aten.sub.Tensor,
164164
torch.ops.aten.mul.Tensor,
165+
torch.ops.aten.linear.default,
165166
)
166167
or len(producer.users) != 1
167-
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
168168
or Q_ANNOTATION_KEY in producer.meta
169169
or Q_ANNOTATION_KEY in activation.meta
170170
):
171171
continue
172-
if (
173-
producer.target != torch.ops.aten.mul.Tensor
174-
and get_arg(producer, "alpha") != 1
172+
if producer.target != torch.ops.aten.linear.default and (
173+
not isinstance(get_arg(producer, "other"), torch.fx.Node)
174+
or (
175+
producer.target != torch.ops.aten.mul.Tensor
176+
and get_arg(producer, "alpha") != 1
177+
)
175178
):
176179
continue
177180
groups.append((producer, activation))

0 commit comments

Comments
 (0)